Merge branch 'ls/filter-process'
authorJunio C Hamano <gitster@pobox.com>
Mon, 31 Oct 2016 20:15:21 +0000 (13:15 -0700)
committerJunio C Hamano <gitster@pobox.com>
Mon, 31 Oct 2016 20:15:21 +0000 (13:15 -0700)
The smudge/clean filter API expect an external process is spawned
to filter the contents for each path that has a filter defined. A
new type of "process" filter API has been added to allow the first
request to run the filter for a path to spawn a single process, and
all filtering need is served by this single process for multiple
paths, reducing the process creation overhead.

* ls/filter-process:
contrib/long-running-filter: add long running filter example
convert: add filter.<driver>.process option
convert: prepare filter.<driver>.process option
convert: make apply_filter() adhere to standard Git error handling
pkt-line: add functions to read/write flush terminated packet streams
pkt-line: add packet_write_gently()
pkt-line: add packet_flush_gently()
pkt-line: add packet_write_fmt_gently()
pkt-line: extract set_packet_header()
pkt-line: rename packet_write() to packet_write_fmt()
run-command: add clean_on_exit_handler
run-command: move check_pipe() from write_or_die to run_command
convert: modernize tests
convert: quote filter names in error messages

1  2 
builtin/receive-pack.c
connect.c
convert.c
daemon.c
shallow.c
upload-pack.c
diff --combined builtin/receive-pack.c
index 680759d256a14917945a96dd16253606a88a5f8e,173d081647cc699bb17657176d730dc6b5139ea7..e6b3879a5b90034937dfb8eee32f8d2981f6bb53
@@@ -20,7 -20,6 +20,7 @@@
  #include "gpg-interface.h"
  #include "sigchain.h"
  #include "fsck.h"
 +#include "tmp-objdir.h"
  
  static const char * const receive_pack_usage[] = {
        N_("git receive-pack <git-dir>"),
@@@ -87,8 -86,6 +87,8 @@@ static enum 
  } use_keepalive;
  static int keepalive_in_sec = 5;
  
 +static struct tmp_objdir *tmp_objdir;
 +
  static enum deny_action parse_deny_action(const char *var, const char *value)
  {
        if (value) {
@@@ -227,7 -224,7 +227,7 @@@ static int receive_pack_config(const ch
  static void show_ref(const char *path, const unsigned char *sha1)
  {
        if (sent_capabilities) {
-               packet_write(1, "%s %s\n", sha1_to_hex(sha1), path);
+               packet_write_fmt(1, "%s %s\n", sha1_to_hex(sha1), path);
        } else {
                struct strbuf cap = STRBUF_INIT;
  
                if (advertise_push_options)
                        strbuf_addstr(&cap, " push-options");
                strbuf_addf(&cap, " agent=%s", git_user_agent_sanitized());
-               packet_write(1, "%s %s%c%s\n",
+               packet_write_fmt(1, "%s %s%c%s\n",
                             sha1_to_hex(sha1), path, 0, cap.buf);
                strbuf_release(&cap);
                sent_capabilities = 1;
@@@ -271,10 -268,9 +271,10 @@@ static int show_ref_cb(const char *path
        return 0;
  }
  
 -static void show_one_alternate_sha1(const unsigned char sha1[20], void *unused)
 +static int show_one_alternate_sha1(const unsigned char sha1[20], void *unused)
  {
        show_ref(".have", sha1);
 +      return 0;
  }
  
  static void collect_one_alternate_ref(const struct ref *ref, void *data)
@@@ -667,9 -663,6 +667,9 @@@ static int run_and_feed_hook(const cha
        } else
                argv_array_pushf(&proc.env_array, "GIT_PUSH_OPTION_COUNT");
  
 +      if (tmp_objdir)
 +              argv_array_pushv(&proc.env_array, tmp_objdir_env(tmp_objdir));
 +
        if (use_sideband) {
                memset(&muxer, 0, sizeof(muxer));
                muxer.proc = copy_to_sideband;
@@@ -769,7 -762,6 +769,7 @@@ static int run_update_hook(struct comma
        proc.stdout_to_stderr = 1;
        proc.err = use_sideband ? -1 : 0;
        proc.argv = argv;
 +      proc.env = tmp_objdir_env(tmp_objdir);
  
        code = start_command(&proc);
        if (code)
@@@ -789,39 -781,47 +789,39 @@@ static int is_ref_checked_out(const cha
        return !strcmp(head_name, ref);
  }
  
 -static char *refuse_unconfigured_deny_msg[] = {
 -      "By default, updating the current branch in a non-bare repository",
 -      "is denied, because it will make the index and work tree inconsistent",
 -      "with what you pushed, and will require 'git reset --hard' to match",
 -      "the work tree to HEAD.",
 -      "",
 -      "You can set 'receive.denyCurrentBranch' configuration variable to",
 -      "'ignore' or 'warn' in the remote repository to allow pushing into",
 -      "its current branch; however, this is not recommended unless you",
 -      "arranged to update its work tree to match what you pushed in some",
 -      "other way.",
 -      "",
 -      "To squelch this message and still keep the default behaviour, set",
 -      "'receive.denyCurrentBranch' configuration variable to 'refuse'."
 -};
 +static char *refuse_unconfigured_deny_msg =
 +      N_("By default, updating the current branch in a non-bare repository\n"
 +         "is denied, because it will make the index and work tree inconsistent\n"
 +         "with what you pushed, and will require 'git reset --hard' to match\n"
 +         "the work tree to HEAD.\n"
 +         "\n"
 +         "You can set 'receive.denyCurrentBranch' configuration variable to\n"
 +         "'ignore' or 'warn' in the remote repository to allow pushing into\n"
 +         "its current branch; however, this is not recommended unless you\n"
 +         "arranged to update its work tree to match what you pushed in some\n"
 +         "other way.\n"
 +         "\n"
 +         "To squelch this message and still keep the default behaviour, set\n"
 +         "'receive.denyCurrentBranch' configuration variable to 'refuse'.");
  
  static void refuse_unconfigured_deny(void)
  {
 -      int i;
 -      for (i = 0; i < ARRAY_SIZE(refuse_unconfigured_deny_msg); i++)
 -              rp_error("%s", refuse_unconfigured_deny_msg[i]);
 +      rp_error("%s", _(refuse_unconfigured_deny_msg));
  }
  
 -static char *refuse_unconfigured_deny_delete_current_msg[] = {
 -      "By default, deleting the current branch is denied, because the next",
 -      "'git clone' won't result in any file checked out, causing confusion.",
 -      "",
 -      "You can set 'receive.denyDeleteCurrent' configuration variable to",
 -      "'warn' or 'ignore' in the remote repository to allow deleting the",
 -      "current branch, with or without a warning message.",
 -      "",
 -      "To squelch this message, you can set it to 'refuse'."
 -};
 +static char *refuse_unconfigured_deny_delete_current_msg =
 +      N_("By default, deleting the current branch is denied, because the next\n"
 +         "'git clone' won't result in any file checked out, causing confusion.\n"
 +         "\n"
 +         "You can set 'receive.denyDeleteCurrent' configuration variable to\n"
 +         "'warn' or 'ignore' in the remote repository to allow deleting the\n"
 +         "current branch, with or without a warning message.\n"
 +         "\n"
 +         "To squelch this message, you can set it to 'refuse'.");
  
  static void refuse_unconfigured_deny_delete_current(void)
  {
 -      int i;
 -      for (i = 0;
 -           i < ARRAY_SIZE(refuse_unconfigured_deny_delete_current_msg);
 -           i++)
 -              rp_error("%s", refuse_unconfigured_deny_delete_current_msg[i]);
 +      rp_error("%s", _(refuse_unconfigured_deny_delete_current_msg));
  }
  
  static int command_singleton_iterator(void *cb_data, unsigned char sha1[20]);
@@@ -841,7 -841,6 +841,7 @@@ static int update_shallow_ref(struct co
                    !delayed_reachability_test(si, i))
                        sha1_array_append(&extra, si->shallow->sha1[i]);
  
 +      opt.env = tmp_objdir_env(tmp_objdir);
        setup_alternate_shallow(&shallow_lock, &opt.shallow_file, &extra);
        if (check_connected(command_singleton_iterator, cmd, &opt)) {
                rollback_lock_file(&shallow_lock);
@@@ -1163,6 -1162,10 +1163,6 @@@ static void check_aliased_update(struc
        struct string_list_item *item;
        struct command *dst_cmd;
        unsigned char sha1[GIT_SHA1_RAWSZ];
 -      char cmd_oldh[GIT_SHA1_HEXSZ + 1],
 -           cmd_newh[GIT_SHA1_HEXSZ + 1],
 -           dst_oldh[GIT_SHA1_HEXSZ + 1],
 -           dst_newh[GIT_SHA1_HEXSZ + 1];
        int flag;
  
        strbuf_addf(&buf, "%s%s", get_git_namespace(), cmd->ref_name);
  
        dst_cmd->skip_update = 1;
  
 -      find_unique_abbrev_r(cmd_oldh, cmd->old_sha1, DEFAULT_ABBREV);
 -      find_unique_abbrev_r(cmd_newh, cmd->new_sha1, DEFAULT_ABBREV);
 -      find_unique_abbrev_r(dst_oldh, dst_cmd->old_sha1, DEFAULT_ABBREV);
 -      find_unique_abbrev_r(dst_newh, dst_cmd->new_sha1, DEFAULT_ABBREV);
        rp_error("refusing inconsistent update between symref '%s' (%s..%s) and"
                 " its target '%s' (%s..%s)",
 -               cmd->ref_name, cmd_oldh, cmd_newh,
 -               dst_cmd->ref_name, dst_oldh, dst_newh);
 +               cmd->ref_name,
 +               find_unique_abbrev(cmd->old_sha1, DEFAULT_ABBREV),
 +               find_unique_abbrev(cmd->new_sha1, DEFAULT_ABBREV),
 +               dst_cmd->ref_name,
 +               find_unique_abbrev(dst_cmd->old_sha1, DEFAULT_ABBREV),
 +               find_unique_abbrev(dst_cmd->new_sha1, DEFAULT_ABBREV));
  
        cmd->error_string = dst_cmd->error_string =
                "inconsistent aliased update";
@@@ -1245,17 -1248,12 +1245,17 @@@ static void set_connectivity_errors(str
  
        for (cmd = commands; cmd; cmd = cmd->next) {
                struct command *singleton = cmd;
 +              struct check_connected_options opt = CHECK_CONNECTED_INIT;
 +
                if (shallow_update && si->shallow_ref[cmd->index])
                        /* to be checked in update_shallow_ref() */
                        continue;
 +
 +              opt.env = tmp_objdir_env(tmp_objdir);
                if (!check_connected(command_singleton_iterator, &singleton,
 -                                   NULL))
 +                                   &opt))
                        continue;
 +
                cmd->error_string = "missing necessary objects";
        }
  }
@@@ -1438,7 -1436,6 +1438,7 @@@ static void execute_commands(struct com
        data.si = si;
        opt.err_fd = err_fd;
        opt.progress = err_fd && !quiet;
 +      opt.env = tmp_objdir_env(tmp_objdir);
        if (check_connected(iterate_receive_command_list, &data, &opt))
                set_connectivity_errors(commands, si);
  
                return;
        }
  
 +      /*
 +       * Now we'll start writing out refs, which means the objects need
 +       * to be in their final positions so that other processes can see them.
 +       */
 +      if (tmp_objdir_migrate(tmp_objdir) < 0) {
 +              for (cmd = commands; cmd; cmd = cmd->next) {
 +                      if (!cmd->error_string)
 +                              cmd->error_string = "unable to migrate objects to permanent storage";
 +              }
 +              return;
 +      }
 +      tmp_objdir = NULL;
 +
        check_aliased_updates(commands);
  
        free(head_name_to_free);
@@@ -1663,18 -1647,6 +1663,18 @@@ static const char *unpack(int err_fd, s
                argv_array_push(&child.args, alt_shallow_file);
        }
  
 +      tmp_objdir = tmp_objdir_create();
 +      if (!tmp_objdir)
 +              return "unable to create temporary object directory";
 +      child.env = tmp_objdir_env(tmp_objdir);
 +
 +      /*
 +       * Normally we just pass the tmp_objdir environment to the child
 +       * processes that do the heavy lifting, but we may need to see these
 +       * objects ourselves to set up shallow information.
 +       */
 +      tmp_objdir_add_as_alternate(tmp_objdir);
 +
        if (ntohl(hdr.hdr_entries) < unpack_limit) {
                argv_array_pushl(&child.args, "unpack-objects", hdr_arg, NULL);
                if (quiet)
diff --combined connect.c
index d99d6435fd01754e3b0f23537d7e1b385a26f262,5330d9c1629607e335e056180794ab27b5b42277..8cb93b0720d9a33d8fc0cddb21c3d1c1a789da96
+++ b/connect.c
@@@ -43,14 -43,14 +43,14 @@@ int check_ref_type(const struct ref *re
        return check_ref(ref->name, flags);
  }
  
 -static void die_initial_contact(int got_at_least_one_head)
 +static void die_initial_contact(int unexpected)
  {
 -      if (got_at_least_one_head)
 -              die("The remote end hung up upon initial contact");
 +      if (unexpected)
 +              die(_("The remote end hung up upon initial contact"));
        else
 -              die("Could not read from remote repository.\n\n"
 -                  "Please make sure you have the correct access rights\n"
 -                  "and the repository exists.");
 +              die(_("Could not read from remote repository.\n\n"
 +                    "Please make sure you have the correct access rights\n"
 +                    "and the repository exists."));
  }
  
  static void parse_one_symref_info(struct string_list *symref, const char *val, int len)
@@@ -115,18 -115,10 +115,18 @@@ struct ref **get_remote_heads(int in, c
                              struct sha1_array *shallow_points)
  {
        struct ref **orig_list = list;
 -      int got_at_least_one_head = 0;
 +
 +      /*
 +       * A hang-up after seeing some response from the other end
 +       * means that it is unexpected, as we know the other end is
 +       * willing to talk to us.  A hang-up before seeing any
 +       * response does not necessarily mean an ACL problem, though.
 +       */
 +      int saw_response;
 +      int got_dummy_ref_with_capabilities_declaration = 0;
  
        *list = NULL;
 -      for (;;) {
 +      for (saw_response = 0; ; saw_response = 1) {
                struct ref *ref;
                struct object_id old_oid;
                char *name;
                                  PACKET_READ_GENTLE_ON_EOF |
                                  PACKET_READ_CHOMP_NEWLINE);
                if (len < 0)
 -                      die_initial_contact(got_at_least_one_head);
 +                      die_initial_contact(saw_response);
  
                if (!len)
                        break;
                        continue;
                }
  
 +              if (!strcmp(name, "capabilities^{}")) {
 +                      if (saw_response)
 +                              die("protocol error: unexpected capabilities^{}");
 +                      if (got_dummy_ref_with_capabilities_declaration)
 +                              die("protocol error: multiple capabilities^{}");
 +                      got_dummy_ref_with_capabilities_declaration = 1;
 +                      continue;
 +              }
 +
                if (!check_ref(name, flags))
                        continue;
 +
 +              if (got_dummy_ref_with_capabilities_declaration)
 +                      die("protocol error: unexpected ref after capabilities^{}");
 +
                ref = alloc_ref(buffer + GIT_SHA1_HEXSZ + 1);
                oidcpy(&ref->old_oid, &old_oid);
                *list = ref;
                list = &ref->next;
 -              got_at_least_one_head = 1;
        }
  
        annotate_refs_with_symref_info(*orig_list);
@@@ -750,7 -730,7 +750,7 @@@ struct child_process *git_connect(int f
                 * Note: Do not add any other headers here!  Doing so
                 * will cause older git-daemon servers to crash.
                 */
-               packet_write(fd[1],
+               packet_write_fmt(fd[1],
                             "%s %s%chost=%s%c",
                             prog, path, 0,
                             target_host, 0);
diff --combined convert.c
index 0ad39b16cc40c27ef9af493d95542c52c60439a0,bc242276ff095845f5a51d7d6ec3a9e97f02ddd0..be91358462a72c9d5a6c02f2b0e44585a76d4783
+++ b/convert.c
@@@ -3,6 -3,7 +3,7 @@@
  #include "run-command.h"
  #include "quote.h"
  #include "sigchain.h"
+ #include "pkt-line.h"
  
  /*
   * convert.c - convert a file when checking it out and checking it in.
@@@ -197,21 -198,17 +198,21 @@@ static void check_safe_crlf(const char 
                 * CRLFs would not be restored by checkout
                 */
                if (checksafe == SAFE_CRLF_WARN)
 -                      warning("CRLF will be replaced by LF in %s.\nThe file will have its original line endings in your working directory.", path);
 +                      warning(_("CRLF will be replaced by LF in %s.\n"
 +                                "The file will have its original line"
 +                                " endings in your working directory."), path);
                else /* i.e. SAFE_CRLF_FAIL */
 -                      die("CRLF would be replaced by LF in %s.", path);
 +                      die(_("CRLF would be replaced by LF in %s."), path);
        } else if (old_stats->lonelf && !new_stats->lonelf ) {
                /*
                 * CRLFs would be added by checkout
                 */
                if (checksafe == SAFE_CRLF_WARN)
 -                      warning("LF will be replaced by CRLF in %s.\nThe file will have its original line endings in your working directory.", path);
 +                      warning(_("LF will be replaced by CRLF in %s.\n"
 +                                "The file will have its original line"
 +                                " endings in your working directory."), path);
                else /* i.e. SAFE_CRLF_FAIL */
 -                      die("LF would be replaced by CRLF in %s", path);
 +                      die(_("LF would be replaced by CRLF in %s"), path);
        }
  }
  
@@@ -416,7 -413,7 +417,7 @@@ static int filter_buffer_or_fd(int in, 
        child_process.out = out;
  
        if (start_command(&child_process))
-               return error("cannot fork to run external filter %s", params->cmd);
+               return error("cannot fork to run external filter '%s'", params->cmd);
  
        sigchain_push(SIGPIPE, SIG_IGN);
  
        if (close(child_process.in))
                write_err = 1;
        if (write_err)
-               error("cannot feed the input to external filter %s", params->cmd);
+               error("cannot feed the input to external filter '%s'", params->cmd);
  
        sigchain_pop(SIGPIPE);
  
        status = finish_command(&child_process);
        if (status)
-               error("external filter %s failed %d", params->cmd, status);
+               error("external filter '%s' failed %d", params->cmd, status);
  
        strbuf_release(&cmd);
        return (write_err || status);
  }
  
- static int apply_filter(const char *path, const char *src, size_t len, int fd,
+ static int apply_single_file_filter(const char *path, const char *src, size_t len, int fd,
                          struct strbuf *dst, const char *cmd)
  {
        /*
         *
         * (child --> cmd) --> us
         */
-       int ret = 1;
+       int err = 0;
        struct strbuf nbuf = STRBUF_INIT;
        struct async async;
        struct filter_params params;
  
-       if (!cmd || !*cmd)
-               return 0;
-       if (!dst)
-               return 1;
        memset(&async, 0, sizeof(async));
        async.proc = filter_buffer_or_fd;
        async.data = &params;
                return 0;       /* error was already reported */
  
        if (strbuf_read(&nbuf, async.out, len) < 0) {
-               error("read from external filter %s failed", cmd);
-               ret = 0;
+               err = error("read from external filter '%s' failed", cmd);
        }
        if (close(async.out)) {
-               error("read from external filter %s failed", cmd);
-               ret = 0;
+               err = error("read from external filter '%s' failed", cmd);
        }
        if (finish_async(&async)) {
-               error("external filter %s failed", cmd);
-               ret = 0;
+               err = error("external filter '%s' failed", cmd);
        }
  
-       if (ret) {
+       if (!err) {
                strbuf_swap(dst, &nbuf);
        }
        strbuf_release(&nbuf);
-       return ret;
+       return !err;
+ }
+ #define CAP_CLEAN    (1u<<0)
+ #define CAP_SMUDGE   (1u<<1)
+ struct cmd2process {
+       struct hashmap_entry ent; /* must be the first member! */
+       unsigned int supported_capabilities;
+       const char *cmd;
+       struct child_process process;
+ };
+ static int cmd_process_map_initialized;
+ static struct hashmap cmd_process_map;
+ static int cmd2process_cmp(const struct cmd2process *e1,
+                          const struct cmd2process *e2,
+                          const void *unused)
+ {
+       return strcmp(e1->cmd, e2->cmd);
+ }
+ static struct cmd2process *find_multi_file_filter_entry(struct hashmap *hashmap, const char *cmd)
+ {
+       struct cmd2process key;
+       hashmap_entry_init(&key, strhash(cmd));
+       key.cmd = cmd;
+       return hashmap_get(hashmap, &key, NULL);
+ }
+ static int packet_write_list(int fd, const char *line, ...)
+ {
+       va_list args;
+       int err;
+       va_start(args, line);
+       for (;;) {
+               if (!line)
+                       break;
+               if (strlen(line) > LARGE_PACKET_DATA_MAX)
+                       return -1;
+               err = packet_write_fmt_gently(fd, "%s\n", line);
+               if (err)
+                       return err;
+               line = va_arg(args, const char*);
+       }
+       va_end(args);
+       return packet_flush_gently(fd);
+ }
+ static void read_multi_file_filter_status(int fd, struct strbuf *status)
+ {
+       struct strbuf **pair;
+       char *line;
+       for (;;) {
+               line = packet_read_line(fd, NULL);
+               if (!line)
+                       break;
+               pair = strbuf_split_str(line, '=', 2);
+               if (pair[0] && pair[0]->len && pair[1]) {
+                       /* the last "status=<foo>" line wins */
+                       if (!strcmp(pair[0]->buf, "status=")) {
+                               strbuf_reset(status);
+                               strbuf_addbuf(status, pair[1]);
+                       }
+               }
+               strbuf_list_free(pair);
+       }
+ }
+ static void kill_multi_file_filter(struct hashmap *hashmap, struct cmd2process *entry)
+ {
+       if (!entry)
+               return;
+       entry->process.clean_on_exit = 0;
+       kill(entry->process.pid, SIGTERM);
+       finish_command(&entry->process);
+       hashmap_remove(hashmap, entry, NULL);
+       free(entry);
+ }
+ static void stop_multi_file_filter(struct child_process *process)
+ {
+       sigchain_push(SIGPIPE, SIG_IGN);
+       /* Closing the pipe signals the filter to initiate a shutdown. */
+       close(process->in);
+       close(process->out);
+       sigchain_pop(SIGPIPE);
+       /* Finish command will wait until the shutdown is complete. */
+       finish_command(process);
+ }
+ static struct cmd2process *start_multi_file_filter(struct hashmap *hashmap, const char *cmd)
+ {
+       int err;
+       struct cmd2process *entry;
+       struct child_process *process;
+       const char *argv[] = { cmd, NULL };
+       struct string_list cap_list = STRING_LIST_INIT_NODUP;
+       char *cap_buf;
+       const char *cap_name;
+       entry = xmalloc(sizeof(*entry));
+       entry->cmd = cmd;
+       entry->supported_capabilities = 0;
+       process = &entry->process;
+       child_process_init(process);
+       process->argv = argv;
+       process->use_shell = 1;
+       process->in = -1;
+       process->out = -1;
+       process->clean_on_exit = 1;
+       process->clean_on_exit_handler = stop_multi_file_filter;
+       if (start_command(process)) {
+               error("cannot fork to run external filter '%s'", cmd);
+               return NULL;
+       }
+       hashmap_entry_init(entry, strhash(cmd));
+       sigchain_push(SIGPIPE, SIG_IGN);
+       err = packet_write_list(process->in, "git-filter-client", "version=2", NULL);
+       if (err)
+               goto done;
+       err = strcmp(packet_read_line(process->out, NULL), "git-filter-server");
+       if (err) {
+               error("external filter '%s' does not support filter protocol version 2", cmd);
+               goto done;
+       }
+       err = strcmp(packet_read_line(process->out, NULL), "version=2");
+       if (err)
+               goto done;
+       err = packet_read_line(process->out, NULL) != NULL;
+       if (err)
+               goto done;
+       err = packet_write_list(process->in, "capability=clean", "capability=smudge", NULL);
+       for (;;) {
+               cap_buf = packet_read_line(process->out, NULL);
+               if (!cap_buf)
+                       break;
+               string_list_split_in_place(&cap_list, cap_buf, '=', 1);
+               if (cap_list.nr != 2 || strcmp(cap_list.items[0].string, "capability"))
+                       continue;
+               cap_name = cap_list.items[1].string;
+               if (!strcmp(cap_name, "clean")) {
+                       entry->supported_capabilities |= CAP_CLEAN;
+               } else if (!strcmp(cap_name, "smudge")) {
+                       entry->supported_capabilities |= CAP_SMUDGE;
+               } else {
+                       warning(
+                               "external filter '%s' requested unsupported filter capability '%s'",
+                               cmd, cap_name
+                       );
+               }
+               string_list_clear(&cap_list, 0);
+       }
+ done:
+       sigchain_pop(SIGPIPE);
+       if (err || errno == EPIPE) {
+               error("initialization for external filter '%s' failed", cmd);
+               kill_multi_file_filter(hashmap, entry);
+               return NULL;
+       }
+       hashmap_add(hashmap, entry);
+       return entry;
+ }
+ static int apply_multi_file_filter(const char *path, const char *src, size_t len,
+                                  int fd, struct strbuf *dst, const char *cmd,
+                                  const unsigned int wanted_capability)
+ {
+       int err;
+       struct cmd2process *entry;
+       struct child_process *process;
+       struct strbuf nbuf = STRBUF_INIT;
+       struct strbuf filter_status = STRBUF_INIT;
+       const char *filter_type;
+       if (!cmd_process_map_initialized) {
+               cmd_process_map_initialized = 1;
+               hashmap_init(&cmd_process_map, (hashmap_cmp_fn) cmd2process_cmp, 0);
+               entry = NULL;
+       } else {
+               entry = find_multi_file_filter_entry(&cmd_process_map, cmd);
+       }
+       fflush(NULL);
+       if (!entry) {
+               entry = start_multi_file_filter(&cmd_process_map, cmd);
+               if (!entry)
+                       return 0;
+       }
+       process = &entry->process;
+       if (!(wanted_capability & entry->supported_capabilities))
+               return 0;
+       if (CAP_CLEAN & wanted_capability)
+               filter_type = "clean";
+       else if (CAP_SMUDGE & wanted_capability)
+               filter_type = "smudge";
+       else
+               die("unexpected filter type");
+       sigchain_push(SIGPIPE, SIG_IGN);
+       assert(strlen(filter_type) < LARGE_PACKET_DATA_MAX - strlen("command=\n"));
+       err = packet_write_fmt_gently(process->in, "command=%s\n", filter_type);
+       if (err)
+               goto done;
+       err = strlen(path) > LARGE_PACKET_DATA_MAX - strlen("pathname=\n");
+       if (err) {
+               error("path name too long for external filter");
+               goto done;
+       }
+       err = packet_write_fmt_gently(process->in, "pathname=%s\n", path);
+       if (err)
+               goto done;
+       err = packet_flush_gently(process->in);
+       if (err)
+               goto done;
+       if (fd >= 0)
+               err = write_packetized_from_fd(fd, process->in);
+       else
+               err = write_packetized_from_buf(src, len, process->in);
+       if (err)
+               goto done;
+       read_multi_file_filter_status(process->out, &filter_status);
+       err = strcmp(filter_status.buf, "success");
+       if (err)
+               goto done;
+       err = read_packetized_to_strbuf(process->out, &nbuf) < 0;
+       if (err)
+               goto done;
+       read_multi_file_filter_status(process->out, &filter_status);
+       err = strcmp(filter_status.buf, "success");
+ done:
+       sigchain_pop(SIGPIPE);
+       if (err || errno == EPIPE) {
+               if (!strcmp(filter_status.buf, "error")) {
+                       /* The filter signaled a problem with the file. */
+               } else if (!strcmp(filter_status.buf, "abort")) {
+                       /*
+                        * The filter signaled a permanent problem. Don't try to filter
+                        * files with the same command for the lifetime of the current
+                        * Git process.
+                        */
+                        entry->supported_capabilities &= ~wanted_capability;
+               } else {
+                       /*
+                        * Something went wrong with the protocol filter.
+                        * Force shutdown and restart if another blob requires filtering.
+                        */
+                       error("external filter '%s' failed", cmd);
+                       kill_multi_file_filter(&cmd_process_map, entry);
+               }
+       } else {
+               strbuf_swap(dst, &nbuf);
+       }
+       strbuf_release(&nbuf);
+       return !err;
  }
  
  static struct convert_driver {
        struct convert_driver *next;
        const char *smudge;
        const char *clean;
+       const char *process;
        int required;
  } *user_convert, **user_convert_tail;
  
+ static int apply_filter(const char *path, const char *src, size_t len,
+                       int fd, struct strbuf *dst, struct convert_driver *drv,
+                       const unsigned int wanted_capability)
+ {
+       const char *cmd = NULL;
+       if (!drv)
+               return 0;
+       if (!dst)
+               return 1;
+       if ((CAP_CLEAN & wanted_capability) && !drv->process && drv->clean)
+               cmd = drv->clean;
+       else if ((CAP_SMUDGE & wanted_capability) && !drv->process && drv->smudge)
+               cmd = drv->smudge;
+       if (cmd && *cmd)
+               return apply_single_file_filter(path, src, len, fd, dst, cmd);
+       else if (drv->process && *drv->process)
+               return apply_multi_file_filter(path, src, len, fd, dst, drv->process, wanted_capability);
+       return 0;
+ }
  static int read_convert_config(const char *var, const char *value, void *cb)
  {
        const char *key, *name;
        if (!strcmp("clean", key))
                return git_config_string(&drv->clean, var, value);
  
+       if (!strcmp("process", key))
+               return git_config_string(&drv->process, var, value);
        if (!strcmp("required", key)) {
                drv->required = git_config_bool(var, value);
                return 0;
@@@ -846,7 -1147,7 +1151,7 @@@ int would_convert_to_git_filter_fd(cons
        if (!ca.drv->required)
                return 0;
  
-       return apply_filter(path, NULL, 0, -1, NULL, ca.drv->clean);
+       return apply_filter(path, NULL, 0, -1, NULL, ca.drv, CAP_CLEAN);
  }
  
  const char *get_convert_attr_ascii(const char *path)
@@@ -879,18 -1180,12 +1184,12 @@@ int convert_to_git(const char *path, co
                     struct strbuf *dst, enum safe_crlf checksafe)
  {
        int ret = 0;
-       const char *filter = NULL;
-       int required = 0;
        struct conv_attrs ca;
  
        convert_attrs(&ca, path);
-       if (ca.drv) {
-               filter = ca.drv->clean;
-               required = ca.drv->required;
-       }
  
-       ret |= apply_filter(path, src, len, -1, dst, filter);
-       if (!ret && required)
+       ret |= apply_filter(path, src, len, -1, dst, ca.drv, CAP_CLEAN);
+       if (!ret && ca.drv && ca.drv->required)
                die("%s: clean filter '%s' failed", path, ca.drv->name);
  
        if (ret && dst) {
@@@ -912,9 -1207,9 +1211,9 @@@ void convert_to_git_filter_fd(const cha
        convert_attrs(&ca, path);
  
        assert(ca.drv);
-       assert(ca.drv->clean);
+       assert(ca.drv->clean || ca.drv->process);
  
-       if (!apply_filter(path, NULL, 0, fd, dst, ca.drv->clean))
+       if (!apply_filter(path, NULL, 0, fd, dst, ca.drv, CAP_CLEAN))
                die("%s: clean filter '%s' failed", path, ca.drv->name);
  
        crlf_to_git(path, dst->buf, dst->len, dst, ca.crlf_action, checksafe);
@@@ -926,15 -1221,9 +1225,9 @@@ static int convert_to_working_tree_inte
                                            int normalizing)
  {
        int ret = 0, ret_filter = 0;
-       const char *filter = NULL;
-       int required = 0;
        struct conv_attrs ca;
  
        convert_attrs(&ca, path);
-       if (ca.drv) {
-               filter = ca.drv->smudge;
-               required = ca.drv->required;
-       }
  
        ret |= ident_to_worktree(path, src, len, dst, ca.ident);
        if (ret) {
        }
        /*
         * CRLF conversion can be skipped if normalizing, unless there
-        * is a smudge filter.  The filter might expect CRLFs.
+        * is a smudge or process filter (even if the process filter doesn't
+        * support smudge).  The filters might expect CRLFs.
         */
-       if (filter || !normalizing) {
+       if ((ca.drv && (ca.drv->smudge || ca.drv->process)) || !normalizing) {
                ret |= crlf_to_worktree(path, src, len, dst, ca.crlf_action);
                if (ret) {
                        src = dst->buf;
                }
        }
  
-       ret_filter = apply_filter(path, src, len, -1, dst, filter);
-       if (!ret_filter && required)
+       ret_filter = apply_filter(path, src, len, -1, dst, ca.drv, CAP_SMUDGE);
+       if (!ret_filter && ca.drv && ca.drv->required)
                die("%s: smudge filter %s failed", path, ca.drv->name);
  
        return ret | ret_filter;
@@@ -1406,7 -1696,7 +1700,7 @@@ struct stream_filter *get_stream_filter
        struct stream_filter *filter = NULL;
  
        convert_attrs(&ca, path);
-       if (ca.drv && (ca.drv->smudge || ca.drv->clean))
+       if (ca.drv && (ca.drv->process || ca.drv->smudge || ca.drv->clean))
                return NULL;
  
        if (ca.crlf_action == CRLF_AUTO || ca.crlf_action == CRLF_AUTO_CRLF)
diff --combined daemon.c
index ff0fa583b0497d886c54ff93d97cc45fd909e7a5,afce1b9b7f8d2846a8c99855f1e20d0398a31c47..473e6b6b63c42e59d5a89985b2573ab5c2815157
+++ b/daemon.c
@@@ -160,7 -160,6 +160,7 @@@ static const char *path_ok(const char *
  {
        static char rpath[PATH_MAX];
        static char interp_path[PATH_MAX];
 +      size_t rlen;
        const char *path;
        const char *dir;
  
                        namlen = slash - dir;
                        restlen -= namlen;
                        loginfo("userpath <%s>, request <%s>, namlen %d, restlen %d, slash <%s>", user_path, dir, namlen, restlen, slash);
 -                      snprintf(rpath, PATH_MAX, "%.*s/%s%.*s",
 -                               namlen, dir, user_path, restlen, slash);
 +                      rlen = snprintf(rpath, sizeof(rpath), "%.*s/%s%.*s",
 +                                      namlen, dir, user_path, restlen, slash);
 +                      if (rlen >= sizeof(rpath)) {
 +                              logerror("user-path too large: %s", rpath);
 +                              return NULL;
 +                      }
                        dir = rpath;
                }
        }
  
                strbuf_expand(&expanded_path, interpolated_path,
                              expand_path, &context);
 -              strlcpy(interp_path, expanded_path.buf, PATH_MAX);
 +
 +              rlen = strlcpy(interp_path, expanded_path.buf,
 +                             sizeof(interp_path));
 +              if (rlen >= sizeof(interp_path)) {
 +                      logerror("interpolated path too large: %s",
 +                               interp_path);
 +                      return NULL;
 +              }
 +
                strbuf_release(&expanded_path);
                loginfo("Interpolated dir '%s'", interp_path);
  
                        logerror("'%s': Non-absolute path denied (base-path active)", dir);
                        return NULL;
                }
 -              snprintf(rpath, PATH_MAX, "%s%s", base_path, dir);
 +              rlen = snprintf(rpath, sizeof(rpath), "%s%s", base_path, dir);
 +              if (rlen >= sizeof(rpath)) {
 +                      logerror("base-path too large: %s", rpath);
 +                      return NULL;
 +              }
                dir = rpath;
        }
  
@@@ -298,7 -281,7 +298,7 @@@ static int daemon_error(const char *dir
  {
        if (!informative_errors)
                msg = "access denied or repository not exported";
-       packet_write(1, "ERR %s: %s", msg, dir);
+       packet_write_fmt(1, "ERR %s: %s", msg, dir);
        return -1;
  }
  
diff --combined shallow.c
index 2531e3af3b0d87e1f78fe09208cb85917ceff207,d666e24f1255738670d3251c82595e133a9c19ff..4d0b005d39c1c1ab2379911666d4df75d66b824c
+++ b/shallow.c
@@@ -10,8 -10,6 +10,8 @@@
  #include "diff.h"
  #include "revision.h"
  #include "commit-slab.h"
 +#include "revision.h"
 +#include "list-objects.h"
  
  static int is_shallow = -1;
  static struct stat_validity shallow_stat;
@@@ -139,82 -137,6 +139,82 @@@ struct commit_list *get_shallow_commits
        return result;
  }
  
 +static void show_commit(struct commit *commit, void *data)
 +{
 +      commit_list_insert(commit, data);
 +}
 +
 +/*
 + * Given rev-list arguments, run rev-list. All reachable commits
 + * except border ones are marked with not_shallow_flag. Border commits
 + * are marked with shallow_flag. The list of border/shallow commits
 + * are also returned.
 + */
 +struct commit_list *get_shallow_commits_by_rev_list(int ac, const char **av,
 +                                                  int shallow_flag,
 +                                                  int not_shallow_flag)
 +{
 +      struct commit_list *result = NULL, *p;
 +      struct commit_list *not_shallow_list = NULL;
 +      struct rev_info revs;
 +      int both_flags = shallow_flag | not_shallow_flag;
 +
 +      /*
 +       * SHALLOW (excluded) and NOT_SHALLOW (included) should not be
 +       * set at this point. But better be safe than sorry.
 +       */
 +      clear_object_flags(both_flags);
 +
 +      is_repository_shallow(); /* make sure shallows are read */
 +
 +      init_revisions(&revs, NULL);
 +      save_commit_buffer = 0;
 +      setup_revisions(ac, av, &revs, NULL);
 +
 +      if (prepare_revision_walk(&revs))
 +              die("revision walk setup failed");
 +      traverse_commit_list(&revs, show_commit, NULL, &not_shallow_list);
 +
 +      /* Mark all reachable commits as NOT_SHALLOW */
 +      for (p = not_shallow_list; p; p = p->next)
 +              p->item->object.flags |= not_shallow_flag;
 +
 +      /*
 +       * mark border commits SHALLOW + NOT_SHALLOW.
 +       * We cannot clear NOT_SHALLOW right now. Imagine border
 +       * commit A is processed first, then commit B, whose parent is
 +       * A, later. If NOT_SHALLOW on A is cleared at step 1, B
 +       * itself is considered border at step 2, which is incorrect.
 +       */
 +      for (p = not_shallow_list; p; p = p->next) {
 +              struct commit *c = p->item;
 +              struct commit_list *parent;
 +
 +              if (parse_commit(c))
 +                      die("unable to parse commit %s",
 +                          oid_to_hex(&c->object.oid));
 +
 +              for (parent = c->parents; parent; parent = parent->next)
 +                      if (!(parent->item->object.flags & not_shallow_flag)) {
 +                              c->object.flags |= shallow_flag;
 +                              commit_list_insert(c, &result);
 +                              break;
 +                      }
 +      }
 +      free_commit_list(not_shallow_list);
 +
 +      /*
 +       * Now we can clean up NOT_SHALLOW on border commits. Having
 +       * both flags set can confuse the caller.
 +       */
 +      for (p = result; p; p = p->next) {
 +              struct object *o = &p->item->object;
 +              if ((o->flags & both_flags) == both_flags)
 +                      o->flags &= ~not_shallow_flag;
 +      }
 +      return result;
 +}
 +
  static void check_shallow_file_for_update(void)
  {
        if (is_shallow == -1)
@@@ -338,7 -260,7 +338,7 @@@ static int advertise_shallow_grafts_cb(
  {
        int fd = *(int *)cb;
        if (graft->nr_parent == -1)
-               packet_write(fd, "shallow %s\n", oid_to_hex(&graft->oid));
+               packet_write_fmt(fd, "shallow %s\n", oid_to_hex(&graft->oid));
        return 0;
  }
  
diff --combined upload-pack.c
index d9e381f29126313aa7068478430dd5e3f704f9ef,cd47de69ab4da0cdb82fe71cda9a82ff0334561a..e0db8b42be2d6ba400da987f8cb2ba573c5eda01
@@@ -15,8 -15,6 +15,8 @@@
  #include "version.h"
  #include "string-list.h"
  #include "parse-options.h"
 +#include "argv-array.h"
 +#include "prio-queue.h"
  
  static const char * const upload_pack_usage[] = {
        N_("git upload-pack [<options>] <dir>"),
@@@ -37,7 -35,6 +37,7 @@@
  
  static unsigned long oldest_have;
  
 +static int deepen_relative;
  static int multi_ack;
  static int no_done;
  static int use_thin_pack, use_ofs_delta, use_include_tag;
@@@ -284,7 -281,7 +284,7 @@@ static void create_pack_file(void
        die("git upload-pack: %s", abort_msg);
  }
  
 -static int got_sha1(char *hex, unsigned char *sha1)
 +static int got_sha1(const char *hex, unsigned char *sha1)
  {
        struct object *o;
        int we_knew_they_have = 0;
  
  static int reachable(struct commit *want)
  {
 -      struct commit_list *work = NULL;
 +      struct prio_queue work = { compare_commits_by_commit_date };
  
 -      commit_list_insert_by_date(want, &work);
 -      while (work) {
 +      prio_queue_put(&work, want);
 +      while (work.nr) {
                struct commit_list *list;
 -              struct commit *commit = pop_commit(&work);
 +              struct commit *commit = prio_queue_get(&work);
  
                if (commit->object.flags & THEY_HAVE) {
                        want->object.flags |= COMMON_KNOWN;
                for (list = commit->parents; list; list = list->next) {
                        struct commit *parent = list->item;
                        if (!(parent->object.flags & REACHABLE))
 -                              commit_list_insert_by_date(parent, &work);
 +                              prio_queue_put(&work, parent);
                }
        }
        want->object.flags |= REACHABLE;
        clear_commit_marks(want, REACHABLE);
 -      free_commit_list(work);
 +      clear_prio_queue(&work);
        return (want->object.flags & COMMON_KNOWN);
  }
  
@@@ -390,21 -387,19 +390,21 @@@ static int get_common_commits(void
  
        for (;;) {
                char *line = packet_read_line(0, NULL);
 +              const char *arg;
 +
                reset_timeout();
  
                if (!line) {
                        if (multi_ack == 2 && got_common
                            && !got_other && ok_to_give_up()) {
                                sent_ready = 1;
-                               packet_write(1, "ACK %s ready\n", last_hex);
+                               packet_write_fmt(1, "ACK %s ready\n", last_hex);
                        }
                        if (have_obj.nr == 0 || multi_ack)
-                               packet_write(1, "NAK\n");
+                               packet_write_fmt(1, "NAK\n");
  
                        if (no_done && sent_ready) {
-                               packet_write(1, "ACK %s\n", last_hex);
+                               packet_write_fmt(1, "ACK %s\n", last_hex);
                                return 0;
                        }
                        if (stateless_rpc)
                        got_other = 0;
                        continue;
                }
 -              if (starts_with(line, "have ")) {
 -                      switch (got_sha1(line+5, sha1)) {
 +              if (skip_prefix(line, "have ", &arg)) {
 +                      switch (got_sha1(arg, sha1)) {
                        case -1: /* they have what we do not */
                                got_other = 1;
                                if (multi_ack && ok_to_give_up()) {
                                        const char *hex = sha1_to_hex(sha1);
                                        if (multi_ack == 2) {
                                                sent_ready = 1;
-                                               packet_write(1, "ACK %s ready\n", hex);
+                                               packet_write_fmt(1, "ACK %s ready\n", hex);
                                        } else
-                                               packet_write(1, "ACK %s continue\n", hex);
+                                               packet_write_fmt(1, "ACK %s continue\n", hex);
                                }
                                break;
                        default:
                                got_common = 1;
                                memcpy(last_hex, sha1_to_hex(sha1), 41);
                                if (multi_ack == 2)
-                                       packet_write(1, "ACK %s common\n", last_hex);
+                                       packet_write_fmt(1, "ACK %s common\n", last_hex);
                                else if (multi_ack)
-                                       packet_write(1, "ACK %s continue\n", last_hex);
+                                       packet_write_fmt(1, "ACK %s continue\n", last_hex);
                                else if (have_obj.nr == 1)
-                                       packet_write(1, "ACK %s\n", last_hex);
+                                       packet_write_fmt(1, "ACK %s\n", last_hex);
                                break;
                        }
                        continue;
                if (!strcmp(line, "done")) {
                        if (have_obj.nr > 0) {
                                if (multi_ack)
-                                       packet_write(1, "ACK %s\n", last_hex);
+                                       packet_write_fmt(1, "ACK %s\n", last_hex);
                                return 0;
                        }
-                       packet_write(1, "NAK\n");
+                       packet_write_fmt(1, "NAK\n");
                        return -1;
                }
                die("git upload-pack: expected SHA1 list, got '%s'", line);
@@@ -459,136 -454,73 +459,136 @@@ static int is_our_ref(struct object *o
        return o->flags & ((allow_hidden_ref ? HIDDEN_REF : 0) | OUR_REF);
  }
  
 -static void check_non_tip(void)
 +/*
 + * on successful case, it's up to the caller to close cmd->out
 + */
 +static int do_reachable_revlist(struct child_process *cmd,
 +                              struct object_array *src,
 +                              struct object_array *reachable)
  {
        static const char *argv[] = {
                "rev-list", "--stdin", NULL,
        };
 -      static struct child_process cmd = CHILD_PROCESS_INIT;
        struct object *o;
        char namebuf[42]; /* ^ + SHA-1 + LF */
        int i;
  
 -      /*
 -       * In the normal in-process case without
 -       * uploadpack.allowReachableSHA1InWant,
 -       * non-tip requests can never happen.
 -       */
 -      if (!stateless_rpc && !(allow_unadvertised_object_request & ALLOW_REACHABLE_SHA1))
 -              goto error;
 -
 -      cmd.argv = argv;
 -      cmd.git_cmd = 1;
 -      cmd.no_stderr = 1;
 -      cmd.in = -1;
 -      cmd.out = -1;
 -
 -      if (start_command(&cmd))
 -              goto error;
 +      cmd->argv = argv;
 +      cmd->git_cmd = 1;
 +      cmd->no_stderr = 1;
 +      cmd->in = -1;
 +      cmd->out = -1;
  
        /*
 -       * If rev-list --stdin encounters an unknown commit, it
 -       * terminates, which will cause SIGPIPE in the write loop
 +       * If the next rev-list --stdin encounters an unknown commit,
 +       * it terminates, which will cause SIGPIPE in the write loop
         * below.
         */
        sigchain_push(SIGPIPE, SIG_IGN);
  
 +      if (start_command(cmd))
 +              goto error;
 +
        namebuf[0] = '^';
        namebuf[41] = '\n';
        for (i = get_max_object_index(); 0 < i; ) {
                o = get_indexed_object(--i);
                if (!o)
                        continue;
 +              if (reachable && o->type == OBJ_COMMIT)
 +                      o->flags &= ~TMP_MARK;
                if (!is_our_ref(o))
                        continue;
                memcpy(namebuf + 1, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ);
 -              if (write_in_full(cmd.in, namebuf, 42) < 0)
 +              if (write_in_full(cmd->in, namebuf, 42) < 0)
                        goto error;
        }
        namebuf[40] = '\n';
 -      for (i = 0; i < want_obj.nr; i++) {
 -              o = want_obj.objects[i].item;
 -              if (is_our_ref(o))
 +      for (i = 0; i < src->nr; i++) {
 +              o = src->objects[i].item;
 +              if (is_our_ref(o)) {
 +                      if (reachable)
 +                              add_object_array(o, NULL, reachable);
                        continue;
 +              }
 +              if (reachable && o->type == OBJ_COMMIT)
 +                      o->flags |= TMP_MARK;
                memcpy(namebuf, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ);
 -              if (write_in_full(cmd.in, namebuf, 41) < 0)
 +              if (write_in_full(cmd->in, namebuf, 41) < 0)
                        goto error;
        }
 -      close(cmd.in);
 +      close(cmd->in);
 +      cmd->in = -1;
 +      sigchain_pop(SIGPIPE);
  
 +      return 0;
 +
 +error:
        sigchain_pop(SIGPIPE);
  
 +      if (cmd->in >= 0)
 +              close(cmd->in);
 +      if (cmd->out >= 0)
 +              close(cmd->out);
 +      return -1;
 +}
 +
 +static int get_reachable_list(struct object_array *src,
 +                            struct object_array *reachable)
 +{
 +      struct child_process cmd = CHILD_PROCESS_INIT;
 +      int i;
 +      struct object *o;
 +      char namebuf[42]; /* ^ + SHA-1 + LF */
 +
 +      if (do_reachable_revlist(&cmd, src, reachable) < 0)
 +              return -1;
 +
 +      while ((i = read_in_full(cmd.out, namebuf, 41)) == 41) {
 +              struct object_id sha1;
 +
 +              if (namebuf[40] != '\n' || get_oid_hex(namebuf, &sha1))
 +                      break;
 +
 +              o = lookup_object(sha1.hash);
 +              if (o && o->type == OBJ_COMMIT) {
 +                      o->flags &= ~TMP_MARK;
 +              }
 +      }
 +      for (i = get_max_object_index(); 0 < i; i--) {
 +              o = get_indexed_object(i - 1);
 +              if (o && o->type == OBJ_COMMIT &&
 +                  (o->flags & TMP_MARK)) {
 +                      add_object_array(o, NULL, reachable);
 +                              o->flags &= ~TMP_MARK;
 +              }
 +      }
 +      close(cmd.out);
 +
 +      if (finish_command(&cmd))
 +              return -1;
 +
 +      return 0;
 +}
 +
 +static int has_unreachable(struct object_array *src)
 +{
 +      struct child_process cmd = CHILD_PROCESS_INIT;
 +      char buf[1];
 +      int i;
 +
 +      if (do_reachable_revlist(&cmd, src, NULL) < 0)
 +              return 1;
 +
        /*
         * The commits out of the rev-list are not ancestors of
         * our ref.
         */
 -      i = read_in_full(cmd.out, namebuf, 1);
 +      i = read_in_full(cmd.out, buf, 1);
        if (i)
                goto error;
        close(cmd.out);
 +      cmd.out = -1;
  
        /*
         * rev-list may have died by encountering a bad commit
                goto error;
  
        /* All the non-tip ones are ancestors of what we advertised */
 -      return;
 +      return 0;
 +
 +error:
 +      sigchain_pop(SIGPIPE);
 +      if (cmd.out >= 0)
 +              close(cmd.out);
 +      return 1;
 +}
 +
 +static void check_non_tip(void)
 +{
 +      int i;
 +
 +      /*
 +       * In the normal in-process case without
 +       * uploadpack.allowReachableSHA1InWant,
 +       * non-tip requests can never happen.
 +       */
 +      if (!stateless_rpc && !(allow_unadvertised_object_request & ALLOW_REACHABLE_SHA1))
 +              goto error;
 +      if (!has_unreachable(&want_obj))
 +              /* All the non-tip ones are ancestors of what we advertised */
 +              return;
  
  error:
        /* Pick one of them (we know there at least is one) */
        for (i = 0; i < want_obj.nr; i++) {
 -              o = want_obj.objects[i].item;
 +              struct object *o = want_obj.objects[i].item;
                if (!is_our_ref(o))
                        die("git upload-pack: not our ref %s",
                            oid_to_hex(&o->oid));
        }
  }
  
-                       packet_write(1, "shallow %s",
-                                    oid_to_hex(&object->oid));
 +static void send_shallow(struct commit_list *result)
 +{
 +      while (result) {
 +              struct object *object = &result->item->object;
 +              if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) {
-                       packet_write(1, "unshallow %s",
-                                    oid_to_hex(&object->oid));
++                      packet_write_fmt(1, "shallow %s",
++                                       oid_to_hex(&object->oid));
 +                      register_shallow(object->oid.hash);
 +                      shallow_nr++;
 +              }
 +              result = result->next;
 +      }
 +}
 +
 +static void send_unshallow(const struct object_array *shallows)
 +{
 +      int i;
 +
 +      for (i = 0; i < shallows->nr; i++) {
 +              struct object *object = shallows->objects[i].item;
 +              if (object->flags & NOT_SHALLOW) {
 +                      struct commit_list *parents;
++                      packet_write_fmt(1, "unshallow %s",
++                                       oid_to_hex(&object->oid));
 +                      object->flags &= ~CLIENT_SHALLOW;
 +                      /*
 +                       * We want to _register_ "object" as shallow, but we
 +                       * also need to traverse object's parents to deepen a
 +                       * shallow clone. Unregister it for now so we can
 +                       * parse and add the parents to the want list, then
 +                       * re-register it.
 +                       */
 +                      unregister_shallow(object->oid.hash);
 +                      object->parsed = 0;
 +                      parse_commit_or_die((struct commit *)object);
 +                      parents = ((struct commit *)object)->parents;
 +                      while (parents) {
 +                              add_object_array(&parents->item->object,
 +                                               NULL, &want_obj);
 +                              parents = parents->next;
 +                      }
 +                      add_object_array(object, NULL, &extra_edge_obj);
 +              }
 +              /* make sure commit traversal conforms to client */
 +              register_shallow(object->oid.hash);
 +      }
 +}
 +
 +static void deepen(int depth, int deepen_relative,
 +                 struct object_array *shallows)
 +{
 +      if (depth == INFINITE_DEPTH && !is_repository_shallow()) {
 +              int i;
 +
 +              for (i = 0; i < shallows->nr; i++) {
 +                      struct object *object = shallows->objects[i].item;
 +                      object->flags |= NOT_SHALLOW;
 +              }
 +      } else if (deepen_relative) {
 +              struct object_array reachable_shallows = OBJECT_ARRAY_INIT;
 +              struct commit_list *result;
 +
 +              get_reachable_list(shallows, &reachable_shallows);
 +              result = get_shallow_commits(&reachable_shallows,
 +                                           depth + 1,
 +                                           SHALLOW, NOT_SHALLOW);
 +              send_shallow(result);
 +              free_commit_list(result);
 +              object_array_clear(&reachable_shallows);
 +      } else {
 +              struct commit_list *result;
 +
 +              result = get_shallow_commits(&want_obj, depth,
 +                                           SHALLOW, NOT_SHALLOW);
 +              send_shallow(result);
 +              free_commit_list(result);
 +      }
 +
 +      send_unshallow(shallows);
 +      packet_flush(1);
 +}
 +
 +static void deepen_by_rev_list(int ac, const char **av,
 +                             struct object_array *shallows)
 +{
 +      struct commit_list *result;
 +
 +      result = get_shallow_commits_by_rev_list(ac, av, SHALLOW, NOT_SHALLOW);
 +      send_shallow(result);
 +      free_commit_list(result);
 +      send_unshallow(shallows);
 +      packet_flush(1);
 +}
 +
  static void receive_needs(void)
  {
        struct object_array shallows = OBJECT_ARRAY_INIT;
 +      struct string_list deepen_not = STRING_LIST_INIT_DUP;
        int depth = 0;
        int has_non_tip = 0;
 +      unsigned long deepen_since = 0;
 +      int deepen_rev_list = 0;
  
        shallow_nr = 0;
        for (;;) {
                const char *features;
                unsigned char sha1_buf[20];
                char *line = packet_read_line(0, NULL);
 +              const char *arg;
 +
                reset_timeout();
                if (!line)
                        break;
  
 -              if (starts_with(line, "shallow ")) {
 +              if (skip_prefix(line, "shallow ", &arg)) {
                        unsigned char sha1[20];
                        struct object *object;
 -                      if (get_sha1_hex(line + 8, sha1))
 +                      if (get_sha1_hex(arg, sha1))
                                die("invalid shallow line: %s", line);
                        object = parse_object(sha1);
                        if (!object)
                        }
                        continue;
                }
 -              if (starts_with(line, "deepen ")) {
 -                      char *end;
 -                      depth = strtol(line + 7, &end, 0);
 -                      if (end == line + 7 || depth <= 0)
 +              if (skip_prefix(line, "deepen ", &arg)) {
 +                      char *end = NULL;
 +                      depth = strtol(arg, &end, 0);
 +                      if (!end || *end || depth <= 0)
                                die("Invalid deepen: %s", line);
                        continue;
                }
 -              if (!starts_with(line, "want ") ||
 -                  get_sha1_hex(line+5, sha1_buf))
 +              if (skip_prefix(line, "deepen-since ", &arg)) {
 +                      char *end = NULL;
 +                      deepen_since = strtoul(arg, &end, 0);
 +                      if (!end || *end || !deepen_since ||
 +                          /* revisions.c's max_age -1 is special */
 +                          deepen_since == -1)
 +                              die("Invalid deepen-since: %s", line);
 +                      deepen_rev_list = 1;
 +                      continue;
 +              }
 +              if (skip_prefix(line, "deepen-not ", &arg)) {
 +                      char *ref = NULL;
 +                      unsigned char sha1[20];
 +                      if (expand_ref(arg, strlen(arg), sha1, &ref) != 1)
 +                              die("git upload-pack: ambiguous deepen-not: %s", line);
 +                      string_list_append(&deepen_not, ref);
 +                      free(ref);
 +                      deepen_rev_list = 1;
 +                      continue;
 +              }
 +              if (!skip_prefix(line, "want ", &arg) ||
 +                  get_sha1_hex(arg, sha1_buf))
                        die("git upload-pack: protocol error, "
                            "expected to get sha, not '%s'", line);
  
 -              features = line + 45;
 +              features = arg + 40;
  
 +              if (parse_feature_request(features, "deepen-relative"))
 +                      deepen_relative = 1;
                if (parse_feature_request(features, "multi_ack_detailed"))
                        multi_ack = 2;
                else if (parse_feature_request(features, "multi_ack"))
        if (!use_sideband && daemon_mode)
                no_progress = 1;
  
 -      if (depth == 0 && shallows.nr == 0)
 +      if (depth == 0 && !deepen_rev_list && shallows.nr == 0)
                return;
 -      if (depth > 0) {
 -              struct commit_list *result = NULL, *backup = NULL;
 +      if (depth > 0 && deepen_rev_list)
 +              die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together");
 +      if (depth > 0)
 +              deepen(depth, deepen_relative, &shallows);
 +      else if (deepen_rev_list) {
 +              struct argv_array av = ARGV_ARRAY_INIT;
                int i;
 -              if (depth == INFINITE_DEPTH && !is_repository_shallow())
 -                      for (i = 0; i < shallows.nr; i++) {
 -                              struct object *object = shallows.objects[i].item;
 -                              object->flags |= NOT_SHALLOW;
 -                      }
 -              else
 -                      backup = result =
 -                              get_shallow_commits(&want_obj, depth,
 -                                                  SHALLOW, NOT_SHALLOW);
 -              while (result) {
 -                      struct object *object = &result->item->object;
 -                      if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) {
 -                              packet_write_fmt(1, "shallow %s",
 -                                              oid_to_hex(&object->oid));
 -                              register_shallow(object->oid.hash);
 -                              shallow_nr++;
 +
 +              argv_array_push(&av, "rev-list");
 +              if (deepen_since)
 +                      argv_array_pushf(&av, "--max-age=%lu", deepen_since);
 +              if (deepen_not.nr) {
 +                      argv_array_push(&av, "--not");
 +                      for (i = 0; i < deepen_not.nr; i++) {
 +                              struct string_list_item *s = deepen_not.items + i;
 +                              argv_array_push(&av, s->string);
                        }
 -                      result = result->next;
 +                      argv_array_push(&av, "--not");
                }
 -              free_commit_list(backup);
 -              for (i = 0; i < shallows.nr; i++) {
 -                      struct object *object = shallows.objects[i].item;
 -                      if (object->flags & NOT_SHALLOW) {
 -                              struct commit_list *parents;
 -                              packet_write_fmt(1, "unshallow %s",
 -                                      oid_to_hex(&object->oid));
 -                              object->flags &= ~CLIENT_SHALLOW;
 -                              /* make sure the real parents are parsed */
 -                              unregister_shallow(object->oid.hash);
 -                              object->parsed = 0;
 -                              parse_commit_or_die((struct commit *)object);
 -                              parents = ((struct commit *)object)->parents;
 -                              while (parents) {
 -                                      add_object_array(&parents->item->object,
 -                                                      NULL, &want_obj);
 -                                      parents = parents->next;
 -                              }
 -                              add_object_array(object, NULL, &extra_edge_obj);
 -                      }
 -                      /* make sure commit traversal conforms to client */
 -                      register_shallow(object->oid.hash);
 +              for (i = 0; i < want_obj.nr; i++) {
 +                      struct object *o = want_obj.objects[i].item;
 +                      argv_array_push(&av, oid_to_hex(&o->oid));
                }
 -              packet_flush(1);
 -      } else
 +              deepen_by_rev_list(av.argc, av.argv, &shallows);
 +              argv_array_clear(&av);
 +      }
 +      else
                if (shallows.nr > 0) {
                        int i;
                        for (i = 0; i < shallows.nr; i++)
@@@ -920,8 -729,8 +920,8 @@@ static int send_ref(const char *refname
                    int flag, void *cb_data)
  {
        static const char *capabilities = "multi_ack thin-pack side-band"
 -              " side-band-64k ofs-delta shallow no-progress"
 -              " include-tag multi_ack_detailed";
 +              " side-band-64k ofs-delta shallow deepen-since deepen-not"
 +              " deepen-relative no-progress include-tag multi_ack_detailed";
        const char *refname_nons = strip_namespace(refname);
        struct object_id peeled;
  
                struct strbuf symref_info = STRBUF_INIT;
  
                format_symref_info(&symref_info, cb_data);
-               packet_write(1, "%s %s%c%s%s%s%s%s agent=%s\n",
+               packet_write_fmt(1, "%s %s%c%s%s%s%s%s agent=%s\n",
                             oid_to_hex(oid), refname_nons,
                             0, capabilities,
                             (allow_unadvertised_object_request & ALLOW_TIP_SHA1) ?
                             git_user_agent_sanitized());
                strbuf_release(&symref_info);
        } else {
-               packet_write(1, "%s %s\n", oid_to_hex(oid), refname_nons);
+               packet_write_fmt(1, "%s %s\n", oid_to_hex(oid), refname_nons);
        }
        capabilities = NULL;
        if (!peel_ref(refname, peeled.hash))
-               packet_write(1, "%s %s^{}\n", oid_to_hex(&peeled), refname_nons);
+               packet_write_fmt(1, "%s %s^{}\n", oid_to_hex(&peeled), refname_nons);
        return 0;
  }