Merge branch 'it/fetch-pack-many-refs' into maint
authorJunio C Hamano <gitster@pobox.com>
Wed, 2 May 2012 04:12:36 +0000 (21:12 -0700)
committerJunio C Hamano <gitster@pobox.com>
Wed, 2 May 2012 04:12:36 +0000 (21:12 -0700)
When "git fetch" encounters repositories with too many references, the
command line of "fetch-pack" that is run by a helper e.g. remote-curl, may
fail to hold all of them. Now such an internal invocation can feed the
references through the standard input of "fetch-pack".

By Ivan Todoroski
* it/fetch-pack-many-refs:
remote-curl: main test case for the OS command line overflow
fetch-pack: test cases for the new --stdin option
remote-curl: send the refs to fetch-pack on stdin
fetch-pack: new --stdin option to read refs from stdin

Conflicts:
t/t5500-fetch-pack.sh

1  2 
builtin/fetch-pack.c
remote-curl.c
t/t5500-fetch-pack.sh
diff --combined builtin/fetch-pack.c
index 7124c4b49cfba7985c5ba2046296f006de04e3bb,a9b6077af3ccfe8a66d3226af994ed77289bd544..10db15b18403f01b9ee5db27b37d1e0cdbb2abb1
@@@ -23,7 -23,9 +23,9 @@@ static struct fetch_pack_args args = 
  };
  
  static const char fetch_pack_usage[] =
- "git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
+ "git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
+ "[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
+ "[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
  
  #define COMPLETE      (1U << 0)
  #define COMMON                (1U << 1)
@@@ -58,9 -60,9 +60,9 @@@ static void rev_list_push(struct commi
        }
  }
  
 -static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 +static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  {
 -      struct object *o = deref_tag(parse_object(sha1), path, 0);
 +      struct object *o = deref_tag(parse_object(sha1), refname, 0);
  
        if (o && o->type == OBJ_COMMIT)
                rev_list_push((struct commit *)o, SEEN);
@@@ -68,9 -70,9 +70,9 @@@
        return 0;
  }
  
 -static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 +static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  {
 -      struct object *o = deref_tag(parse_object(sha1), path, 0);
 +      struct object *o = deref_tag(parse_object(sha1), refname, 0);
  
        if (o && o->type == OBJ_COMMIT)
                clear_commit_marks((struct commit *)o,
@@@ -256,6 -258,11 +258,6 @@@ static void insert_one_alternate_ref(co
        rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
  }
  
 -static void insert_alternate_refs(void)
 -{
 -      for_each_alternate_ref(insert_one_alternate_ref, NULL);
 -}
 -
  #define INITIAL_FLUSH 16
  #define PIPESAFE_FLUSH 32
  #define LARGE_FLUSH 1024
@@@ -290,7 -297,7 +292,7 @@@ static int find_common(int fd[2], unsig
        marked = 1;
  
        for_each_ref(rev_list_insert_ref, NULL);
 -      insert_alternate_refs();
 +      for_each_alternate_ref(insert_one_alternate_ref, NULL);
  
        fetching = 0;
        for ( ; refs ; refs = refs->next) {
@@@ -488,7 -495,7 +490,7 @@@ done
  
  static struct commit_list *complete;
  
 -static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 +static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  {
        struct object *o = parse_object(sha1);
  
@@@ -581,11 -588,6 +583,11 @@@ static void filter_refs(struct ref **re
        *refs = newlist;
  }
  
 +static void mark_alternate_complete(const struct ref *ref, void *unused)
 +{
 +      mark_complete(NULL, ref->old_sha1, 0, NULL);
 +}
 +
  static int everything_local(struct ref **refs, int nr_match, char **match)
  {
        struct ref *ref;
  
        if (!args.depth) {
                for_each_ref(mark_complete, NULL);
 +              for_each_alternate_ref(mark_alternate_complete, NULL);
                if (cutoff)
                        mark_recent_complete_commits(cutoff);
        }
@@@ -737,7 -738,7 +739,7 @@@ static int get_pack(int xd[2], char **p
        }
        else {
                *av++ = "unpack-objects";
 -              if (args.quiet)
 +              if (args.quiet || args.no_progress)
                        *av++ = "-q";
        }
        if (*hdr_arg)
@@@ -942,6 -943,10 +944,10 @@@ int cmd_fetch_pack(int argc, const cha
                                args.fetch_all = 1;
                                continue;
                        }
+                       if (!strcmp("--stdin", arg)) {
+                               args.stdin_refs = 1;
+                               continue;
+                       }
                        if (!strcmp("-v", arg)) {
                                args.verbose = 1;
                                continue;
        if (!dest)
                usage(fetch_pack_usage);
  
+       if (args.stdin_refs) {
+               /*
+                * Copy refs from cmdline to new growable list, then
+                * append the refs from the standard input.
+                */
+               int alloc_heads = nr_heads;
+               int size = nr_heads * sizeof(*heads);
+               heads = memcpy(xmalloc(size), heads, size);
+               if (args.stateless_rpc) {
+                       /* in stateless RPC mode we use pkt-line to read
+                        * from stdin, until we get a flush packet
+                        */
+                       static char line[1000];
+                       for (;;) {
+                               int n = packet_read_line(0, line, sizeof(line));
+                               if (!n)
+                                       break;
+                               if (line[n-1] == '\n')
+                                       n--;
+                               ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
+                               heads[nr_heads++] = xmemdupz(line, n);
+                       }
+               }
+               else {
+                       /* read from stdin one ref per line, until EOF */
+                       struct strbuf line = STRBUF_INIT;
+                       while (strbuf_getline(&line, stdin, '\n') != EOF) {
+                               ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
+                               heads[nr_heads++] = strbuf_detach(&line, NULL);
+                       }
+                       strbuf_release(&line);
+               }
+       }
        if (args.stateless_rpc) {
                conn = NULL;
                fd[0] = 0;
diff --combined remote-curl.c
index d159fe7f3433ccf6e8c8908961736951e42b9c35,543f2be8b00159e6715f288c624307e5927957fb..08962214db6f715c0416c61ce0ab70b50b7888fc
@@@ -290,6 -290,7 +290,7 @@@ static void output_refs(struct ref *ref
  struct rpc_state {
        const char *service_name;
        const char **argv;
+       struct strbuf *stdin_preamble;
        char *service_url;
        char *hdr_content_type;
        char *hdr_accept;
@@@ -535,6 -536,7 +536,7 @@@ static int rpc_service(struct rpc_stat
  {
        const char *svc = rpc->service_name;
        struct strbuf buf = STRBUF_INIT;
+       struct strbuf *preamble = rpc->stdin_preamble;
        struct child_process client;
        int err = 0;
  
        client.argv = rpc->argv;
        if (start_command(&client))
                exit(1);
+       if (preamble)
+               write_or_die(client.in, preamble->buf, preamble->len);
        if (heads)
                write_or_die(client.in, heads->buf, heads->len);
  
@@@ -626,13 -630,14 +630,14 @@@ static int fetch_git(struct discovery *
        int nr_heads, struct ref **to_fetch)
  {
        struct rpc_state rpc;
+       struct strbuf preamble = STRBUF_INIT;
        char *depth_arg = NULL;
-       const char **argv;
        int argc = 0, i, err;
+       const char *argv[15];
  
-       argv = xmalloc((15 + nr_heads) * sizeof(char*));
        argv[argc++] = "fetch-pack";
        argv[argc++] = "--stateless-rpc";
+       argv[argc++] = "--stdin";
        argv[argc++] = "--lock-pack";
        if (options.followtags)
                argv[argc++] = "--include-tag";
                argv[argc++] = depth_arg;
        }
        argv[argc++] = url;
+       argv[argc++] = NULL;
        for (i = 0; i < nr_heads; i++) {
                struct ref *ref = to_fetch[i];
                if (!ref->name || !*ref->name)
                        die("cannot fetch by sha1 over smart http");
-               argv[argc++] = ref->name;
+               packet_buf_write(&preamble, "%s\n", ref->name);
        }
-       argv[argc++] = NULL;
+       packet_buf_flush(&preamble);
  
        memset(&rpc, 0, sizeof(rpc));
        rpc.service_name = "git-upload-pack",
        rpc.argv = argv;
+       rpc.stdin_preamble = &preamble;
        rpc.gzip_request = 1;
  
        err = rpc_service(&rpc, heads);
        if (rpc.result.len)
                safe_write(1, rpc.result.buf, rpc.result.len);
        strbuf_release(&rpc.result);
-       free(argv);
+       strbuf_release(&preamble);
        free(depth_arg);
        return err;
  }
@@@ -770,9 -778,7 +778,9 @@@ static int push_git(struct discovery *h
                argv[argc++] = "--thin";
        if (options.dry_run)
                argv[argc++] = "--dry-run";
 -      if (options.verbosity > 1)
 +      if (options.verbosity == 0)
 +              argv[argc++] = "--quiet";
 +      else if (options.verbosity > 1)
                argv[argc++] = "--verbose";
        argv[argc++] = url;
        for (i = 0; i < nr_spec; i++)
@@@ -807,7 -813,7 +815,7 @@@ static int push(int nr_spec, char **spe
  static void parse_push(struct strbuf *buf)
  {
        char **specs = NULL;
 -      int alloc_spec = 0, nr_spec = 0, i;
 +      int alloc_spec = 0, nr_spec = 0, i, ret;
  
        do {
                if (!prefixcmp(buf->buf, "push ")) {
                        break;
        } while (1);
  
 -      if (push(nr_spec, specs))
 -              exit(128); /* error already reported */
 -
 +      ret = push(nr_spec, specs);
        printf("\n");
        fflush(stdout);
  
 +      if (ret)
 +              exit(128); /* error already reported */
 +
   free_specs:
        for (i = 0; i < nr_spec; i++)
                free(specs[i]);
diff --combined t/t5500-fetch-pack.sh
index ce51692bb2b9ae221d11458a01ab8ef669f24659,1c56d9b883bdb2c55da1558071835ff80f00aac9..1d1ca98588bd7e8ae264179b5a4a93371c567346
@@@ -114,19 -114,8 +114,19 @@@ pull_to_client 2nd "refs/heads/B" $((64
  
  pull_to_client 3rd "refs/heads/A" $((1*3))
  
 +test_expect_success 'single branch clone' '
 +      git clone --single-branch "file://$(pwd)/." singlebranch
 +'
 +
 +test_expect_success 'single branch object count' '
 +      GIT_DIR=singlebranch/.git git count-objects -v |
 +              grep "^in-pack:" > count.singlebranch &&
 +      echo "in-pack: 198" >expected &&
 +      test_cmp expected count.singlebranch
 +'
 +
  test_expect_success 'clone shallow' '
 -      git clone --depth 2 "file://$(pwd)/." shallow
 +      git clone --no-single-branch --depth 2 "file://$(pwd)/." shallow
  '
  
  test_expect_success 'clone shallow object count' '
@@@ -259,71 -248,70 +259,137 @@@ test_expect_success 'clone shallow obje
        grep "^count: 52" count.shallow
  '
  
 +test_expect_success 'clone shallow without --no-single-branch' '
 +      git clone --depth 1 "file://$(pwd)/." shallow2
 +'
 +
 +test_expect_success 'clone shallow object count' '
 +      (
 +              cd shallow2 &&
 +              git count-objects -v
 +      ) > count.shallow2 &&
 +      grep "^in-pack: 6" count.shallow2
 +'
 +
 +test_expect_success 'clone shallow with --branch' '
 +      git clone --depth 1 --branch A "file://$(pwd)/." shallow3
 +'
 +
 +test_expect_success 'clone shallow object count' '
 +      echo "in-pack: 12" > count3.expected &&
 +      GIT_DIR=shallow3/.git git count-objects -v |
 +              grep "^in-pack" > count3.actual &&
 +      test_cmp count3.expected count3.actual
 +'
 +
 +test_expect_success 'clone shallow with detached HEAD' '
 +      git checkout HEAD^ &&
 +      git clone --depth 1 "file://$(pwd)/." shallow5 &&
 +      git checkout - &&
 +      GIT_DIR=shallow5/.git git rev-parse HEAD >actual &&
 +      git rev-parse HEAD^ >expected &&
 +      test_cmp expected actual
 +'
 +
 +test_expect_success 'shallow clone pulling tags' '
 +      git tag -a -m A TAGA1 A &&
 +      git tag -a -m B TAGB1 B &&
 +      git tag TAGA2 A &&
 +      git tag TAGB2 B &&
 +      git clone --depth 1 "file://$(pwd)/." shallow6 &&
 +
 +      cat >taglist.expected <<\EOF &&
 +TAGB1
 +TAGB2
 +EOF
 +      GIT_DIR=shallow6/.git git tag -l >taglist.actual &&
 +      test_cmp taglist.expected taglist.actual &&
 +
 +      echo "in-pack: 7" > count6.expected &&
 +      GIT_DIR=shallow6/.git git count-objects -v |
 +              grep "^in-pack" > count6.actual &&
 +      test_cmp count6.expected count6.actual
 +'
 +
 +test_expect_success 'shallow cloning single tag' '
 +      git clone --depth 1 --branch=TAGB1 "file://$(pwd)/." shallow7 &&
 +      cat >taglist.expected <<\EOF &&
 +TAGB1
 +TAGB2
 +EOF
 +      GIT_DIR=shallow7/.git git tag -l >taglist.actual &&
 +      test_cmp taglist.expected taglist.actual &&
 +
 +      echo "in-pack: 7" > count7.expected &&
 +      GIT_DIR=shallow7/.git git count-objects -v |
 +              grep "^in-pack" > count7.actual &&
 +      test_cmp count7.expected count7.actual
 +'
 +
+ test_expect_success 'setup tests for the --stdin parameter' '
+       for head in C D E F
+       do
+               add $head
+       done &&
+       for head in A B C D E F
+       do
+               git tag $head $head
+       done &&
+       cat >input <<-\EOF
+       refs/heads/C
+       refs/heads/A
+       refs/heads/D
+       refs/tags/C
+       refs/heads/B
+       refs/tags/A
+       refs/heads/E
+       refs/tags/B
+       refs/tags/E
+       refs/tags/D
+       EOF
+       sort <input >expect &&
+       (
+               echo refs/heads/E &&
+               echo refs/tags/E &&
+               cat input
+       ) >input.dup
+ '
+ test_expect_success 'fetch refs from cmdline' '
+       (
+               cd client &&
+               git fetch-pack --no-progress .. $(cat ../input)
+       ) >output &&
+       cut -d " " -f 2 <output | sort >actual &&
+       test_cmp expect actual
+ '
+ test_expect_success 'fetch refs from stdin' '
+       (
+               cd client &&
+               git fetch-pack --stdin --no-progress .. <../input
+       ) >output &&
+       cut -d " " -f 2 <output | sort >actual &&
+       test_cmp expect actual
+ '
+ test_expect_success 'fetch mixed refs from cmdline and stdin' '
+       (
+               cd client &&
+               tail -n +5 ../input |
+               git fetch-pack --stdin --no-progress .. $(head -n 4 ../input)
+       ) >output &&
+       cut -d " " -f 2 <output | sort >actual &&
+       test_cmp expect actual
+ '
+ test_expect_success 'test duplicate refs from stdin' '
+       (
+       cd client &&
+       test_must_fail git fetch-pack --stdin --no-progress .. <../input.dup
+       ) >output &&
+       cut -d " " -f 2 <output | sort >actual &&
+       test_cmp expect actual
+ '
  test_done