From: Junio C Hamano <gitster@pobox.com>
Date: Tue, 24 Apr 2012 21:40:51 +0000 (-0700)
Subject: Merge branch 'it/fetch-pack-many-refs'
X-Git-Tag: v1.7.11-rc0~108
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/77cab8af4ad6c25995e40a5ababc7a17dc369fe0?hp=-c

Merge branch 'it/fetch-pack-many-refs'

When "git fetch" encounters repositories with too many references, the
command line of "fetch-pack" that is run by a helper e.g. remote-curl,
may fail to hold all of them. Now such an internal invocation can feed
the references through the standard input of "fetch-pack".

By Ivan Todoroski
* it/fetch-pack-many-refs:
remote-curl: main test case for the OS command line overflow
fetch-pack: test cases for the new --stdin option
remote-curl: send the refs to fetch-pack on stdin
fetch-pack: new --stdin option to read refs from stdin
---

77cab8af4ad6c25995e40a5ababc7a17dc369fe0
diff --combined builtin/fetch-pack.c
index 7124c4b49c,a9b6077af3..10db15b184
--- a/builtin/fetch-pack.c
+++ b/builtin/fetch-pack.c
@@@ -23,7 -23,9 +23,9 @@@ static struct fetch_pack_args args = 
  };
  
  static const char fetch_pack_usage[] =
- "git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
+ "git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
+ "[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
+ "[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
  
  #define COMPLETE	(1U << 0)
  #define COMMON		(1U << 1)
@@@ -58,9 -60,9 +60,9 @@@ static void rev_list_push(struct commi
  	}
  }
  
 -static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 +static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  {
 -	struct object *o = deref_tag(parse_object(sha1), path, 0);
 +	struct object *o = deref_tag(parse_object(sha1), refname, 0);
  
  	if (o && o->type == OBJ_COMMIT)
  		rev_list_push((struct commit *)o, SEEN);
@@@ -68,9 -70,9 +70,9 @@@
  	return 0;
  }
  
 -static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 +static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  {
 -	struct object *o = deref_tag(parse_object(sha1), path, 0);
 +	struct object *o = deref_tag(parse_object(sha1), refname, 0);
  
  	if (o && o->type == OBJ_COMMIT)
  		clear_commit_marks((struct commit *)o,
@@@ -256,6 -258,11 +258,6 @@@ static void insert_one_alternate_ref(co
  	rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
  }
  
 -static void insert_alternate_refs(void)
 -{
 -	for_each_alternate_ref(insert_one_alternate_ref, NULL);
 -}
 -
  #define INITIAL_FLUSH 16
  #define PIPESAFE_FLUSH 32
  #define LARGE_FLUSH 1024
@@@ -290,7 -297,7 +292,7 @@@ static int find_common(int fd[2], unsig
  	marked = 1;
  
  	for_each_ref(rev_list_insert_ref, NULL);
 -	insert_alternate_refs();
 +	for_each_alternate_ref(insert_one_alternate_ref, NULL);
  
  	fetching = 0;
  	for ( ; refs ; refs = refs->next) {
@@@ -488,7 -495,7 +490,7 @@@ done
  
  static struct commit_list *complete;
  
 -static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 +static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  {
  	struct object *o = parse_object(sha1);
  
@@@ -581,11 -588,6 +583,11 @@@ static void filter_refs(struct ref **re
  	*refs = newlist;
  }
  
 +static void mark_alternate_complete(const struct ref *ref, void *unused)
 +{
 +	mark_complete(NULL, ref->old_sha1, 0, NULL);
 +}
 +
  static int everything_local(struct ref **refs, int nr_match, char **match)
  {
  	struct ref *ref;
@@@ -614,7 -616,6 +616,7 @@@
  
  	if (!args.depth) {
  		for_each_ref(mark_complete, NULL);
 +		for_each_alternate_ref(mark_alternate_complete, NULL);
  		if (cutoff)
  			mark_recent_complete_commits(cutoff);
  	}
@@@ -737,7 -738,7 +739,7 @@@ static int get_pack(int xd[2], char **p
  	}
  	else {
  		*av++ = "unpack-objects";
 -		if (args.quiet)
 +		if (args.quiet || args.no_progress)
  			*av++ = "-q";
  	}
  	if (*hdr_arg)
@@@ -942,6 -943,10 +944,10 @@@ int cmd_fetch_pack(int argc, const cha
  				args.fetch_all = 1;
  				continue;
  			}
+ 			if (!strcmp("--stdin", arg)) {
+ 				args.stdin_refs = 1;
+ 				continue;
+ 			}
  			if (!strcmp("-v", arg)) {
  				args.verbose = 1;
  				continue;
@@@ -973,6 -978,40 +979,40 @@@
  	if (!dest)
  		usage(fetch_pack_usage);
  
+ 	if (args.stdin_refs) {
+ 		/*
+ 		 * Copy refs from cmdline to new growable list, then
+ 		 * append the refs from the standard input.
+ 		 */
+ 		int alloc_heads = nr_heads;
+ 		int size = nr_heads * sizeof(*heads);
+ 		heads = memcpy(xmalloc(size), heads, size);
+ 		if (args.stateless_rpc) {
+ 			/* in stateless RPC mode we use pkt-line to read
+ 			 * from stdin, until we get a flush packet
+ 			 */
+ 			static char line[1000];
+ 			for (;;) {
+ 				int n = packet_read_line(0, line, sizeof(line));
+ 				if (!n)
+ 					break;
+ 				if (line[n-1] == '\n')
+ 					n--;
+ 				ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
+ 				heads[nr_heads++] = xmemdupz(line, n);
+ 			}
+ 		}
+ 		else {
+ 			/* read from stdin one ref per line, until EOF */
+ 			struct strbuf line = STRBUF_INIT;
+ 			while (strbuf_getline(&line, stdin, '\n') != EOF) {
+ 				ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
+ 				heads[nr_heads++] = strbuf_detach(&line, NULL);
+ 			}
+ 			strbuf_release(&line);
+ 		}
+ 	}
+ 
  	if (args.stateless_rpc) {
  		conn = NULL;
  		fd[0] = 0;
diff --combined remote-curl.c
index d159fe7f34,543f2be8b0..08962214db
--- a/remote-curl.c
+++ b/remote-curl.c
@@@ -290,6 -290,7 +290,7 @@@ static void output_refs(struct ref *ref
  struct rpc_state {
  	const char *service_name;
  	const char **argv;
+ 	struct strbuf *stdin_preamble;
  	char *service_url;
  	char *hdr_content_type;
  	char *hdr_accept;
@@@ -535,6 -536,7 +536,7 @@@ static int rpc_service(struct rpc_stat
  {
  	const char *svc = rpc->service_name;
  	struct strbuf buf = STRBUF_INIT;
+ 	struct strbuf *preamble = rpc->stdin_preamble;
  	struct child_process client;
  	int err = 0;
  
@@@ -545,6 -547,8 +547,8 @@@
  	client.argv = rpc->argv;
  	if (start_command(&client))
  		exit(1);
+ 	if (preamble)
+ 		write_or_die(client.in, preamble->buf, preamble->len);
  	if (heads)
  		write_or_die(client.in, heads->buf, heads->len);
  
@@@ -626,13 -630,14 +630,14 @@@ static int fetch_git(struct discovery *
  	int nr_heads, struct ref **to_fetch)
  {
  	struct rpc_state rpc;
+ 	struct strbuf preamble = STRBUF_INIT;
  	char *depth_arg = NULL;
- 	const char **argv;
  	int argc = 0, i, err;
+ 	const char *argv[15];
  
- 	argv = xmalloc((15 + nr_heads) * sizeof(char*));
  	argv[argc++] = "fetch-pack";
  	argv[argc++] = "--stateless-rpc";
+ 	argv[argc++] = "--stdin";
  	argv[argc++] = "--lock-pack";
  	if (options.followtags)
  		argv[argc++] = "--include-tag";
@@@ -651,24 -656,27 +656,27 @@@
  		argv[argc++] = depth_arg;
  	}
  	argv[argc++] = url;
+ 	argv[argc++] = NULL;
+ 
  	for (i = 0; i < nr_heads; i++) {
  		struct ref *ref = to_fetch[i];
  		if (!ref->name || !*ref->name)
  			die("cannot fetch by sha1 over smart http");
- 		argv[argc++] = ref->name;
+ 		packet_buf_write(&preamble, "%s\n", ref->name);
  	}
- 	argv[argc++] = NULL;
+ 	packet_buf_flush(&preamble);
  
  	memset(&rpc, 0, sizeof(rpc));
  	rpc.service_name = "git-upload-pack",
  	rpc.argv = argv;
+ 	rpc.stdin_preamble = &preamble;
  	rpc.gzip_request = 1;
  
  	err = rpc_service(&rpc, heads);
  	if (rpc.result.len)
  		safe_write(1, rpc.result.buf, rpc.result.len);
  	strbuf_release(&rpc.result);
- 	free(argv);
+ 	strbuf_release(&preamble);
  	free(depth_arg);
  	return err;
  }
@@@ -770,9 -778,7 +778,9 @@@ static int push_git(struct discovery *h
  		argv[argc++] = "--thin";
  	if (options.dry_run)
  		argv[argc++] = "--dry-run";
 -	if (options.verbosity > 1)
 +	if (options.verbosity == 0)
 +		argv[argc++] = "--quiet";
 +	else if (options.verbosity > 1)
  		argv[argc++] = "--verbose";
  	argv[argc++] = url;
  	for (i = 0; i < nr_spec; i++)
@@@ -807,7 -813,7 +815,7 @@@ static int push(int nr_spec, char **spe
  static void parse_push(struct strbuf *buf)
  {
  	char **specs = NULL;
 -	int alloc_spec = 0, nr_spec = 0, i;
 +	int alloc_spec = 0, nr_spec = 0, i, ret;
  
  	do {
  		if (!prefixcmp(buf->buf, "push ")) {
@@@ -824,13 -830,12 +832,13 @@@
  			break;
  	} while (1);
  
 -	if (push(nr_spec, specs))
 -		exit(128); /* error already reported */
 -
 +	ret = push(nr_spec, specs);
  	printf("\n");
  	fflush(stdout);
  
 +	if (ret)
 +		exit(128); /* error already reported */
 +
   free_specs:
  	for (i = 0; i < nr_spec; i++)
  		free(specs[i]);
diff --combined t/t5500-fetch-pack.sh
index ce51692bb2,1c56d9b883..1d1ca98588
--- a/t/t5500-fetch-pack.sh
+++ b/t/t5500-fetch-pack.sh
@@@ -114,19 -114,8 +114,19 @@@ pull_to_client 2nd "refs/heads/B" $((64
  
  pull_to_client 3rd "refs/heads/A" $((1*3))
  
 +test_expect_success 'single branch clone' '
 +	git clone --single-branch "file://$(pwd)/." singlebranch
 +'
 +
 +test_expect_success 'single branch object count' '
 +	GIT_DIR=singlebranch/.git git count-objects -v |
 +		grep "^in-pack:" > count.singlebranch &&
 +	echo "in-pack: 198" >expected &&
 +	test_cmp expected count.singlebranch
 +'
 +
  test_expect_success 'clone shallow' '
 -	git clone --depth 2 "file://$(pwd)/." shallow
 +	git clone --no-single-branch --depth 2 "file://$(pwd)/." shallow
  '
  
  test_expect_success 'clone shallow object count' '
@@@ -259,71 -248,70 +259,137 @@@ test_expect_success 'clone shallow obje
  	grep "^count: 52" count.shallow
  '
  
 +test_expect_success 'clone shallow without --no-single-branch' '
 +	git clone --depth 1 "file://$(pwd)/." shallow2
 +'
 +
 +test_expect_success 'clone shallow object count' '
 +	(
 +		cd shallow2 &&
 +		git count-objects -v
 +	) > count.shallow2 &&
 +	grep "^in-pack: 6" count.shallow2
 +'
 +
 +test_expect_success 'clone shallow with --branch' '
 +	git clone --depth 1 --branch A "file://$(pwd)/." shallow3
 +'
 +
 +test_expect_success 'clone shallow object count' '
 +	echo "in-pack: 12" > count3.expected &&
 +	GIT_DIR=shallow3/.git git count-objects -v |
 +		grep "^in-pack" > count3.actual &&
 +	test_cmp count3.expected count3.actual
 +'
 +
 +test_expect_success 'clone shallow with detached HEAD' '
 +	git checkout HEAD^ &&
 +	git clone --depth 1 "file://$(pwd)/." shallow5 &&
 +	git checkout - &&
 +	GIT_DIR=shallow5/.git git rev-parse HEAD >actual &&
 +	git rev-parse HEAD^ >expected &&
 +	test_cmp expected actual
 +'
 +
 +test_expect_success 'shallow clone pulling tags' '
 +	git tag -a -m A TAGA1 A &&
 +	git tag -a -m B TAGB1 B &&
 +	git tag TAGA2 A &&
 +	git tag TAGB2 B &&
 +	git clone --depth 1 "file://$(pwd)/." shallow6 &&
 +
 +	cat >taglist.expected <<\EOF &&
 +TAGB1
 +TAGB2
 +EOF
 +	GIT_DIR=shallow6/.git git tag -l >taglist.actual &&
 +	test_cmp taglist.expected taglist.actual &&
 +
 +	echo "in-pack: 7" > count6.expected &&
 +	GIT_DIR=shallow6/.git git count-objects -v |
 +		grep "^in-pack" > count6.actual &&
 +	test_cmp count6.expected count6.actual
 +'
 +
 +test_expect_success 'shallow cloning single tag' '
 +	git clone --depth 1 --branch=TAGB1 "file://$(pwd)/." shallow7 &&
 +	cat >taglist.expected <<\EOF &&
 +TAGB1
 +TAGB2
 +EOF
 +	GIT_DIR=shallow7/.git git tag -l >taglist.actual &&
 +	test_cmp taglist.expected taglist.actual &&
 +
 +	echo "in-pack: 7" > count7.expected &&
 +	GIT_DIR=shallow7/.git git count-objects -v |
 +		grep "^in-pack" > count7.actual &&
 +	test_cmp count7.expected count7.actual
 +'
 +
+ test_expect_success 'setup tests for the --stdin parameter' '
+ 	for head in C D E F
+ 	do
+ 		add $head
+ 	done &&
+ 	for head in A B C D E F
+ 	do
+ 		git tag $head $head
+ 	done &&
+ 	cat >input <<-\EOF
+ 	refs/heads/C
+ 	refs/heads/A
+ 	refs/heads/D
+ 	refs/tags/C
+ 	refs/heads/B
+ 	refs/tags/A
+ 	refs/heads/E
+ 	refs/tags/B
+ 	refs/tags/E
+ 	refs/tags/D
+ 	EOF
+ 	sort <input >expect &&
+ 	(
+ 		echo refs/heads/E &&
+ 		echo refs/tags/E &&
+ 		cat input
+ 	) >input.dup
+ '
+ 
+ test_expect_success 'fetch refs from cmdline' '
+ 	(
+ 		cd client &&
+ 		git fetch-pack --no-progress .. $(cat ../input)
+ 	) >output &&
+ 	cut -d " " -f 2 <output | sort >actual &&
+ 	test_cmp expect actual
+ '
+ 
+ test_expect_success 'fetch refs from stdin' '
+ 	(
+ 		cd client &&
+ 		git fetch-pack --stdin --no-progress .. <../input
+ 	) >output &&
+ 	cut -d " " -f 2 <output | sort >actual &&
+ 	test_cmp expect actual
+ '
+ 
+ test_expect_success 'fetch mixed refs from cmdline and stdin' '
+ 	(
+ 		cd client &&
+ 		tail -n +5 ../input |
+ 		git fetch-pack --stdin --no-progress .. $(head -n 4 ../input)
+ 	) >output &&
+ 	cut -d " " -f 2 <output | sort >actual &&
+ 	test_cmp expect actual
+ '
+ 
+ test_expect_success 'test duplicate refs from stdin' '
+ 	(
+ 	cd client &&
+ 	test_must_fail git fetch-pack --stdin --no-progress .. <../input.dup
+ 	) >output &&
+ 	cut -d " " -f 2 <output | sort >actual &&
+ 	test_cmp expect actual
+ '
+ 
  test_done