From: Junio C Hamano <gitster@pobox.com>
Date: Wed, 30 May 2018 05:04:10 +0000 (+0900)
Subject: Merge branch 'jt/partial-clone-proto-v2'
X-Git-Tag: v2.18.0-rc0~16
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/54db5c0e1edf0d92bd474f515fab349b98c18a24?hp=-c

Merge branch 'jt/partial-clone-proto-v2'

Transfer protocol v2 learned to support the partial clone.

* jt/partial-clone-proto-v2:
{fetch,upload}-pack: support filter in protocol v2
upload-pack: read config when serving protocol v2
upload-pack: fix error message typo
---

54db5c0e1edf0d92bd474f515fab349b98c18a24
diff --combined Documentation/technical/protocol-v2.txt
index d7b6f38e0a,38d24fd2bc..49bda76d23
--- a/Documentation/technical/protocol-v2.txt
+++ b/Documentation/technical/protocol-v2.txt
@@@ -290,6 -290,15 +290,15 @@@ included in the clients request as wel
  	Cannot be used with "deepen", but can be used with
  	"deepen-since".
  
+ If the 'filter' feature is advertised, the following argument can be
+ included in the client's request:
+ 
+     filter <filter-spec>
+ 	Request that various objects from the packfile be omitted
+ 	using one of several filtering techniques. These are intended
+ 	for use with partial clone and partial fetch operations. See
+ 	`rev-list` for possible "filter-spec" values.
+ 
  The response of `fetch` is broken into a number of sections separated by
  delimiter packets (0001), with each section beginning with its section
  header.
@@@ -393,13 -402,3 +402,13 @@@
  		1 - pack data
  		2 - progress messages
  		3 - fatal error message just before stream aborts
 +
 + server-option
 +~~~~~~~~~~~~~~~
 +
 +If advertised, indicates that any number of server specific options can be
 +included in a request.  This is done by sending each option as a
 +"server-option=<option>" capability line in the capability-list section of
 +a request.
 +
 +The provided options must not contain a NUL or LF character.
diff --combined fetch-pack.c
index 490c38f833,3ed40aa464..a320ce9872
--- a/fetch-pack.c
+++ b/fetch-pack.c
@@@ -1174,13 -1174,6 +1174,13 @@@ static int send_fetch_request(int fd_ou
  		packet_buf_write(&req_buf, "command=fetch");
  	if (server_supports_v2("agent", 0))
  		packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
 +	if (args->server_options && args->server_options->nr &&
 +	    server_supports_v2("server-option", 1)) {
 +		int i;
 +		for (i = 0; i < args->server_options->nr; i++)
 +			packet_write_fmt(fd_out, "server-option=%s",
 +					 args->server_options->items[i].string);
 +	}
  
  	packet_buf_delim(&req_buf);
  	if (args->use_thin_pack)
@@@ -1198,14 -1191,29 +1198,29 @@@
  	else if (is_repository_shallow() || args->deepen)
  		die(_("Server does not support shallow requests"));
  
+ 	/* Add filter */
+ 	if (server_supports_feature("fetch", "filter", 0) &&
+ 	    args->filter_options.choice) {
+ 		print_verbose(args, _("Server supports filter"));
+ 		packet_buf_write(&req_buf, "filter %s",
+ 				 args->filter_options.filter_spec);
+ 	} else if (args->filter_options.choice) {
+ 		warning("filtering not recognized by server, ignoring");
+ 	}
+ 
  	/* add wants */
  	add_wants(wants, &req_buf);
  
- 	/* Add all of the common commits we've found in previous rounds */
- 	add_common(&req_buf, common);
+ 	if (args->no_dependents) {
+ 		packet_buf_write(&req_buf, "done");
+ 		ret = 1;
+ 	} else {
+ 		/* Add all of the common commits we've found in previous rounds */
+ 		add_common(&req_buf, common);
  
- 	/* Add initial haves */
- 	ret = add_haves(&req_buf, haves_to_send, in_vain);
+ 		/* Add initial haves */
+ 		ret = add_haves(&req_buf, haves_to_send, in_vain);
+ 	}
  
  	/* Send request */
  	packet_buf_flush(&req_buf);
diff --combined t/t5701-git-serve.sh
index 011a5796db,1b4b13cc2b..75ec79e6cb
--- a/t/t5701-git-serve.sh
+++ b/t/t5701-git-serve.sh
@@@ -10,7 -10,6 +10,7 @@@ test_expect_success 'test capability ad
  	agent=git/$(git version | cut -d" " -f3)
  	ls-refs
  	fetch=shallow
 +	server-option
  	0000
  	EOF
  
@@@ -174,24 -173,18 +174,38 @@@ test_expect_success 'symrefs parameter
  	test_cmp actual expect
  '
  
 +test_expect_success 'sending server-options' '
 +	test-pkt-line pack >in <<-EOF &&
 +	command=ls-refs
 +	server-option=hello
 +	server-option=world
 +	0001
 +	ref-prefix HEAD
 +	0000
 +	EOF
 +
 +	cat >expect <<-EOF &&
 +	$(git rev-parse HEAD) HEAD
 +	0000
 +	EOF
 +
 +	git serve --stateless-rpc <in >out &&
 +	test-pkt-line unpack <out >actual &&
 +	test_cmp actual expect
 +'
 +
+ test_expect_success 'unexpected lines are not allowed in fetch request' '
+ 	git init server &&
+ 
+ 	test-pkt-line pack >in <<-EOF &&
+ 	command=fetch
+ 	0001
+ 	this-is-not-a-command
+ 	0000
+ 	EOF
+ 
+ 	test_must_fail git -C server serve --stateless-rpc <in >/dev/null 2>err &&
+ 	grep "unexpected line: .this-is-not-a-command." err
+ '
+ 
  test_done
diff --combined t/t5702-protocol-v2.sh
index dbfd0691c0,25bf046b36..4ad5c5769b
--- a/t/t5702-protocol-v2.sh
+++ b/t/t5702-protocol-v2.sh
@@@ -154,22 -154,6 +154,22 @@@ test_expect_success 'ref advertisment i
  	test_cmp actual expect
  '
  
 +test_expect_success 'server-options are sent when using ls-remote' '
 +	test_when_finished "rm -f log" &&
 +
 +	GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \
 +		ls-remote -o hello -o world "file://$(pwd)/file_parent" master >actual &&
 +
 +	cat >expect <<-EOF &&
 +	$(git -C file_parent rev-parse refs/heads/master)$(printf "\t")refs/heads/master
 +	EOF
 +
 +	test_cmp actual expect &&
 +	grep "server-option=hello" log &&
 +	grep "server-option=world" log
 +'
 +
 +
  test_expect_success 'clone with file:// using protocol v2' '
  	test_when_finished "rm -f log" &&
  
@@@ -217,22 -201,118 +217,134 @@@ test_expect_success 'ref advertisment i
  	! grep "refs/tags/three" log
  '
  
 +test_expect_success 'server-options are sent when fetching' '
 +	test_when_finished "rm -f log" &&
 +
 +	test_commit -C file_parent four &&
 +
 +	GIT_TRACE_PACKET="$(pwd)/log" git -C file_child -c protocol.version=2 \
 +		fetch -o hello -o world origin master &&
 +
 +	git -C file_child log -1 --format=%s origin/master >actual &&
 +	git -C file_parent log -1 --format=%s >expect &&
 +	test_cmp expect actual &&
 +
 +	grep "server-option=hello" log &&
 +	grep "server-option=world" log
 +'
 +
+ test_expect_success 'upload-pack respects config using protocol v2' '
+ 	git init server &&
+ 	write_script server/.git/hook <<-\EOF &&
+ 		touch hookout
+ 		"$@"
+ 	EOF
+ 	test_commit -C server one &&
+ 
+ 	test_config_global uploadpack.packobjectshook ./hook &&
+ 	test_path_is_missing server/.git/hookout &&
+ 	git -c protocol.version=2 clone "file://$(pwd)/server" client &&
+ 	test_path_is_file server/.git/hookout
+ '
+ 
+ test_expect_success 'setup filter tests' '
+ 	rm -rf server client &&
+ 	git init server &&
+ 
+ 	# 1 commit to create a file, and 1 commit to modify it
+ 	test_commit -C server message1 a.txt &&
+ 	test_commit -C server message2 a.txt &&
+ 	git -C server config protocol.version 2 &&
+ 	git -C server config uploadpack.allowfilter 1 &&
+ 	git -C server config uploadpack.allowanysha1inwant 1 &&
+ 	git -C server config protocol.version 2
+ '
+ 
+ test_expect_success 'partial clone' '
+ 	GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 \
+ 		clone --filter=blob:none "file://$(pwd)/server" client &&
+ 	grep "version 2" trace &&
+ 
+ 	# Ensure that the old version of the file is missing
+ 	git -C client rev-list master --quiet --objects --missing=print \
+ 		>observed.oids &&
+ 	grep "$(git -C server rev-parse message1:a.txt)" observed.oids &&
+ 
+ 	# Ensure that client passes fsck
+ 	git -C client fsck
+ '
+ 
+ test_expect_success 'dynamically fetch missing object' '
+ 	rm "$(pwd)/trace" &&
+ 	GIT_TRACE_PACKET="$(pwd)/trace" git -C client -c protocol.version=2 \
+ 		cat-file -p $(git -C server rev-parse message1:a.txt) &&
+ 	grep "version 2" trace
+ '
+ 
+ test_expect_success 'partial fetch' '
+ 	rm -rf client "$(pwd)/trace" &&
+ 	git init client &&
+ 	SERVER="file://$(pwd)/server" &&
+ 	test_config -C client extensions.partialClone "$SERVER" &&
+ 
+ 	GIT_TRACE_PACKET="$(pwd)/trace" git -C client -c protocol.version=2 \
+ 		fetch --filter=blob:none "$SERVER" master:refs/heads/other &&
+ 	grep "version 2" trace &&
+ 
+ 	# Ensure that the old version of the file is missing
+ 	git -C client rev-list other --quiet --objects --missing=print \
+ 		>observed.oids &&
+ 	grep "$(git -C server rev-parse message1:a.txt)" observed.oids &&
+ 
+ 	# Ensure that client passes fsck
+ 	git -C client fsck
+ '
+ 
+ test_expect_success 'do not advertise filter if not configured to do so' '
+ 	SERVER="file://$(pwd)/server" &&
+ 
+ 	rm "$(pwd)/trace" &&
+ 	git -C server config uploadpack.allowfilter 1 &&
+ 	GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 \
+ 		ls-remote "$SERVER" &&
+ 	grep "fetch=.*filter" trace &&
+ 
+ 	rm "$(pwd)/trace" &&
+ 	git -C server config uploadpack.allowfilter 0 &&
+ 	GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 \
+ 		ls-remote "$SERVER" &&
+ 	grep "fetch=" trace >fetch_capabilities &&
+ 	! grep filter fetch_capabilities
+ '
+ 
+ test_expect_success 'partial clone warns if filter is not advertised' '
+ 	rm -rf client &&
+ 	git -C server config uploadpack.allowfilter 0 &&
+ 	git -c protocol.version=2 \
+ 		clone --filter=blob:none "file://$(pwd)/server" client 2>err &&
+ 	test_i18ngrep "filtering not recognized by server, ignoring" err
+ '
+ 
+ test_expect_success 'even with handcrafted request, filter does not work if not advertised' '
+ 	git -C server config uploadpack.allowfilter 0 &&
+ 
+ 	# Custom request that tries to filter even though it is not advertised.
+ 	test-pkt-line pack >in <<-EOF &&
+ 	command=fetch
+ 	0001
+ 	want $(git -C server rev-parse master)
+ 	filter blob:none
+ 	0000
+ 	EOF
+ 
+ 	test_must_fail git -C server serve --stateless-rpc <in >/dev/null 2>err &&
+ 	grep "unexpected line: .filter blob:none." err &&
+ 
+ 	# Exercise to ensure that if advertised, filter works
+ 	git -C server config uploadpack.allowfilter 1 &&
+ 	git -C server serve --stateless-rpc <in >/dev/null
+ '
+ 
  # Test protocol v2 with 'http://' transport
  #
  . "$TEST_DIRECTORY"/lib-httpd.sh
diff --combined upload-pack.c
index bacc92bbb6,82c16cae32..87c6722ea5
--- a/upload-pack.c
+++ b/upload-pack.c
@@@ -444,7 -444,7 +444,7 @@@ static int get_common_commits(void
  				break;
  			default:
  				got_common = 1;
 -				memcpy(last_hex, oid_to_hex(&oid), 41);
 +				oid_to_hex_r(last_hex, &oid);
  				if (multi_ack == 2)
  					packet_write_fmt(1, "ACK %s common\n", last_hex);
  				else if (multi_ack)
@@@ -486,7 -486,7 +486,7 @@@ static int do_reachable_revlist(struct 
  		"rev-list", "--stdin", NULL,
  	};
  	struct object *o;
 -	char namebuf[42]; /* ^ + SHA-1 + LF */
 +	char namebuf[GIT_MAX_HEXSZ + 2]; /* ^ + hash + LF */
  	int i;
  
  	cmd->argv = argv;
@@@ -555,17 -555,15 +555,17 @@@ static int get_reachable_list(struct ob
  	struct child_process cmd = CHILD_PROCESS_INIT;
  	int i;
  	struct object *o;
 -	char namebuf[42]; /* ^ + SHA-1 + LF */
 +	char namebuf[GIT_MAX_HEXSZ + 2]; /* ^ + hash + LF */
 +	const unsigned hexsz = the_hash_algo->hexsz;
  
  	if (do_reachable_revlist(&cmd, src, reachable) < 0)
  		return -1;
  
 -	while ((i = read_in_full(cmd.out, namebuf, 41)) == 41) {
 +	while ((i = read_in_full(cmd.out, namebuf, hexsz + 1)) == hexsz + 1) {
  		struct object_id sha1;
 +		const char *p;
  
 -		if (namebuf[40] != '\n' || get_oid_hex(namebuf, &sha1))
 +		if (parse_oid_hex(namebuf, &sha1, &p) || *p != '\n')
  			break;
  
  		o = lookup_object(sha1.hash);
@@@ -897,9 -895,11 +897,9 @@@ static void receive_needs(void
  		}
  
  		if (!skip_prefix(line, "want ", &arg) ||
 -		    get_oid_hex(arg, &oid_buf))
 +		    parse_oid_hex(arg, &oid_buf, &features))
  			die("git upload-pack: protocol error, "
 -			    "expected to get sha, not '%s'", line);
 -
 -		features = arg + 40;
 +			    "expected to get object ID, not '%s'", line);
  
  		if (parse_feature_request(features, "deepen-relative"))
  			deepen_relative = 1;
@@@ -1205,6 -1205,7 +1205,7 @@@ static void process_args(struct packet_
  {
  	while (packet_reader_read(request) != PACKET_READ_FLUSH) {
  		const char *arg = request->line;
+ 		const char *p;
  
  		/* process want */
  		if (parse_want(arg))
@@@ -1251,8 -1252,13 +1252,13 @@@
  			continue;
  		}
  
+ 		if (allow_filter && skip_prefix(arg, "filter ", &p)) {
+ 			parse_list_objects_filter(&filter_options, p);
+ 			continue;
+ 		}
+ 
  		/* ignore unknown lines maybe? */
- 		die("unexpect line: '%s'", arg);
+ 		die("unexpected line: '%s'", arg);
  	}
  }
  
@@@ -1376,6 -1382,8 +1382,8 @@@ int upload_pack_v2(struct repository *r
  	enum fetch_state state = FETCH_PROCESS_ARGS;
  	struct upload_pack_data data;
  
+ 	git_config(upload_pack_config, NULL);
+ 
  	upload_pack_data_init(&data);
  	use_sideband = LARGE_PACKET_MAX;
  
@@@ -1428,7 -1436,14 +1436,14 @@@
  int upload_pack_advertise(struct repository *r,
  			  struct strbuf *value)
  {
- 	if (value)
+ 	if (value) {
+ 		int allow_filter_value;
  		strbuf_addstr(value, "shallow");
+ 		if (!repo_config_get_bool(the_repository,
+ 					 "uploadpack.allowfilter",
+ 					 &allow_filter_value) &&
+ 		    allow_filter_value)
+ 			strbuf_addstr(value, " filter");
+ 	}
  	return 1;
  }