Transfer protocol v2 learned to support the partial clone.
* jt/partial-clone-proto-v2:
{fetch,upload}-pack: support filter in protocol v2
upload-pack: read config when serving protocol v2
upload-pack: fix error message typo
Cannot be used with "deepen", but can be used with
"deepen-since".
+ If the 'filter' feature is advertised, the following argument can be
+ included in the client's request:
+
+ filter <filter-spec>
+ Request that various objects from the packfile be omitted
+ using one of several filtering techniques. These are intended
+ for use with partial clone and partial fetch operations. See
+ `rev-list` for possible "filter-spec" values.
+
The response of `fetch` is broken into a number of sections separated by
delimiter packets (0001), with each section beginning with its section
header.
1 - pack data
2 - progress messages
3 - fatal error message just before stream aborts
+
+ server-option
+~~~~~~~~~~~~~~~
+
+If advertised, indicates that any number of server specific options can be
+included in a request. This is done by sending each option as a
+"server-option=<option>" capability line in the capability-list section of
+a request.
+
+The provided options must not contain a NUL or LF character.
packet_buf_write(&req_buf, "command=fetch");
if (server_supports_v2("agent", 0))
packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
+ if (args->server_options && args->server_options->nr &&
+ server_supports_v2("server-option", 1)) {
+ int i;
+ for (i = 0; i < args->server_options->nr; i++)
+ packet_write_fmt(fd_out, "server-option=%s",
+ args->server_options->items[i].string);
+ }
packet_buf_delim(&req_buf);
if (args->use_thin_pack)
else if (is_repository_shallow() || args->deepen)
die(_("Server does not support shallow requests"));
+ /* Add filter */
+ if (server_supports_feature("fetch", "filter", 0) &&
+ args->filter_options.choice) {
+ print_verbose(args, _("Server supports filter"));
+ packet_buf_write(&req_buf, "filter %s",
+ args->filter_options.filter_spec);
+ } else if (args->filter_options.choice) {
+ warning("filtering not recognized by server, ignoring");
+ }
+
/* add wants */
add_wants(wants, &req_buf);
- /* Add all of the common commits we've found in previous rounds */
- add_common(&req_buf, common);
+ if (args->no_dependents) {
+ packet_buf_write(&req_buf, "done");
+ ret = 1;
+ } else {
+ /* Add all of the common commits we've found in previous rounds */
+ add_common(&req_buf, common);
- /* Add initial haves */
- ret = add_haves(&req_buf, haves_to_send, in_vain);
+ /* Add initial haves */
+ ret = add_haves(&req_buf, haves_to_send, in_vain);
+ }
/* Send request */
packet_buf_flush(&req_buf);
agent=git/$(git version | cut -d" " -f3)
ls-refs
fetch=shallow
+ server-option
0000
EOF
test_cmp actual expect
'
+test_expect_success 'sending server-options' '
+ test-pkt-line pack >in <<-EOF &&
+ command=ls-refs
+ server-option=hello
+ server-option=world
+ 0001
+ ref-prefix HEAD
+ 0000
+ EOF
+
+ cat >expect <<-EOF &&
+ $(git rev-parse HEAD) HEAD
+ 0000
+ EOF
+
+ git serve --stateless-rpc <in >out &&
+ test-pkt-line unpack <out >actual &&
+ test_cmp actual expect
+'
+
+ test_expect_success 'unexpected lines are not allowed in fetch request' '
+ git init server &&
+
+ test-pkt-line pack >in <<-EOF &&
+ command=fetch
+ 0001
+ this-is-not-a-command
+ 0000
+ EOF
+
+ test_must_fail git -C server serve --stateless-rpc <in >/dev/null 2>err &&
+ grep "unexpected line: .this-is-not-a-command." err
+ '
+
test_done
test_cmp actual expect
'
+test_expect_success 'server-options are sent when using ls-remote' '
+ test_when_finished "rm -f log" &&
+
+ GIT_TRACE_PACKET="$(pwd)/log" git -c protocol.version=2 \
+ ls-remote -o hello -o world "file://$(pwd)/file_parent" master >actual &&
+
+ cat >expect <<-EOF &&
+ $(git -C file_parent rev-parse refs/heads/master)$(printf "\t")refs/heads/master
+ EOF
+
+ test_cmp actual expect &&
+ grep "server-option=hello" log &&
+ grep "server-option=world" log
+'
+
+
test_expect_success 'clone with file:// using protocol v2' '
test_when_finished "rm -f log" &&
! grep "refs/tags/three" log
'
+test_expect_success 'server-options are sent when fetching' '
+ test_when_finished "rm -f log" &&
+
+ test_commit -C file_parent four &&
+
+ GIT_TRACE_PACKET="$(pwd)/log" git -C file_child -c protocol.version=2 \
+ fetch -o hello -o world origin master &&
+
+ git -C file_child log -1 --format=%s origin/master >actual &&
+ git -C file_parent log -1 --format=%s >expect &&
+ test_cmp expect actual &&
+
+ grep "server-option=hello" log &&
+ grep "server-option=world" log
+'
+
+ test_expect_success 'upload-pack respects config using protocol v2' '
+ git init server &&
+ write_script server/.git/hook <<-\EOF &&
+ touch hookout
+ "$@"
+ EOF
+ test_commit -C server one &&
+
+ test_config_global uploadpack.packobjectshook ./hook &&
+ test_path_is_missing server/.git/hookout &&
+ git -c protocol.version=2 clone "file://$(pwd)/server" client &&
+ test_path_is_file server/.git/hookout
+ '
+
+ test_expect_success 'setup filter tests' '
+ rm -rf server client &&
+ git init server &&
+
+ # 1 commit to create a file, and 1 commit to modify it
+ test_commit -C server message1 a.txt &&
+ test_commit -C server message2 a.txt &&
+ git -C server config protocol.version 2 &&
+ git -C server config uploadpack.allowfilter 1 &&
+ git -C server config uploadpack.allowanysha1inwant 1 &&
+ git -C server config protocol.version 2
+ '
+
+ test_expect_success 'partial clone' '
+ GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 \
+ clone --filter=blob:none "file://$(pwd)/server" client &&
+ grep "version 2" trace &&
+
+ # Ensure that the old version of the file is missing
+ git -C client rev-list master --quiet --objects --missing=print \
+ >observed.oids &&
+ grep "$(git -C server rev-parse message1:a.txt)" observed.oids &&
+
+ # Ensure that client passes fsck
+ git -C client fsck
+ '
+
+ test_expect_success 'dynamically fetch missing object' '
+ rm "$(pwd)/trace" &&
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client -c protocol.version=2 \
+ cat-file -p $(git -C server rev-parse message1:a.txt) &&
+ grep "version 2" trace
+ '
+
+ test_expect_success 'partial fetch' '
+ rm -rf client "$(pwd)/trace" &&
+ git init client &&
+ SERVER="file://$(pwd)/server" &&
+ test_config -C client extensions.partialClone "$SERVER" &&
+
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client -c protocol.version=2 \
+ fetch --filter=blob:none "$SERVER" master:refs/heads/other &&
+ grep "version 2" trace &&
+
+ # Ensure that the old version of the file is missing
+ git -C client rev-list other --quiet --objects --missing=print \
+ >observed.oids &&
+ grep "$(git -C server rev-parse message1:a.txt)" observed.oids &&
+
+ # Ensure that client passes fsck
+ git -C client fsck
+ '
+
+ test_expect_success 'do not advertise filter if not configured to do so' '
+ SERVER="file://$(pwd)/server" &&
+
+ rm "$(pwd)/trace" &&
+ git -C server config uploadpack.allowfilter 1 &&
+ GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 \
+ ls-remote "$SERVER" &&
+ grep "fetch=.*filter" trace &&
+
+ rm "$(pwd)/trace" &&
+ git -C server config uploadpack.allowfilter 0 &&
+ GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 \
+ ls-remote "$SERVER" &&
+ grep "fetch=" trace >fetch_capabilities &&
+ ! grep filter fetch_capabilities
+ '
+
+ test_expect_success 'partial clone warns if filter is not advertised' '
+ rm -rf client &&
+ git -C server config uploadpack.allowfilter 0 &&
+ git -c protocol.version=2 \
+ clone --filter=blob:none "file://$(pwd)/server" client 2>err &&
+ test_i18ngrep "filtering not recognized by server, ignoring" err
+ '
+
+ test_expect_success 'even with handcrafted request, filter does not work if not advertised' '
+ git -C server config uploadpack.allowfilter 0 &&
+
+ # Custom request that tries to filter even though it is not advertised.
+ test-pkt-line pack >in <<-EOF &&
+ command=fetch
+ 0001
+ want $(git -C server rev-parse master)
+ filter blob:none
+ 0000
+ EOF
+
+ test_must_fail git -C server serve --stateless-rpc <in >/dev/null 2>err &&
+ grep "unexpected line: .filter blob:none." err &&
+
+ # Exercise to ensure that if advertised, filter works
+ git -C server config uploadpack.allowfilter 1 &&
+ git -C server serve --stateless-rpc <in >/dev/null
+ '
+
# Test protocol v2 with 'http://' transport
#
. "$TEST_DIRECTORY"/lib-httpd.sh
break;
default:
got_common = 1;
- memcpy(last_hex, oid_to_hex(&oid), 41);
+ oid_to_hex_r(last_hex, &oid);
if (multi_ack == 2)
packet_write_fmt(1, "ACK %s common\n", last_hex);
else if (multi_ack)
"rev-list", "--stdin", NULL,
};
struct object *o;
- char namebuf[42]; /* ^ + SHA-1 + LF */
+ char namebuf[GIT_MAX_HEXSZ + 2]; /* ^ + hash + LF */
int i;
cmd->argv = argv;
struct child_process cmd = CHILD_PROCESS_INIT;
int i;
struct object *o;
- char namebuf[42]; /* ^ + SHA-1 + LF */
+ char namebuf[GIT_MAX_HEXSZ + 2]; /* ^ + hash + LF */
+ const unsigned hexsz = the_hash_algo->hexsz;
if (do_reachable_revlist(&cmd, src, reachable) < 0)
return -1;
- while ((i = read_in_full(cmd.out, namebuf, 41)) == 41) {
+ while ((i = read_in_full(cmd.out, namebuf, hexsz + 1)) == hexsz + 1) {
struct object_id sha1;
+ const char *p;
- if (namebuf[40] != '\n' || get_oid_hex(namebuf, &sha1))
+ if (parse_oid_hex(namebuf, &sha1, &p) || *p != '\n')
break;
o = lookup_object(sha1.hash);
}
if (!skip_prefix(line, "want ", &arg) ||
- get_oid_hex(arg, &oid_buf))
+ parse_oid_hex(arg, &oid_buf, &features))
die("git upload-pack: protocol error, "
- "expected to get sha, not '%s'", line);
-
- features = arg + 40;
+ "expected to get object ID, not '%s'", line);
if (parse_feature_request(features, "deepen-relative"))
deepen_relative = 1;
{
while (packet_reader_read(request) != PACKET_READ_FLUSH) {
const char *arg = request->line;
+ const char *p;
/* process want */
if (parse_want(arg))
continue;
}
+ if (allow_filter && skip_prefix(arg, "filter ", &p)) {
+ parse_list_objects_filter(&filter_options, p);
+ continue;
+ }
+
/* ignore unknown lines maybe? */
- die("unexpect line: '%s'", arg);
+ die("unexpected line: '%s'", arg);
}
}
enum fetch_state state = FETCH_PROCESS_ARGS;
struct upload_pack_data data;
+ git_config(upload_pack_config, NULL);
+
upload_pack_data_init(&data);
use_sideband = LARGE_PACKET_MAX;
int upload_pack_advertise(struct repository *r,
struct strbuf *value)
{
- if (value)
+ if (value) {
+ int allow_filter_value;
strbuf_addstr(value, "shallow");
+ if (!repo_config_get_bool(the_repository,
+ "uploadpack.allowfilter",
+ &allow_filter_value) &&
+ allow_filter_value)
+ strbuf_addstr(value, " filter");
+ }
return 1;
}