Merge branch 'jt/partial-clone-proto-v2'
authorJunio C Hamano <gitster@pobox.com>
Wed, 30 May 2018 05:04:10 +0000 (14:04 +0900)
committerJunio C Hamano <gitster@pobox.com>
Wed, 30 May 2018 05:04:10 +0000 (14:04 +0900)
Transfer protocol v2 learned to support the partial clone.

* jt/partial-clone-proto-v2:
{fetch,upload}-pack: support filter in protocol v2
upload-pack: read config when serving protocol v2
upload-pack: fix error message typo

Documentation/technical/protocol-v2.txt
fetch-pack.c
t/t5701-git-serve.sh
t/t5702-protocol-v2.sh
upload-pack.c
index d7b6f38e0a7830f91cec19d566ed6d7868464bb7..49bda76d231d08173a48593d0910b3cbc934f01c 100644 (file)
@@ -290,6 +290,15 @@ included in the clients request as well as the potential addition of the
        Cannot be used with "deepen", but can be used with
        "deepen-since".
 
+If the 'filter' feature is advertised, the following argument can be
+included in the client's request:
+
+    filter <filter-spec>
+       Request that various objects from the packfile be omitted
+       using one of several filtering techniques. These are intended
+       for use with partial clone and partial fetch operations. See
+       `rev-list` for possible "filter-spec" values.
+
 The response of `fetch` is broken into a number of sections separated by
 delimiter packets (0001), with each section beginning with its section
 header.
index 490c38f833419cde65b97f75f454a7d148fff6a2..a320ce9872b5e752aeb592a16cc9108ae87e9d71 100644 (file)
@@ -1198,14 +1198,29 @@ static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
        else if (is_repository_shallow() || args->deepen)
                die(_("Server does not support shallow requests"));
 
+       /* Add filter */
+       if (server_supports_feature("fetch", "filter", 0) &&
+           args->filter_options.choice) {
+               print_verbose(args, _("Server supports filter"));
+               packet_buf_write(&req_buf, "filter %s",
+                                args->filter_options.filter_spec);
+       } else if (args->filter_options.choice) {
+               warning("filtering not recognized by server, ignoring");
+       }
+
        /* add wants */
        add_wants(wants, &req_buf);
 
-       /* Add all of the common commits we've found in previous rounds */
-       add_common(&req_buf, common);
+       if (args->no_dependents) {
+               packet_buf_write(&req_buf, "done");
+               ret = 1;
+       } else {
+               /* Add all of the common commits we've found in previous rounds */
+               add_common(&req_buf, common);
 
-       /* Add initial haves */
-       ret = add_haves(&req_buf, haves_to_send, in_vain);
+               /* Add initial haves */
+               ret = add_haves(&req_buf, haves_to_send, in_vain);
+       }
 
        /* Send request */
        packet_buf_flush(&req_buf);
index 011a5796db5d98af3b2151563a8e9683b1139f1a..75ec79e6cb839e71f6830d9301d1982d0fc68925 100755 (executable)
@@ -194,4 +194,18 @@ test_expect_success 'sending server-options' '
        test_cmp actual expect
 '
 
+test_expect_success 'unexpected lines are not allowed in fetch request' '
+       git init server &&
+
+       test-pkt-line pack >in <<-EOF &&
+       command=fetch
+       0001
+       this-is-not-a-command
+       0000
+       EOF
+
+       test_must_fail git -C server serve --stateless-rpc <in >/dev/null 2>err &&
+       grep "unexpected line: .this-is-not-a-command." err
+'
+
 test_done
index dbfd0691c081c26dba76c70834966adaa3aacdae..4ad5c5769b4eab56d840709ee0bc0affa8de2e01 100755 (executable)
@@ -233,6 +233,118 @@ test_expect_success 'server-options are sent when fetching' '
        grep "server-option=world" log
 '
 
+test_expect_success 'upload-pack respects config using protocol v2' '
+       git init server &&
+       write_script server/.git/hook <<-\EOF &&
+               touch hookout
+               "$@"
+       EOF
+       test_commit -C server one &&
+
+       test_config_global uploadpack.packobjectshook ./hook &&
+       test_path_is_missing server/.git/hookout &&
+       git -c protocol.version=2 clone "file://$(pwd)/server" client &&
+       test_path_is_file server/.git/hookout
+'
+
+test_expect_success 'setup filter tests' '
+       rm -rf server client &&
+       git init server &&
+
+       # 1 commit to create a file, and 1 commit to modify it
+       test_commit -C server message1 a.txt &&
+       test_commit -C server message2 a.txt &&
+       git -C server config protocol.version 2 &&
+       git -C server config uploadpack.allowfilter 1 &&
+       git -C server config uploadpack.allowanysha1inwant 1 &&
+       git -C server config protocol.version 2
+'
+
+test_expect_success 'partial clone' '
+       GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 \
+               clone --filter=blob:none "file://$(pwd)/server" client &&
+       grep "version 2" trace &&
+
+       # Ensure that the old version of the file is missing
+       git -C client rev-list master --quiet --objects --missing=print \
+               >observed.oids &&
+       grep "$(git -C server rev-parse message1:a.txt)" observed.oids &&
+
+       # Ensure that client passes fsck
+       git -C client fsck
+'
+
+test_expect_success 'dynamically fetch missing object' '
+       rm "$(pwd)/trace" &&
+       GIT_TRACE_PACKET="$(pwd)/trace" git -C client -c protocol.version=2 \
+               cat-file -p $(git -C server rev-parse message1:a.txt) &&
+       grep "version 2" trace
+'
+
+test_expect_success 'partial fetch' '
+       rm -rf client "$(pwd)/trace" &&
+       git init client &&
+       SERVER="file://$(pwd)/server" &&
+       test_config -C client extensions.partialClone "$SERVER" &&
+
+       GIT_TRACE_PACKET="$(pwd)/trace" git -C client -c protocol.version=2 \
+               fetch --filter=blob:none "$SERVER" master:refs/heads/other &&
+       grep "version 2" trace &&
+
+       # Ensure that the old version of the file is missing
+       git -C client rev-list other --quiet --objects --missing=print \
+               >observed.oids &&
+       grep "$(git -C server rev-parse message1:a.txt)" observed.oids &&
+
+       # Ensure that client passes fsck
+       git -C client fsck
+'
+
+test_expect_success 'do not advertise filter if not configured to do so' '
+       SERVER="file://$(pwd)/server" &&
+
+       rm "$(pwd)/trace" &&
+       git -C server config uploadpack.allowfilter 1 &&
+       GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 \
+               ls-remote "$SERVER" &&
+       grep "fetch=.*filter" trace &&
+
+       rm "$(pwd)/trace" &&
+       git -C server config uploadpack.allowfilter 0 &&
+       GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 \
+               ls-remote "$SERVER" &&
+       grep "fetch=" trace >fetch_capabilities &&
+       ! grep filter fetch_capabilities
+'
+
+test_expect_success 'partial clone warns if filter is not advertised' '
+       rm -rf client &&
+       git -C server config uploadpack.allowfilter 0 &&
+       git -c protocol.version=2 \
+               clone --filter=blob:none "file://$(pwd)/server" client 2>err &&
+       test_i18ngrep "filtering not recognized by server, ignoring" err
+'
+
+test_expect_success 'even with handcrafted request, filter does not work if not advertised' '
+       git -C server config uploadpack.allowfilter 0 &&
+
+       # Custom request that tries to filter even though it is not advertised.
+       test-pkt-line pack >in <<-EOF &&
+       command=fetch
+       0001
+       want $(git -C server rev-parse master)
+       filter blob:none
+       0000
+       EOF
+
+       test_must_fail git -C server serve --stateless-rpc <in >/dev/null 2>err &&
+       grep "unexpected line: .filter blob:none." err &&
+
+       # Exercise to ensure that if advertised, filter works
+       git -C server config uploadpack.allowfilter 1 &&
+       git -C server serve --stateless-rpc <in >/dev/null
+'
+
 # Test protocol v2 with 'http://' transport
 #
 . "$TEST_DIRECTORY"/lib-httpd.sh
index bacc92bbb6b58180aa65b2edf1cd9b5358c3e1cd..87c6722ea58207a9b369e75f7cf7ac16a705330b 100644 (file)
@@ -1205,6 +1205,7 @@ static void process_args(struct packet_reader *request,
 {
        while (packet_reader_read(request) != PACKET_READ_FLUSH) {
                const char *arg = request->line;
+               const char *p;
 
                /* process want */
                if (parse_want(arg))
@@ -1251,8 +1252,13 @@ static void process_args(struct packet_reader *request,
                        continue;
                }
 
+               if (allow_filter && skip_prefix(arg, "filter ", &p)) {
+                       parse_list_objects_filter(&filter_options, p);
+                       continue;
+               }
+
                /* ignore unknown lines maybe? */
-               die("unexpect line: '%s'", arg);
+               die("unexpected line: '%s'", arg);
        }
 }
 
@@ -1376,6 +1382,8 @@ int upload_pack_v2(struct repository *r, struct argv_array *keys,
        enum fetch_state state = FETCH_PROCESS_ARGS;
        struct upload_pack_data data;
 
+       git_config(upload_pack_config, NULL);
+
        upload_pack_data_init(&data);
        use_sideband = LARGE_PACKET_MAX;
 
@@ -1428,7 +1436,14 @@ int upload_pack_v2(struct repository *r, struct argv_array *keys,
 int upload_pack_advertise(struct repository *r,
                          struct strbuf *value)
 {
-       if (value)
+       if (value) {
+               int allow_filter_value;
                strbuf_addstr(value, "shallow");
+               if (!repo_config_get_bool(the_repository,
+                                        "uploadpack.allowfilter",
+                                        &allow_filter_value) &&
+                   allow_filter_value)
+                       strbuf_addstr(value, " filter");
+       }
        return 1;
 }