upload-pack: add object filtering for partial clone

Teach upload-pack to negotiate object filtering over the protocol and
to send filter parameters to pack-objects.  This is intended for partial
clone and fetch.

The idea to make upload-pack configurable using uploadpack.allowFilter
comes from Jonathan Tan's work in [1].

[1] https://public-inbox.org/git/f211093280b422c32cc1b7034130072f35c5ed51.1506714999.git.jonathantanmy@google.com/

Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/config.txt b/Documentation/config.txt
index 1ac0ae6adb0460a3460084aef2302434e24ee6de..e528210c67a4c66bef22e560b425d3bb3bb9f2e6 100644 (file)
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -3268,6 +3268,10 @@ uploadpack.packObjectsHook::
        was run. I.e., `upload-pack` will feed input intended for
        `pack-objects` to the hook, and expects a completed packfile on
        stdout.
        was run. I.e., `upload-pack` will feed input intended for
        `pack-objects` to the hook, and expects a completed packfile on
        stdout.

+
+uploadpack.allowFilter::
+       If this option is set, `upload-pack` will advertise partial
+       clone and partial fetch object filtering.

 +
 Note that this configuration variable is ignored if it is seen in the
 repository-level config (this is a safety measure against fetching from
 +
 Note that this configuration variable is ignored if it is seen in the
 repository-level config (this is a safety measure against fetching from

diff --git a/Documentation/technical/pack-protocol.txt b/Documentation/technical/pack-protocol.txt
index ed1eae8b83a651d34f9ce2488f58d02a763fbfd4..a43a113e44f3d959f2a527a7039c28199d12be25 100644 (file)
--- a/Documentation/technical/pack-protocol.txt
+++ b/Documentation/technical/pack-protocol.txt
@@ -212,6 +212,7 @@ out of what the server said it could do with the first 'want' line.
   upload-request    =  want-list
                       *shallow-line
                       *1depth-request
   upload-request    =  want-list
                       *shallow-line
                       *1depth-request

+                      [filter-request]

                       flush-pkt
 
   want-list         =  first-want
                       flush-pkt
 
   want-list         =  first-want


@@ -227,6 +228,8 @@ out of what the server said it could do with the first 'want' line.
   additional-want   =  PKT-LINE("want" SP obj-id)
 
   depth             =  1*DIGIT
   additional-want   =  PKT-LINE("want" SP obj-id)
 
   depth             =  1*DIGIT

+
+  filter-request    =  PKT-LINE("filter" SP filter-spec)

 ----
 
 Clients MUST send all the obj-ids it wants from the reference
 ----
 
 Clients MUST send all the obj-ids it wants from the reference


@@ -249,6 +252,11 @@ complete those commits. Commits whose parents are not received as a
 result are defined as shallow and marked as such in the server. This
 information is sent back to the client in the next step.
 
 result are defined as shallow and marked as such in the server. This
 information is sent back to the client in the next step.
 

+The client can optionally request that pack-objects omit various
+objects from the packfile using one of several filtering techniques.
+These are intended for use with partial clone and partial fetch
+operations.  See `rev-list` for possible "filter-spec" values.
+

 Once all the 'want's and 'shallow's (and optional 'deepen') are
 transferred, clients MUST send a flush-pkt, to tell the server side
 that it is done sending the list.
 Once all the 'want's and 'shallow's (and optional 'deepen') are
 transferred, clients MUST send a flush-pkt, to tell the server side
 that it is done sending the list.

diff --git a/Documentation/technical/protocol-capabilities.txt b/Documentation/technical/protocol-capabilities.txt
index 26dcc6f502020da5214f1e161f1678813dd24059..332d209b58ca42dc303c757f2b31f7a4f3d033f4 100644 (file)
--- a/Documentation/technical/protocol-capabilities.txt
+++ b/Documentation/technical/protocol-capabilities.txt
@@ -309,3 +309,11 @@ to accept a signed push certificate, and asks the <nonce> to be
 included in the push certificate.  A send-pack client MUST NOT
 send a push-cert packet unless the receive-pack server advertises
 this capability.
 included in the push certificate.  A send-pack client MUST NOT
 send a push-cert packet unless the receive-pack server advertises
 this capability.

+
+filter
+------
+
+If the upload-pack server advertises the 'filter' capability,
+fetch-pack may send "filter" commands to request a partial clone
+or partial fetch and request that the server omit various objects
+from the packfile.

diff --git a/upload-pack.c b/upload-pack.c
index e25f725c0feaa57c9a09c976628217c99dfb5652..e6d38b9ca06797ab369b0f583a79ced3ecdbf128 100644 (file)
--- a/upload-pack.c
+++ b/upload-pack.c
@@ -10,6 +10,8 @@
 #include "diff.h"
 #include "revision.h"
 #include "list-objects.h"
 #include "diff.h"
 #include "revision.h"
 #include "list-objects.h"

+#include "list-objects-filter.h"
+#include "list-objects-filter-options.h"

 #include "run-command.h"
 #include "connect.h"
 #include "sigchain.h"
 #include "run-command.h"
 #include "connect.h"
 #include "sigchain.h"


@@ -18,6 +20,7 @@
 #include "parse-options.h"
 #include "argv-array.h"
 #include "prio-queue.h"
 #include "parse-options.h"
 #include "argv-array.h"
 #include "prio-queue.h"

+#include "quote.h"

 
 static const char * const upload_pack_usage[] = {
        N_("git upload-pack [<options>] <dir>"),
 
 static const char * const upload_pack_usage[] = {
        N_("git upload-pack [<options>] <dir>"),


@@ -64,6 +67,10 @@ static int advertise_refs;
 static int stateless_rpc;
 static const char *pack_objects_hook;
 
 static int stateless_rpc;
 static const char *pack_objects_hook;
 

+static int filter_capability_requested;
+static int filter_advertise;
+static struct list_objects_filter_options filter_options;
+

 static void reset_timeout(void)
 {
        alarm(timeout);
 static void reset_timeout(void)
 {
        alarm(timeout);


@@ -131,6 +138,12 @@ static void create_pack_file(void)
                argv_array_push(&pack_objects.args, "--delta-base-offset");
        if (use_include_tag)
                argv_array_push(&pack_objects.args, "--include-tag");
                argv_array_push(&pack_objects.args, "--delta-base-offset");
        if (use_include_tag)
                argv_array_push(&pack_objects.args, "--include-tag");

+       if (filter_options.filter_spec) {
+               struct strbuf buf = STRBUF_INIT;
+               sq_quote_buf(&buf, filter_options.filter_spec);
+               argv_array_pushf(&pack_objects.args, "--filter=%s", buf.buf);
+               strbuf_release(&buf);
+       }

 
        pack_objects.in = -1;
        pack_objects.out = -1;
 
        pack_objects.in = -1;
        pack_objects.out = -1;


@@ -794,6 +807,12 @@ static void receive_needs(void)
                        deepen_rev_list = 1;
                        continue;
                }
                        deepen_rev_list = 1;
                        continue;
                }

+               if (skip_prefix(line, "filter ", &arg)) {
+                       if (!filter_capability_requested)
+                               die("git upload-pack: filtering capability not negotiated");
+                       parse_list_objects_filter(&filter_options, arg);
+                       continue;
+               }

                if (!skip_prefix(line, "want ", &arg) ||
                    get_oid_hex(arg, &oid_buf))
                        die("git upload-pack: protocol error, "
                if (!skip_prefix(line, "want ", &arg) ||
                    get_oid_hex(arg, &oid_buf))
                        die("git upload-pack: protocol error, "


@@ -821,6 +840,8 @@ static void receive_needs(void)
                        no_progress = 1;
                if (parse_feature_request(features, "include-tag"))
                        use_include_tag = 1;
                        no_progress = 1;
                if (parse_feature_request(features, "include-tag"))
                        use_include_tag = 1;

+               if (parse_feature_request(features, "filter"))
+                       filter_capability_requested = 1;

 
                o = parse_object(&oid_buf);
                if (!o) {
 
                o = parse_object(&oid_buf);
                if (!o) {


@@ -940,7 +961,7 @@ static int send_ref(const char *refname, const struct object_id *oid,
                struct strbuf symref_info = STRBUF_INIT;
 
                format_symref_info(&symref_info, cb_data);
                struct strbuf symref_info = STRBUF_INIT;
 
                format_symref_info(&symref_info, cb_data);

-               packet_write_fmt(1, "%s %s%c%s%s%s%s%s agent=%s\n",
+               packet_write_fmt(1, "%s %s%c%s%s%s%s%s%s agent=%s\n",

                             oid_to_hex(oid), refname_nons,
                             0, capabilities,
                             (allow_unadvertised_object_request & ALLOW_TIP_SHA1) ?
                             oid_to_hex(oid), refname_nons,
                             0, capabilities,
                             (allow_unadvertised_object_request & ALLOW_TIP_SHA1) ?


@@ -949,6 +970,7 @@ static int send_ref(const char *refname, const struct object_id *oid,
                                     " allow-reachable-sha1-in-want" : "",
                             stateless_rpc ? " no-done" : "",
                             symref_info.buf,
                                     " allow-reachable-sha1-in-want" : "",
                             stateless_rpc ? " no-done" : "",
                             symref_info.buf,

+                            filter_advertise ? " filter" : "",

                             git_user_agent_sanitized());
                strbuf_release(&symref_info);
        } else {
                             git_user_agent_sanitized());
                strbuf_release(&symref_info);
        } else {


@@ -1027,6 +1049,8 @@ static int upload_pack_config(const char *var, const char *value, void *unused)
        } else if (current_config_scope() != CONFIG_SCOPE_REPO) {
                if (!strcmp("uploadpack.packobjectshook", var))
                        return git_config_string(&pack_objects_hook, var, value);
        } else if (current_config_scope() != CONFIG_SCOPE_REPO) {
                if (!strcmp("uploadpack.packobjectshook", var))
                        return git_config_string(&pack_objects_hook, var, value);

+       } else if (!strcmp("uploadpack.allowfilter", var)) {
+               filter_advertise = git_config_bool(var, value);

        }
        return parse_hide_refs_config(var, value, "uploadpack");
 }
        }
        return parse_hide_refs_config(var, value, "uploadpack");
 }