Merge branch 'jt/partial-clone-fsck-connectivity'

Partial clone support of "git clone" has been updated to correctly
validate the objects it receives from the other side.  The server
side has been corrected to send objects that are directly
requested, even if they may match the filtering criteria (e.g. when
doing a "lazy blob" partial clone).

* jt/partial-clone-fsck-connectivity:
clone: check connectivity even if clone is partial
upload-pack: send refs' objects despite "filter"

diff --git a/Documentation/technical/pack-protocol.txt b/Documentation/technical/pack-protocol.txt
index 7fee6b780a869fcb9af979fb95a54587002b92f2..508a344cf19fa702219c62b61ef5e04960c246b9 100644 (file)
--- a/Documentation/technical/pack-protocol.txt
+++ b/Documentation/technical/pack-protocol.txt
@@ -284,7 +284,9 @@ information is sent back to the client in the next step.
 The client can optionally request that pack-objects omit various
 objects from the packfile using one of several filtering techniques.
 These are intended for use with partial clone and partial fetch
-operations.  See `rev-list` for possible "filter-spec" values.
+operations. An object that does not meet a filter-spec value is
+omitted unless explicitly requested in a 'want' line. See `rev-list`
+for possible filter-spec values.
 
 Once all the 'want's and 'shallow's (and optional 'deepen') are
 transferred, clients MUST send a flush-pkt, to tell the server side

diff --git a/builtin/clone.c b/builtin/clone.c
index 2f84717e11c22e6176cd28f850c756aad0b3e2fe..5c439f1394c9a097e48f24d73790f276a958bf2d 100644 (file)
--- a/builtin/clone.c
+++ b/builtin/clone.c
@@ -1202,7 +1202,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 
        update_remote_refs(refs, mapped_refs, remote_head_points_at,
                           branch_top.buf, reflog_msg.buf, transport,
-                          !is_local && !filter_options.choice);
+                          !is_local);
 
        update_head(our_head_points_at, remote_head, reflog_msg.buf);
 

diff --git a/list-objects.c b/list-objects.c
index 3e5e1992eb0cd1c535ed42e9729941403ca89054..b1f2138c294adb50cacdcf43601ff82d69eebbae 100644 (file)
--- a/list-objects.c
+++ b/list-objects.c
@@ -48,7 +48,7 @@ static void process_blob(struct rev_info *revs,
 
        pathlen = path->len;
        strbuf_addstr(path, name);
-       if (filter_fn)
+       if (!(obj->flags & USER_GIVEN) && filter_fn)
                r = filter_fn(LOFS_BLOB, obj,
                              path->buf, &path->buf[pathlen],
                              filter_data);
@@ -133,7 +133,7 @@ static void process_tree(struct rev_info *revs,
        }
 
        strbuf_addstr(base, name);
-       if (filter_fn)
+       if (!(obj->flags & USER_GIVEN) && filter_fn)
                r = filter_fn(LOFS_BEGIN_TREE, obj,
                              base->buf, &base->buf[baselen],
                              filter_data);
@@ -172,7 +172,7 @@ static void process_tree(struct rev_info *revs,
                                     cb_data, filter_fn, filter_data);
        }
 
-       if (filter_fn) {
+       if (!(obj->flags & USER_GIVEN) && filter_fn) {
                r = filter_fn(LOFS_END_TREE, obj,
                              base->buf, &base->buf[baselen],
                              filter_data);

diff --git a/object.h b/object.h
index 1b96073601f2cd239e7fe6923a078572ee820bcc..3afd123d735d1d5ac45da096e889d00f0b901161 100644 (file)
--- a/object.h
+++ b/object.h
@@ -53,7 +53,7 @@ struct object_array {
 
 /*
  * object flag allocation:
- * revision.h:               0---------10                                26
+ * revision.h:               0---------10                              2526
  * fetch-pack.c:             0----5
  * walker.c:                 0-2
  * upload-pack.c:                4       11----------------19

diff --git a/revision.c b/revision.c
index 72abe235e4a8045b92d40b18c7d1fb574456e38c..a2570397b6685fbd6e3a7d77ad7e2c554cc03772 100644 (file)
--- a/revision.c
+++ b/revision.c
@@ -175,6 +175,7 @@ static void add_pending_object_with_path(struct rev_info *revs,
                strbuf_release(&buf);
                return; /* do not add the commit itself */
        }
+       obj->flags |= USER_GIVEN;
        add_object_array_with_path(obj, name, &revs->pending, mode, path);
 }
 

diff --git a/revision.h b/revision.h
index bf2239f87689d9486c4b888bed5cba6affa21953..c599c34da91e57572b1c5b315c353d33399e8a09 100644 (file)
--- a/revision.h
+++ b/revision.h
@@ -20,8 +20,9 @@
 #define SYMMETRIC_LEFT (1u<<8)
 #define PATCHSAME      (1u<<9)
 #define BOTTOM         (1u<<10)
+#define USER_GIVEN     (1u<<25) /* given directly by the user */
 #define TRACK_LINEAR   (1u<<26)
-#define ALL_REV_FLAGS  (((1u<<11)-1) | TRACK_LINEAR)
+#define ALL_REV_FLAGS  (((1u<<11)-1) | USER_GIVEN | TRACK_LINEAR)
 
 #define DECORATE_SHORT_REFS    1
 #define DECORATE_FULL_REFS     2

diff --git a/t/t5317-pack-objects-filter-objects.sh b/t/t5317-pack-objects-filter-objects.sh
index 1b0acc383bb216a3f420cd19ff066275a5b4706e..6710c8bc8c4699035bec418258fcaec8a634855c 100755 (executable)
--- a/t/t5317-pack-objects-filter-objects.sh
+++ b/t/t5317-pack-objects-filter-objects.sh
@@ -160,6 +160,22 @@ test_expect_success 'verify blob:limit=1k' '
        test_cmp observed expected
 '
 
+test_expect_success 'verify explicitly specifying oversized blob in input' '
+       git -C r2 ls-files -s large.1000 large.10000 \
+               | awk -f print_2.awk \
+               | sort >expected &&
+       git -C r2 pack-objects --rev --stdout --filter=blob:limit=1k >filter.pack <<-EOF &&
+       HEAD
+       $(git -C r2 rev-parse HEAD:large.10000)
+       EOF
+       git -C r2 index-pack ../filter.pack &&
+       git -C r2 verify-pack -v ../filter.pack \
+               | grep blob \
+               | awk -f print_1.awk \
+               | sort >observed &&
+       test_cmp observed expected
+'
+
 test_expect_success 'verify blob:limit=1m' '
        git -C r2 ls-files -s large.1000 large.10000 \
                | awk -f print_2.awk \

diff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh
index cee556536757128861e166ea8e2cab7f975282c6..44d8e80171a646bc6b4172a38763ca6355c94901 100755 (executable)
--- a/t/t5616-partial-clone.sh
+++ b/t/t5616-partial-clone.sh
@@ -154,4 +154,68 @@ test_expect_success 'partial clone with transfer.fsckobjects=1 uses index-pack -
        grep "git index-pack.*--fsck-objects" trace
 '
 
+test_expect_success 'partial clone fetches blobs pointed to by refs even if normally filtered out' '
+       rm -rf src dst &&
+       git init src &&
+       test_commit -C src x &&
+       test_config -C src uploadpack.allowfilter 1 &&
+       test_config -C src uploadpack.allowanysha1inwant 1 &&
+
+       # Create a tag pointing to a blob.
+       BLOB=$(echo blob-contents | git -C src hash-object --stdin -w) &&
+       git -C src tag myblob "$BLOB" &&
+
+       git clone --filter="blob:none" "file://$(pwd)/src" dst 2>err &&
+       ! grep "does not point to a valid object" err &&
+       git -C dst fsck
+'
+
+. "$TEST_DIRECTORY"/lib-httpd.sh
+start_httpd
+
+# Converts bytes into a form suitable for inclusion in a sed command. For
+# example, "printf 'ab\r\n' | hex_unpack" results in '\x61\x62\x0d\x0a'.
+sed_escape () {
+       perl -e '$/ = undef; $input = <>; print unpack("H2" x length($input), $input)' |
+               sed 's/\(..\)/\\x\1/g'
+}
+
+test_expect_success 'upon cloning, check that all refs point to objects' '
+       SERVER="$HTTPD_DOCUMENT_ROOT_PATH/server" &&
+       rm -rf "$SERVER" repo &&
+       test_create_repo "$SERVER" &&
+       test_commit -C "$SERVER" foo &&
+       test_config -C "$SERVER" uploadpack.allowfilter 1 &&
+       test_config -C "$SERVER" uploadpack.allowanysha1inwant 1 &&
+
+       # Create a tag pointing to a blob.
+       BLOB=$(echo blob-contents | git -C "$SERVER" hash-object --stdin -w) &&
+       git -C "$SERVER" tag myblob "$BLOB" &&
+
+       # Craft a packfile not including that blob.
+       git -C "$SERVER" rev-parse HEAD |
+               git -C "$SERVER" pack-objects --stdout >incomplete.pack &&
+
+       # Replace the existing packfile with the crafted one. The protocol
+       # requires that the packfile be sent in sideband 1, hence the extra
+       # \x01 byte at the beginning.
+       printf "1,/packfile/!c %04x\\\\x01%s0000" \
+               "$(($(wc -c <incomplete.pack) + 5))" \
+               "$(sed_escape <incomplete.pack)" \
+               >"$HTTPD_ROOT_PATH/one-time-sed" &&
+
+       # Use protocol v2 because the sed command looks for the "packfile"
+       # section header.
+       test_config -C "$SERVER" protocol.version 2 &&
+       test_must_fail git -c protocol.version=2 clone \
+               --filter=blob:none $HTTPD_URL/one_time_sed/server repo 2>err &&
+
+       grep "did not send all necessary objects" err &&
+
+       # Ensure that the one-time-sed script was used.
+       ! test -e "$HTTPD_ROOT_PATH/one-time-sed"
+'
+
+stop_httpd
+
 test_done