clone: support remote shallow repository

Cloning from a shallow repository does not follow the "8 steps for new
.git/shallow" because if it does we need to get through step 6 for all
refs. That means commit walking down to the bottom.

Instead the rule to create .git/shallow is simpler and, more
importantly, cheap: if a shallow commit is found in the pack, it's
probably used (i.e. reachable from some refs), so we add it. Others
are dropped.

One may notice this method seems flawed by the word "probably". A
shallow commit may not be reachable from any refs at all if it's
attached to an object island (a group of objects that are not
reachable by any refs).

If that object island is not complete, a new fetch request may send
more objects to connect it to some ref. At that time, because we
incorrectly installed the shallow commit in this island, the user will
not see anything after that commit (fsck is still ok). This is not
desired.

Given that object islands are rare (C Git never sends such islands for
security reasons) and do not really harm the repository integrity, a
tradeoff is made to surprise the user occasionally but work faster
everyday.

A new option --strict could be added later that follows exactly the 8
steps. "git prune" can also learn to remove dangling objects _and_ the
shallow commits that are attached to them from .git/shallow.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/builtin/clone.c b/builtin/clone.c
index 900f56476ad9089ed2edb8ffc0b24cd7a11cf7af..0b182cefc24f3ee790cb63a944cb98eb3b7a6206 100644 (file)
--- a/builtin/clone.c
+++ b/builtin/clone.c
@@ -889,6 +889,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 
        remote = remote_get(option_origin);
        transport = transport_get(remote, remote->url[0]);
 
        remote = remote_get(option_origin);
        transport = transport_get(remote, remote->url[0]);

+       transport->cloning = 1;

 
        if (!transport->get_refs_list || (!is_local && !transport->fetch))
                die(_("Don't know how to clone %s"), transport->url);
 
        if (!transport->get_refs_list || (!is_local && !transport->fetch))
                die(_("Don't know how to clone %s"), transport->url);

diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c
index c1d918fe1bb6772871308e6fabca91fe2f8393e0..927424b6b8feaded0c13fd8e184e735e3ff9a00e 100644 (file)
--- a/builtin/fetch-pack.c
+++ b/builtin/fetch-pack.c
@@ -153,7 +153,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
        get_remote_heads(fd[0], NULL, 0, &ref, 0, NULL, NULL);
 
        ref = fetch_pack(&args, fd, conn, ref, dest,
        get_remote_heads(fd[0], NULL, 0, &ref, 0, NULL, NULL);
 
        ref = fetch_pack(&args, fd, conn, ref, dest,

-                        sought, nr_sought, pack_lockfile_ptr);
+                        sought, nr_sought, NULL, pack_lockfile_ptr);

        if (pack_lockfile) {
                printf("lock %s\n", pack_lockfile);
                fflush(stdout);
        if (pack_lockfile) {
                printf("lock %s\n", pack_lockfile);
                fflush(stdout);

diff --git a/fetch-pack.c b/fetch-pack.c
index 35d097e1b1b294b1afea8064167243612b07a7f6..6c980cd39f19f62cd9fd11a20ad2faff83463a0d 100644 (file)
--- a/fetch-pack.c
+++ b/fetch-pack.c
@@ -13,6 +13,7 @@
 #include "transport.h"
 #include "version.h"
 #include "prio-queue.h"
 #include "transport.h"
 #include "version.h"
 #include "prio-queue.h"

+#include "sha1-array.h"

 
 static int transfer_unpack_limit = -1;
 static int fetch_unpack_limit = -1;
 
 static int transfer_unpack_limit = -1;
 static int fetch_unpack_limit = -1;


@@ -774,6 +775,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
                                 int fd[2],
                                 const struct ref *orig_ref,
                                 struct ref **sought, int nr_sought,
                                 int fd[2],
                                 const struct ref *orig_ref,
                                 struct ref **sought, int nr_sought,

+                                struct shallow_info *si,

                                 char **pack_lockfile)
 {
        struct ref *ref = copy_ref_list(orig_ref);
                                 char **pack_lockfile)
 {
        struct ref *ref = copy_ref_list(orig_ref);


@@ -852,6 +854,8 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
        if (args->depth > 0)
                setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
                                        NULL);
        if (args->depth > 0)
                setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
                                        NULL);

+       else if (args->cloning && si->shallow && si->shallow->nr)
+               alternate_shallow_file = setup_temporary_shallow(si->shallow);

        else
                alternate_shallow_file = NULL;
        if (get_pack(args, fd, pack_lockfile))
        else
                alternate_shallow_file = NULL;
        if (get_pack(args, fd, pack_lockfile))


@@ -925,8 +929,11 @@ static int remove_duplicates_in_refs(struct ref **ref, int nr)
        return dst;
 }
 
        return dst;
 }
 

-static void update_shallow(struct fetch_pack_args *args)
+static void update_shallow(struct fetch_pack_args *args,
+                          struct shallow_info *si)

 {
 {

+       int i;
+

        if (args->depth > 0 && alternate_shallow_file) {
                if (*alternate_shallow_file == '\0') { /* --unshallow */
                        unlink_or_warn(git_path("shallow"));
        if (args->depth > 0 && alternate_shallow_file) {
                if (*alternate_shallow_file == '\0') { /* --unshallow */
                        unlink_or_warn(git_path("shallow"));


@@ -935,6 +942,42 @@ static void update_shallow(struct fetch_pack_args *args)
                        commit_lock_file(&shallow_lock);
                return;
        }
                        commit_lock_file(&shallow_lock);
                return;
        }

+
+       if (!si->shallow || !si->shallow->nr)
+               return;
+
+       if (alternate_shallow_file) {
+               /*
+                * The temporary shallow file is only useful for
+                * index-pack and unpack-objects because it may
+                * contain more roots than we want. Delete it.
+                */
+               if (*alternate_shallow_file)
+                       unlink(alternate_shallow_file);
+               free((char *)alternate_shallow_file);
+       }
+
+       if (args->cloning) {
+               /*
+                * remote is shallow, but this is a clone, there are
+                * no objects in repo to worry about. Accept any
+                * shallow points that exist in the pack (iow in repo
+                * after get_pack() and reprepare_packed_git())
+                */
+               struct sha1_array extra = SHA1_ARRAY_INIT;
+               unsigned char (*sha1)[20] = si->shallow->sha1;
+               for (i = 0; i < si->shallow->nr; i++)
+                       if (has_sha1_file(sha1[i]))
+                               sha1_array_append(&extra, sha1[i]);
+               if (extra.nr) {
+                       setup_alternate_shallow(&shallow_lock,
+                                               &alternate_shallow_file,
+                                               &extra);
+                       commit_lock_file(&shallow_lock);
+               }
+               sha1_array_clear(&extra);
+               return;
+       }

 }
 
 struct ref *fetch_pack(struct fetch_pack_args *args,
 }
 
 struct ref *fetch_pack(struct fetch_pack_args *args,


@@ -942,9 +985,11 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
                       const struct ref *ref,
                       const char *dest,
                       struct ref **sought, int nr_sought,
                       const struct ref *ref,
                       const char *dest,
                       struct ref **sought, int nr_sought,

+                      struct sha1_array *shallow,

                       char **pack_lockfile)
 {
        struct ref *ref_cpy;
                       char **pack_lockfile)
 {
        struct ref *ref_cpy;

+       struct shallow_info si;

 
        fetch_pack_setup();
        if (nr_sought)
 
        fetch_pack_setup();
        if (nr_sought)


@@ -954,8 +999,11 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
                packet_flush(fd[1]);
                die("no matching remote head");
        }
                packet_flush(fd[1]);
                die("no matching remote head");
        }

-       ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile);
-       update_shallow(args);
+       prepare_shallow_info(&si, shallow);
+       ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
+                               &si, pack_lockfile);

        reprepare_packed_git();
        reprepare_packed_git();

+       update_shallow(args, &si);
+       clear_shallow_info(&si);

        return ref_cpy;
 }
        return ref_cpy;
 }

diff --git a/fetch-pack.h b/fetch-pack.h
index 9b08388edfd112318ba727d1959cbc3391c1d6b1..ce595376b74645e06e7247ab285d8c38709347c7 100644 (file)
--- a/fetch-pack.h
+++ b/fetch-pack.h
@@ -4,6 +4,8 @@
 #include "string-list.h"
 #include "run-command.h"
 
 #include "string-list.h"
 #include "run-command.h"
 

+struct sha1_array;
+

 struct fetch_pack_args {
        const char *uploadpack;
        int unpacklimit;
 struct fetch_pack_args {
        const char *uploadpack;
        int unpacklimit;


@@ -20,6 +22,7 @@ struct fetch_pack_args {
        unsigned stateless_rpc:1;
        unsigned check_self_contained_and_connected:1;
        unsigned self_contained_and_connected:1;
        unsigned stateless_rpc:1;
        unsigned check_self_contained_and_connected:1;
        unsigned self_contained_and_connected:1;

+       unsigned cloning:1;

 };
 
 /*
 };
 
 /*


@@ -33,6 +36,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
                       const char *dest,
                       struct ref **sought,
                       int nr_sought,
                       const char *dest,
                       struct ref **sought,
                       int nr_sought,

+                      struct sha1_array *shallow,

                       char **pack_lockfile);
 
 #endif
                       char **pack_lockfile);
 
 #endif

diff --git a/transport.c b/transport.c
index 90453df9c6ffc08b39a49002e31468d4ff6f5b3d..91c466742e160bd2184931f4c4c8fe4f227d95d7 100644 (file)
--- a/transport.c
+++ b/transport.c
@@ -456,6 +456,7 @@ struct git_transport_data {
        int fd[2];
        unsigned got_remote_heads : 1;
        struct sha1_array extra_have;
        int fd[2];
        unsigned got_remote_heads : 1;
        struct sha1_array extra_have;

+       struct sha1_array shallow;

 };
 
 static int set_git_option(struct git_transport_options *opts,
 };
 
 static int set_git_option(struct git_transport_options *opts,


@@ -512,7 +513,9 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus
 
        connect_setup(transport, for_push, 0);
        get_remote_heads(data->fd[0], NULL, 0, &refs,
 
        connect_setup(transport, for_push, 0);
        get_remote_heads(data->fd[0], NULL, 0, &refs,

-                        for_push ? REF_NORMAL : 0, &data->extra_have, NULL);
+                        for_push ? REF_NORMAL : 0,
+                        &data->extra_have,
+                        transport->cloning ? &data->shallow : NULL);

        data->got_remote_heads = 1;
 
        return refs;
        data->got_remote_heads = 1;
 
        return refs;


@@ -539,17 +542,19 @@ static int fetch_refs_via_pack(struct transport *transport,
        args.depth = data->options.depth;
        args.check_self_contained_and_connected =
                data->options.check_self_contained_and_connected;
        args.depth = data->options.depth;
        args.check_self_contained_and_connected =
                data->options.check_self_contained_and_connected;

+       args.cloning = transport->cloning;

 
        if (!data->got_remote_heads) {
                connect_setup(transport, 0, 0);
                get_remote_heads(data->fd[0], NULL, 0, &refs_tmp, 0,
 
        if (!data->got_remote_heads) {
                connect_setup(transport, 0, 0);
                get_remote_heads(data->fd[0], NULL, 0, &refs_tmp, 0,

-                                NULL, NULL);
+                                NULL,
+                                transport->cloning ? &data->shallow : NULL);

                data->got_remote_heads = 1;
        }
 
        refs = fetch_pack(&args, data->fd, data->conn,
                          refs_tmp ? refs_tmp : transport->remote_refs,
                data->got_remote_heads = 1;
        }
 
        refs = fetch_pack(&args, data->fd, data->conn,
                          refs_tmp ? refs_tmp : transport->remote_refs,

-                         dest, to_fetch, nr_heads,
+                         dest, to_fetch, nr_heads, &data->shallow,

                          &transport->pack_lockfile);
        close(data->fd[0]);
        close(data->fd[1]);
                          &transport->pack_lockfile);
        close(data->fd[0]);
        close(data->fd[1]);

diff --git a/transport.h b/transport.h
index b3679bbdc787dcabca09fc801834c3d7f13fc34f..59842d49947b5e6bb502dcd17405c302c61d0c9b 100644 (file)
--- a/transport.h
+++ b/transport.h
@@ -35,6 +35,12 @@ struct transport {
         */
        unsigned cannot_reuse : 1;
 
         */
        unsigned cannot_reuse : 1;
 

+       /*
+        * A hint from caller that it will be performing a clone, not
+        * normal fetch. IOW the repository is guaranteed empty.
+        */
+       unsigned cloning : 1;
+

        /**
         * Returns 0 if successful, positive if the option is not
         * recognized or is inapplicable, and negative if the option
        /**
         * Returns 0 if successful, positive if the option is not
         * recognized or is inapplicable, and negative if the option