clone: do faster object check for partial clones

For partial clones, doing a full connectivity check is wasteful; we skip
promisor objects (which, for a partial clone, is all known objects), and
enumerating them all to exclude them from the connectivity check can
take a significant amount of time on large repos.

At most, we want to make sure that we get the objects referred to by any
wanted refs. For partial clones, just check that these objects were
transferred.

Signed-off-by: Josh Steadmon <steadmon@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/builtin/clone.c b/builtin/clone.c
index 50bde9961809b1d55c74fb6d3cbfbf5a86ddf5c7..fdbbd8942a4632d0698a962f74843e10ee1f4349 100644 (file)
--- a/builtin/clone.c
+++ b/builtin/clone.c
@@ -657,7 +657,8 @@ static void update_remote_refs(const struct ref *refs,
                               const char *branch_top,
                               const char *msg,
                               struct transport *transport,
                               const char *branch_top,
                               const char *msg,
                               struct transport *transport,

-                              int check_connectivity)
+                              int check_connectivity,
+                              int check_refs_only)

 {
        const struct ref *rm = mapped_refs;
 
 {
        const struct ref *rm = mapped_refs;
 


@@ -666,6 +667,7 @@ static void update_remote_refs(const struct ref *refs,
 
                opt.transport = transport;
                opt.progress = transport->progress;
 
                opt.transport = transport;
                opt.progress = transport->progress;

+               opt.check_refs_only = !!check_refs_only;

 
                if (check_connected(iterate_ref_map, &rm, &opt))
                        die(_("remote did not send all necessary objects"));
 
                if (check_connected(iterate_ref_map, &rm, &opt))
                        die(_("remote did not send all necessary objects"));


@@ -1224,7 +1226,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 
        update_remote_refs(refs, mapped_refs, remote_head_points_at,
                           branch_top.buf, reflog_msg.buf, transport,
 
        update_remote_refs(refs, mapped_refs, remote_head_points_at,
                           branch_top.buf, reflog_msg.buf, transport,

-                          !is_local);
+                          !is_local, filter_options.choice);

 
        update_head(our_head_points_at, remote_head, reflog_msg.buf);
 
 
        update_head(our_head_points_at, remote_head, reflog_msg.buf);
 

diff --git a/connected.c b/connected.c
index 1bba888eff90a23e586807fd02b274b8e7fac097..1ab481fed69b33b48bd95c282d2c0faff897123c 100644 (file)
--- a/connected.c
+++ b/connected.c
@@ -1,4 +1,5 @@
 #include "cache.h"
 #include "cache.h"

+#include "object-store.h"

 #include "run-command.h"
 #include "sigchain.h"
 #include "connected.h"
 #include "run-command.h"
 #include "sigchain.h"
 #include "connected.h"


@@ -49,6 +50,22 @@ int check_connected(oid_iterate_fn fn, void *cb_data,
                strbuf_release(&idx_file);
        }
 
                strbuf_release(&idx_file);
        }
 

+       if (opt->check_refs_only) {
+               /*
+                * For partial clones, we don't want to have to do a regular
+                * connectivity check because we have to enumerate and exclude
+                * all promisor objects (slow), and then the connectivity check
+                * itself becomes a no-op because in a partial clone every
+                * object is a promisor object. Instead, just make sure we
+                * received the objects pointed to by each wanted ref.
+                */
+               do {
+                       if (!repo_has_object_file(the_repository, &oid))
+                               return 1;
+               } while (!fn(cb_data, &oid));
+               return 0;
+       }
+

        if (opt->shallow_file) {
                argv_array_push(&rev_list.args, "--shallow-file");
                argv_array_push(&rev_list.args, opt->shallow_file);
        if (opt->shallow_file) {
                argv_array_push(&rev_list.args, "--shallow-file");
                argv_array_push(&rev_list.args, opt->shallow_file);

diff --git a/connected.h b/connected.h
index 8d5a6b3ad6fe4bb0f9ca0930f8eea82543968306..ce2e7d8f2e535aac72fb317448ad20fb2133429b 100644 (file)
--- a/connected.h
+++ b/connected.h
@@ -46,6 +46,14 @@ struct check_connected_options {
         * during a fetch.
         */
        unsigned is_deepening_fetch : 1;
         * during a fetch.
         */
        unsigned is_deepening_fetch : 1;

+
+       /*
+        * If non-zero, only check the top-level objects referenced by the
+        * wanted refs (passed in as cb_data). This is useful for partial
+        * clones, where enumerating and excluding all promisor objects is very
+        * slow and the commit-walk itself becomes a no-op.
+        */
+       unsigned check_refs_only : 1;

 };
 
 #define CHECK_CONNECTED_INIT { 0 }
 };
 
 #define CHECK_CONNECTED_INIT { 0 }