submodules: allow parallel fetching, add tests and documentation

This enables the work of the previous patches.

Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt
index 45583d8454b4cae9bdb2ddb904c077cf1d3a1223..6b109f687aa27e6ad8ee4b7b665d053193975af6 100644 (file)
--- a/Documentation/fetch-options.txt
+++ b/Documentation/fetch-options.txt
@@ -100,6 +100,13 @@ ifndef::git-pull[]
        reference to a commit that isn't already in the local submodule
        clone.
 
        reference to a commit that isn't already in the local submodule
        clone.
 

+-j::
+--jobs=<n>::
+       Number of parallel children to be used for fetching submodules.
+       Each will fetch from different submodules, such that fetching many
+       submodules will be faster. By default submodules will be fetched
+       one at a time.
+

 --no-recurse-submodules::
        Disable recursive fetching of submodules (this has the same effect as
        using the '--recurse-submodules=no' option).
 --no-recurse-submodules::
        Disable recursive fetching of submodules (this has the same effect as
        using the '--recurse-submodules=no' option).

diff --git a/builtin/fetch.c b/builtin/fetch.c
index c85f3471d4b53ce7db7662b5531ddebe5513b7b0..586840d761a9c14142ee35cdd53e2e4f96648209 100644 (file)
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -37,6 +37,7 @@ static int prune = -1; /* unspecified */
 static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity;
 static int progress = -1, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
 static int tags = TAGS_DEFAULT, unshallow, update_shallow;
 static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity;
 static int progress = -1, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
 static int tags = TAGS_DEFAULT, unshallow, update_shallow;

+static int max_children = 1;

 static const char *depth;
 static const char *upload_pack;
 static struct strbuf default_rla = STRBUF_INIT;
 static const char *depth;
 static const char *upload_pack;
 static struct strbuf default_rla = STRBUF_INIT;


@@ -99,6 +100,8 @@ static struct option builtin_fetch_options[] = {
                    N_("fetch all tags and associated objects"), TAGS_SET),
        OPT_SET_INT('n', NULL, &tags,
                    N_("do not fetch all tags (--no-tags)"), TAGS_UNSET),
                    N_("fetch all tags and associated objects"), TAGS_SET),
        OPT_SET_INT('n', NULL, &tags,
                    N_("do not fetch all tags (--no-tags)"), TAGS_UNSET),

+       OPT_INTEGER('j', "jobs", &max_children,
+                   N_("number of submodules fetched in parallel")),

        OPT_BOOL('p', "prune", &prune,
                 N_("prune remote-tracking branches no longer on remote")),
        { OPTION_CALLBACK, 0, "recurse-submodules", NULL, N_("on-demand"),
        OPT_BOOL('p', "prune", &prune,
                 N_("prune remote-tracking branches no longer on remote")),
        { OPTION_CALLBACK, 0, "recurse-submodules", NULL, N_("on-demand"),


@@ -1213,7 +1216,8 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
                result = fetch_populated_submodules(&options,
                                                    submodule_prefix,
                                                    recurse_submodules,
                result = fetch_populated_submodules(&options,
                                                    submodule_prefix,
                                                    recurse_submodules,

-                                                   verbosity < 0);
+                                                   verbosity < 0,
+                                                   max_children);

                argv_array_clear(&options);
        }
 
                argv_array_clear(&options);
        }
 

diff --git a/builtin/pull.c b/builtin/pull.c
index 5145fc60a0377a7fb799555892909f979aca9076..9e3c73809f83af636150cc3edd8628c339ba40c1 100644 (file)
--- a/builtin/pull.c
+++ b/builtin/pull.c
@@ -95,6 +95,7 @@ static int opt_force;
 static char *opt_tags;
 static char *opt_prune;
 static char *opt_recurse_submodules;
 static char *opt_tags;
 static char *opt_prune;
 static char *opt_recurse_submodules;

+static char *max_children;

 static int opt_dry_run;
 static char *opt_keep;
 static char *opt_depth;
 static int opt_dry_run;
 static char *opt_keep;
 static char *opt_depth;


@@ -178,6 +179,9 @@ static struct option pull_options[] = {
                N_("on-demand"),
                N_("control recursive fetching of submodules"),
                PARSE_OPT_OPTARG),
                N_("on-demand"),
                N_("control recursive fetching of submodules"),
                PARSE_OPT_OPTARG),

+       OPT_PASSTHRU('j', "jobs", &max_children, N_("n"),
+               N_("number of submodules pulled in parallel"),
+               PARSE_OPT_OPTARG),

        OPT_BOOL(0, "dry-run", &opt_dry_run,
                N_("dry run")),
        OPT_PASSTHRU('k', "keep", &opt_keep, NULL,
        OPT_BOOL(0, "dry-run", &opt_dry_run,
                N_("dry run")),
        OPT_PASSTHRU('k', "keep", &opt_keep, NULL,


@@ -525,6 +529,8 @@ static int run_fetch(const char *repo, const char **refspecs)
                argv_array_push(&args, opt_prune);
        if (opt_recurse_submodules)
                argv_array_push(&args, opt_recurse_submodules);
                argv_array_push(&args, opt_prune);
        if (opt_recurse_submodules)
                argv_array_push(&args, opt_recurse_submodules);

+       if (max_children)
+               argv_array_push(&args, max_children);

        if (opt_dry_run)
                argv_array_push(&args, "--dry-run");
        if (opt_keep)
        if (opt_dry_run)
                argv_array_push(&args, "--dry-run");
        if (opt_keep)

diff --git a/submodule.c b/submodule.c
index 4bc7a45393daaf14e23427fa6a36f223ffff7d9f..b83939c294782ac59e5c78c8882064a1e272e309 100644 (file)
--- a/submodule.c
+++ b/submodule.c
@@ -729,10 +729,9 @@ static int fetch_finish(int retvalue, struct child_process *cp,
 
 int fetch_populated_submodules(const struct argv_array *options,
                               const char *prefix, int command_line_option,
 
 int fetch_populated_submodules(const struct argv_array *options,
                               const char *prefix, int command_line_option,

-                              int quiet)
+                              int quiet, int max_parallel_jobs)

 {
        int i;
 {
        int i;

-       int max_parallel_jobs = 1;

        struct submodule_parallel_fetch spf = SPF_INIT;
 
        spf.work_tree = get_git_work_tree();
        struct submodule_parallel_fetch spf = SPF_INIT;
 
        spf.work_tree = get_git_work_tree();

diff --git a/submodule.h b/submodule.h
index 5507c3d9a098b1da016855c1fa4233ffc6e0a8cf..cbc000304eacbe318e1950572a19b164a897ce41 100644 (file)
--- a/submodule.h
+++ b/submodule.h
@@ -31,7 +31,7 @@ void set_config_fetch_recurse_submodules(int value);
 void check_for_new_submodule_commits(unsigned char new_sha1[20]);
 int fetch_populated_submodules(const struct argv_array *options,
                               const char *prefix, int command_line_option,
 void check_for_new_submodule_commits(unsigned char new_sha1[20]);
 int fetch_populated_submodules(const struct argv_array *options,
                               const char *prefix, int command_line_option,

-                              int quiet);
+                              int quiet, int max_parallel_jobs);

 unsigned is_submodule_modified(const char *path, int ignore_untracked);
 int submodule_uses_gitfile(const char *path);
 int ok_to_remove_submodule(const char *path);
 unsigned is_submodule_modified(const char *path, int ignore_untracked);
 int submodule_uses_gitfile(const char *path);
 int ok_to_remove_submodule(const char *path);

diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh
index 17759b145fc7f33290f3862c649ef09bcf3b7f79..1241146227aead97022ade343152347bc911444c 100755 (executable)
--- a/t/t5526-fetch-submodules.sh
+++ b/t/t5526-fetch-submodules.sh
@@ -71,6 +71,17 @@ test_expect_success "fetch --recurse-submodules recurses into submodules" '
        test_i18ncmp expect.err actual.err
 '
 
        test_i18ncmp expect.err actual.err
 '
 

+test_expect_success "fetch --recurse-submodules -j2 has the same output behaviour" '
+       add_upstream_commit &&
+       (
+               cd downstream &&
+               GIT_TRACE=$(pwd)/../trace.out git fetch --recurse-submodules -j2 2>../actual.err
+       ) &&
+       test_must_be_empty actual.out &&
+       test_i18ncmp expect.err actual.err &&
+       grep "2 tasks" trace.out
+'
+

 test_expect_success "fetch alone only fetches superproject" '
        add_upstream_commit &&
        (
 test_expect_success "fetch alone only fetches superproject" '
        add_upstream_commit &&
        (


@@ -140,6 +151,15 @@ test_expect_success "--quiet propagates to submodules" '
        ! test -s actual.err
 '
 
        ! test -s actual.err
 '
 

+test_expect_success "--quiet propagates to parallel submodules" '
+       (
+               cd downstream &&
+               git fetch --recurse-submodules -j 2 --quiet  >../actual.out 2>../actual.err
+       ) &&
+       ! test -s actual.out &&
+       ! test -s actual.err
+'
+

 test_expect_success "--dry-run propagates to submodules" '
        add_upstream_commit &&
        (
 test_expect_success "--dry-run propagates to submodules" '
        add_upstream_commit &&
        (