submodule update: expose parallelism to the user

Expose possible parallelism either via the "--jobs" CLI parameter or
the "submodule.fetchJobs" setting.

By having the variable initialized to -1, we make sure 0 can be passed
into the parallel processing machine, which will then pick as many parallel
workers as there are CPUs.

Reviewed-by: Jonathan Nieder <jrnieder@gmail.com>
Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/git-submodule.txt b/Documentation/git-submodule.txt
index 1572f058f59ad1f75c4b5ed6e34fae9110414bad..13adebf7b75f2ab122c4d23708493ef098d3548d 100644 (file)
--- a/Documentation/git-submodule.txt
+++ b/Documentation/git-submodule.txt
@@ -16,7 +16,7 @@ SYNOPSIS
 'git submodule' [--quiet] deinit [-f|--force] [--] <path>...
 'git submodule' [--quiet] update [--init] [--remote] [-N|--no-fetch]
              [-f|--force] [--rebase|--merge] [--reference <repository>]
 'git submodule' [--quiet] deinit [-f|--force] [--] <path>...
 'git submodule' [--quiet] update [--init] [--remote] [-N|--no-fetch]
              [-f|--force] [--rebase|--merge] [--reference <repository>]

-             [--depth <depth>] [--recursive] [--] [<path>...]
+             [--depth <depth>] [--recursive] [--jobs <n>] [--] [<path>...]

 'git submodule' [--quiet] summary [--cached|--files] [(-n|--summary-limit) <n>]
              [commit] [--] [<path>...]
 'git submodule' [--quiet] foreach [--recursive] <command>
 'git submodule' [--quiet] summary [--cached|--files] [(-n|--summary-limit) <n>]
              [commit] [--] [<path>...]
 'git submodule' [--quiet] foreach [--recursive] <command>


@@ -377,6 +377,11 @@ for linkgit:git-clone[1]'s `--reference` and `--shared` options carefully.
        clone with a history truncated to the specified number of revisions.
        See linkgit:git-clone[1]
 
        clone with a history truncated to the specified number of revisions.
        See linkgit:git-clone[1]
 

+-j <n>::
+--jobs <n>::
+       This option is only valid for the update command.
+       Clone new submodules in parallel with as many jobs.
+       Defaults to the `submodule.fetchJobs` option.

 
 <path>...::
        Paths to submodule(s). When specified this will restrict the command
 
 <path>...::
        Paths to submodule(s). When specified this will restrict the command

diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
index 9e8109e542225e0b5b0a24362fa626db3eb53dfe..a484945d37911828554769a38affddaf3067dc54 100644 (file)
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@@ -430,6 +430,7 @@ static int update_clone_task_finished(int result,
 static int update_clone(int argc, const char **argv, const char *prefix)
 {
        const char *update = NULL;
 static int update_clone(int argc, const char **argv, const char *prefix)
 {
        const char *update = NULL;

+       int max_jobs = -1;

        struct string_list_item *item;
        struct pathspec pathspec;
        struct submodule_update_clone suc = SUBMODULE_UPDATE_CLONE_INIT;
        struct string_list_item *item;
        struct pathspec pathspec;
        struct submodule_update_clone suc = SUBMODULE_UPDATE_CLONE_INIT;


@@ -450,6 +451,8 @@ static int update_clone(int argc, const char **argv, const char *prefix)
                OPT_STRING(0, "depth", &suc.depth, "<depth>",
                           N_("Create a shallow clone truncated to the "
                              "specified number of revisions")),
                OPT_STRING(0, "depth", &suc.depth, "<depth>",
                           N_("Create a shallow clone truncated to the "
                              "specified number of revisions")),

+               OPT_INTEGER('j', "jobs", &max_jobs,
+                           N_("parallel jobs")),

                OPT__QUIET(&suc.quiet, N_("don't print cloning progress")),
                OPT_END()
        };
                OPT__QUIET(&suc.quiet, N_("don't print cloning progress")),
                OPT_END()
        };


@@ -477,7 +480,10 @@ static int update_clone(int argc, const char **argv, const char *prefix)
        gitmodules_config();
        git_config(submodule_config, NULL);
 
        gitmodules_config();
        git_config(submodule_config, NULL);
 

-       run_processes_parallel(1,
+       if (max_jobs < 0)
+               max_jobs = parallel_submodules();
+
+       run_processes_parallel(max_jobs,

                               update_clone_get_next_task,
                               update_clone_start_failure,
                               update_clone_task_finished,
                               update_clone_get_next_task,
                               update_clone_start_failure,
                               update_clone_task_finished,

diff --git a/git-submodule.sh b/git-submodule.sh
index a6a82d25aa3617504e66ba49c6379df555f34437..86018ee9c5d66048fc360c76843029311fe1302d 100755 (executable)
--- a/git-submodule.sh
+++ b/git-submodule.sh
@@ -645,6 +645,14 @@ cmd_update()
                --depth=*)
                        depth=$1
                        ;;
                --depth=*)
                        depth=$1
                        ;;

+               -j|--jobs)
+                       case "$2" in '') usage ;; esac
+                       jobs="--jobs=$2"
+                       shift
+                       ;;
+               --jobs=*)
+                       jobs=$1
+                       ;;

                --)
                        shift
                        break
                --)
                        shift
                        break


@@ -671,6 +679,7 @@ cmd_update()
                ${update:+--update "$update"} \
                ${reference:+--reference "$reference"} \
                ${depth:+--depth "$depth"} \
                ${update:+--update "$update"} \
                ${reference:+--reference "$reference"} \
                ${depth:+--depth "$depth"} \

+               ${jobs:+$jobs} \

                "$@" || echo "#unmatched"
        } | {
        err=
                "$@" || echo "#unmatched"
        } | {
        err=

diff --git a/t/t7406-submodule-update.sh b/t/t7406-submodule-update.sh
index dda3929d99aea3ea30d30b5715532fe6184d5259..7fd5142d6c2983aba7a495fdaf0dcbe83ca2cabb 100755 (executable)
--- a/t/t7406-submodule-update.sh
+++ b/t/t7406-submodule-update.sh
@@ -774,4 +774,16 @@ test_expect_success 'submodule update --recursive drops module name before recur
         test_i18ngrep "Submodule path .deeper/submodule/subsubmodule.: checked out" actual
        )
 '
         test_i18ngrep "Submodule path .deeper/submodule/subsubmodule.: checked out" actual
        )
 '

+
+test_expect_success 'submodule update can be run in parallel' '
+       (cd super2 &&
+        GIT_TRACE=$(pwd)/trace.out git submodule update --jobs 7 &&
+        grep "7 tasks" trace.out &&
+        git config submodule.fetchJobs 8 &&
+        GIT_TRACE=$(pwd)/trace.out git submodule update &&
+        grep "8 tasks" trace.out &&
+        GIT_TRACE=$(pwd)/trace.out git submodule update --jobs 9 &&
+        grep "9 tasks" trace.out
+       )
+'

 test_done
 test_done