fetch_populated_submodules: use new parallel job processing

In a later patch we enable parallel processing of submodules, this
only adds the possibility for it. So this change should not change
any user facing behavior.

Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/submodule.c b/submodule.c
index 8386477d6219bb30b8618b39ee04bc622227a2d4..4bc7a45393daaf14e23427fa6a36f223ffff7d9f 100644 (file)
--- a/submodule.c
+++ b/submodule.c
@@ -12,6 +12,7 @@
 #include "sha1-array.h"
 #include "argv-array.h"
 #include "blob.h"
 #include "sha1-array.h"
 #include "argv-array.h"
 #include "blob.h"

+#include "thread-utils.h"

 
 static int config_fetch_recurse_submodules = RECURSE_SUBMODULES_ON_DEMAND;
 static struct string_list changed_submodule_paths;
 
 static int config_fetch_recurse_submodules = RECURSE_SUBMODULES_ON_DEMAND;
 static struct string_list changed_submodule_paths;


@@ -610,37 +611,28 @@ static void calculate_changed_submodule_paths(void)
        initialized_fetch_ref_tips = 0;
 }
 
        initialized_fetch_ref_tips = 0;
 }
 

-int fetch_populated_submodules(const struct argv_array *options,
-                              const char *prefix, int command_line_option,
-                              int quiet)
+struct submodule_parallel_fetch {
+       int count;
+       struct argv_array args;
+       const char *work_tree;
+       const char *prefix;
+       int command_line_option;
+       int quiet;
+       int result;
+};
+#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0}
+
+static int get_next_submodule(struct child_process *cp,
+                             struct strbuf *err, void *data, void **task_cb)

 {
 {

-       int i, result = 0;
-       struct child_process cp = CHILD_PROCESS_INIT;
-       struct argv_array argv = ARGV_ARRAY_INIT;
-       const char *work_tree = get_git_work_tree();
-       if (!work_tree)
-               goto out;
-
-       if (read_cache() < 0)
-               die("index file corrupt");
-
-       argv_array_push(&argv, "fetch");
-       for (i = 0; i < options->argc; i++)
-               argv_array_push(&argv, options->argv[i]);
-       argv_array_push(&argv, "--recurse-submodules-default");
-       /* default value, "--submodule-prefix" and its value are added later */
-
-       cp.env = local_repo_env;
-       cp.git_cmd = 1;
-       cp.no_stdin = 1;
-
-       calculate_changed_submodule_paths();
+       int ret = 0;
+       struct submodule_parallel_fetch *spf = data;

 
 

-       for (i = 0; i < active_nr; i++) {
+       for (; spf->count < active_nr; spf->count++) {

                struct strbuf submodule_path = STRBUF_INIT;
                struct strbuf submodule_git_dir = STRBUF_INIT;
                struct strbuf submodule_prefix = STRBUF_INIT;
                struct strbuf submodule_path = STRBUF_INIT;
                struct strbuf submodule_git_dir = STRBUF_INIT;
                struct strbuf submodule_prefix = STRBUF_INIT;

-               const struct cache_entry *ce = active_cache[i];
+               const struct cache_entry *ce = active_cache[spf->count];

                const char *git_dir, *default_argv;
                const struct submodule *submodule;
 
                const char *git_dir, *default_argv;
                const struct submodule *submodule;
 


@@ -652,7 +644,7 @@ int fetch_populated_submodules(const struct argv_array *options,
                        submodule = submodule_from_name(null_sha1, ce->name);
 
                default_argv = "yes";
                        submodule = submodule_from_name(null_sha1, ce->name);
 
                default_argv = "yes";

-               if (command_line_option == RECURSE_SUBMODULES_DEFAULT) {
+               if (spf->command_line_option == RECURSE_SUBMODULES_DEFAULT) {

                        if (submodule &&
                            submodule->fetch_recurse !=
                                                RECURSE_SUBMODULES_NONE) {
                        if (submodule &&
                            submodule->fetch_recurse !=
                                                RECURSE_SUBMODULES_NONE) {


@@ -675,40 +667,102 @@ int fetch_populated_submodules(const struct argv_array *options,
                                        default_argv = "on-demand";
                                }
                        }
                                        default_argv = "on-demand";
                                }
                        }

-               } else if (command_line_option == RECURSE_SUBMODULES_ON_DEMAND) {
+               } else if (spf->command_line_option == RECURSE_SUBMODULES_ON_DEMAND) {

                        if (!unsorted_string_list_lookup(&changed_submodule_paths, ce->name))
                                continue;
                        default_argv = "on-demand";
                }
 
                        if (!unsorted_string_list_lookup(&changed_submodule_paths, ce->name))
                                continue;
                        default_argv = "on-demand";
                }
 

-               strbuf_addf(&submodule_path, "%s/%s", work_tree, ce->name);
+               strbuf_addf(&submodule_path, "%s/%s", spf->work_tree, ce->name);

                strbuf_addf(&submodule_git_dir, "%s/.git", submodule_path.buf);
                strbuf_addf(&submodule_git_dir, "%s/.git", submodule_path.buf);

-               strbuf_addf(&submodule_prefix, "%s%s/", prefix, ce->name);
+               strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name);

                git_dir = read_gitfile(submodule_git_dir.buf);
                if (!git_dir)
                        git_dir = submodule_git_dir.buf;
                if (is_directory(git_dir)) {
                git_dir = read_gitfile(submodule_git_dir.buf);
                if (!git_dir)
                        git_dir = submodule_git_dir.buf;
                if (is_directory(git_dir)) {

-                       if (!quiet)
-                               fprintf(stderr, "Fetching submodule %s%s\n", prefix, ce->name);
-                       cp.dir = submodule_path.buf;
-                       argv_array_push(&argv, default_argv);
-                       argv_array_push(&argv, "--submodule-prefix");
-                       argv_array_push(&argv, submodule_prefix.buf);
-                       cp.argv = argv.argv;
-                       if (run_command(&cp))
-                               result = 1;
-                       argv_array_pop(&argv);
-                       argv_array_pop(&argv);
-                       argv_array_pop(&argv);
+                       child_process_init(cp);
+                       cp->dir = strbuf_detach(&submodule_path, NULL);
+                       cp->env = local_repo_env;
+                       cp->git_cmd = 1;
+                       if (!spf->quiet)
+                               strbuf_addf(err, "Fetching submodule %s%s\n",
+                                           spf->prefix, ce->name);
+                       argv_array_init(&cp->args);
+                       argv_array_pushv(&cp->args, spf->args.argv);
+                       argv_array_push(&cp->args, default_argv);
+                       argv_array_push(&cp->args, "--submodule-prefix");
+                       argv_array_push(&cp->args, submodule_prefix.buf);
+                       ret = 1;

                }
                strbuf_release(&submodule_path);
                strbuf_release(&submodule_git_dir);
                strbuf_release(&submodule_prefix);
                }
                strbuf_release(&submodule_path);
                strbuf_release(&submodule_git_dir);
                strbuf_release(&submodule_prefix);

+               if (ret) {
+                       spf->count++;
+                       return 1;
+               }

        }
        }

-       argv_array_clear(&argv);
+       return 0;
+}
+
+static int fetch_start_failure(struct child_process *cp,
+                              struct strbuf *err,
+                              void *cb, void *task_cb)
+{
+       struct submodule_parallel_fetch *spf = cb;
+
+       spf->result = 1;
+
+       return 0;
+}
+
+static int fetch_finish(int retvalue, struct child_process *cp,
+                       struct strbuf *err, void *cb, void *task_cb)
+{
+       struct submodule_parallel_fetch *spf = cb;
+
+       if (retvalue)
+               spf->result = 1;
+
+       return 0;
+}
+
+int fetch_populated_submodules(const struct argv_array *options,
+                              const char *prefix, int command_line_option,
+                              int quiet)
+{
+       int i;
+       int max_parallel_jobs = 1;
+       struct submodule_parallel_fetch spf = SPF_INIT;
+
+       spf.work_tree = get_git_work_tree();
+       spf.command_line_option = command_line_option;
+       spf.quiet = quiet;
+       spf.prefix = prefix;
+
+       if (!spf.work_tree)
+               goto out;
+
+       if (read_cache() < 0)
+               die("index file corrupt");
+
+       argv_array_push(&spf.args, "fetch");
+       for (i = 0; i < options->argc; i++)
+               argv_array_push(&spf.args, options->argv[i]);
+       argv_array_push(&spf.args, "--recurse-submodules-default");
+       /* default value, "--submodule-prefix" and its value are added later */
+
+       calculate_changed_submodule_paths();
+       run_processes_parallel(max_parallel_jobs,
+                              get_next_submodule,
+                              fetch_start_failure,
+                              fetch_finish,
+                              &spf);
+
+       argv_array_clear(&spf.args);

 out:
        string_list_clear(&changed_submodule_paths, 1);
 out:
        string_list_clear(&changed_submodule_paths, 1);

-       return result;
+       return spf.result;

 }
 
 unsigned is_submodule_modified(const char *path, int ignore_untracked)
 }
 
 unsigned is_submodule_modified(const char *path, int ignore_untracked)