Merge branch 'sb/submodule-recursive-fetch-gets-the-tip'
authorJunio C Hamano <gitster@pobox.com>
Tue, 29 Jan 2019 20:47:54 +0000 (12:47 -0800)
committerJunio C Hamano <gitster@pobox.com>
Tue, 29 Jan 2019 20:47:54 +0000 (12:47 -0800)
"git fetch --recurse-submodules" may not fetch the necessary commit
that is bound to the superproject, which is getting corrected.

* sb/submodule-recursive-fetch-gets-the-tip:
fetch: ensure submodule objects fetched
submodule.c: fetch in submodules git directory instead of in worktree
submodule: migrate get_next_submodule to use repository structs
repository: repo_submodule_init to take a submodule struct
submodule: store OIDs in changed_submodule_names
submodule.c: tighten scope of changed_submodule_names struct
submodule.c: sort changed_submodule_names before searching it
submodule.c: fix indentation
sha1-array: provide oid_array_filter

1  2 
builtin/fetch.c
builtin/grep.c
builtin/ls-files.c
builtin/submodule--helper.c
repository.c
submodule.c
t/t5526-fetch-submodules.sh
diff --combined builtin/fetch.c
index c0ade48f5d68bef87d85c478ec63b3212a2f4940,91f9b7d9c83fb05c4c640ac8cb6070c3fb1c5110..c316c03ba206b8b04b598cdab8dd259a59014ea5
@@@ -763,9 -763,6 +763,6 @@@ static int update_local_ref(struct ref 
                        what = _("[new ref]");
                }
  
-               if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-                   (recurse_submodules != RECURSE_SUBMODULES_ON))
-                       check_for_new_submodule_commits(&ref->new_oid);
                r = s_update_ref(msg, ref, 0);
                format_display(display, r ? '!' : '*', what,
                               r ? _("unable to update local ref") : NULL,
                strbuf_add_unique_abbrev(&quickref, &current->object.oid, DEFAULT_ABBREV);
                strbuf_addstr(&quickref, "..");
                strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV);
-               if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-                   (recurse_submodules != RECURSE_SUBMODULES_ON))
-                       check_for_new_submodule_commits(&ref->new_oid);
                r = s_update_ref("fast-forward", ref, 1);
                format_display(display, r ? '!' : ' ', quickref.buf,
                               r ? _("unable to update local ref") : NULL,
                strbuf_add_unique_abbrev(&quickref, &current->object.oid, DEFAULT_ABBREV);
                strbuf_addstr(&quickref, "...");
                strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV);
-               if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
-                   (recurse_submodules != RECURSE_SUBMODULES_ON))
-                       check_for_new_submodule_commits(&ref->new_oid);
                r = s_update_ref("forced-update", ref, 1);
                format_display(display, r ? '!' : '+', quickref.buf,
                               r ? _("unable to update local ref") : _("forced update"),
@@@ -892,6 -883,8 +883,8 @@@ static int store_updated_refs(const cha
                                ref->force = rm->peer_ref->force;
                        }
  
+                       if (recurse_submodules != RECURSE_SUBMODULES_OFF)
+                               check_for_new_submodule_commits(&rm->old_oid);
  
                        if (!strcmp(rm->name, "HEAD")) {
                                kind = "";
@@@ -1478,8 -1471,7 +1471,8 @@@ static inline void fetch_one_setup_part
         */
        if (strcmp(remote->name, repository_format_partial_clone)) {
                if (filter_options.choice)
 -                      die(_("--filter can only be used with the remote configured in core.partialClone"));
 +                      die(_("--filter can only be used with the remote "
 +                            "configured in extensions.partialclone"));
                return;
        }
  
@@@ -1647,8 -1639,7 +1640,8 @@@ int cmd_fetch(int argc, const char **ar
                result = fetch_one(remote, argc, argv, prune_tags_ok);
        } else {
                if (filter_options.choice)
 -                      die(_("--filter can only be used with the remote configured in core.partialClone"));
 +                      die(_("--filter can only be used with the remote "
 +                            "configured in extensions.partialclone"));
                /* TODO should this also die if we have a previous partial-clone? */
                result = fetch_multiple(&list);
        }
diff --combined builtin/grep.c
index 4748195ae173786924f75dab9e865ce05b434930,d6bd887b2df28c873cc4b10bab839e5fbe287b83..dc9183876499e30e00f81b10c74c54270160448d
@@@ -404,7 -404,10 +404,10 @@@ static int grep_submodule(struct grep_o
                          const struct object_id *oid,
                          const char *filename, const char *path)
  {
-       struct repository submodule;
+       struct repository subrepo;
+       const struct submodule *sub = submodule_from_path(superproject,
+                                                         &null_oid, path);
        int hit;
  
        /*
                return 0;
        }
  
-       if (repo_submodule_init(&submodule, superproject, path)) {
+       if (repo_submodule_init(&subrepo, superproject, sub)) {
                grep_read_unlock();
                return 0;
        }
  
-       repo_read_gitmodules(&submodule);
+       repo_read_gitmodules(&subrepo);
  
        /*
         * NEEDSWORK: This adds the submodule's object directory to the list of
         * store is no longer global and instead is a member of the repository
         * object.
         */
-       add_to_alternates_memory(submodule.objects->odb->path);
 -      add_to_alternates_memory(subrepo.objects->objectdir);
++      add_to_alternates_memory(subrepo.objects->odb->path);
        grep_read_unlock();
  
        if (oid) {
  
                init_tree_desc(&tree, data, size);
                hit = grep_tree(opt, pathspec, &tree, &base, base.len,
-                               object->type == OBJ_COMMIT, &submodule);
+                               object->type == OBJ_COMMIT, &subrepo);
                strbuf_release(&base);
                free(data);
        } else {
-               hit = grep_cache(opt, &submodule, pathspec, 1);
+               hit = grep_cache(opt, &subrepo, pathspec, 1);
        }
  
-       repo_clear(&submodule);
+       repo_clear(&subrepo);
        return hit;
  }
  
@@@ -553,8 -556,7 +556,8 @@@ static int grep_tree(struct grep_opt *o
  
                if (match != all_entries_interesting) {
                        strbuf_addstr(&name, base->buf + tn_len);
 -                      match = tree_entry_interesting(&entry, &name,
 +                      match = tree_entry_interesting(repo->index,
 +                                                     &entry, &name,
                                                       0, pathspec);
                        strbuf_setlen(&name, name_base_len);
  
diff --combined builtin/ls-files.c
index 7e0dcaa3599da68c62d05655e488467b7756409b,583a0e1ca26cc93c475988bbb579201557b80b79..cde87cbeeb8e8d7389c95c33ff60e6e5c39e10cd
@@@ -206,17 -206,19 +206,19 @@@ static void show_files(struct repositor
  static void show_submodule(struct repository *superproject,
                           struct dir_struct *dir, const char *path)
  {
-       struct repository submodule;
+       struct repository subrepo;
+       const struct submodule *sub = submodule_from_path(superproject,
+                                                         &null_oid, path);
  
-       if (repo_submodule_init(&submodule, superproject, path))
+       if (repo_submodule_init(&subrepo, superproject, sub))
                return;
  
-       if (repo_read_index(&submodule) < 0)
+       if (repo_read_index(&subrepo) < 0)
                die("index file corrupt");
  
-       show_files(&submodule, dir);
+       show_files(&subrepo, dir);
  
-       repo_clear(&submodule);
+       repo_clear(&subrepo);
  }
  
  static void show_ce(struct repository *repo, struct dir_struct *dir,
@@@ -441,7 -443,7 +443,7 @@@ void overlay_tree_on_index(struct index
                               PATHSPEC_PREFER_CWD, prefix, matchbuf);
        } else
                memset(&pathspec, 0, sizeof(pathspec));
 -      if (read_tree(tree, 1, &pathspec, istate))
 +      if (read_tree(the_repository, tree, 1, &pathspec, istate))
                die("unable to read tree entries %s", tree_name);
  
        for (i = 0; i < istate->cache_nr; i++) {
index 6881b6a9cb0fa1cd44118663cd8d7c8eaef1c95a,4eceb8f040b616760205c2f928a04606b80fe190..0e140f176ce3b559478a5bc4245bf36905c543cf
@@@ -1131,8 -1131,6 +1131,8 @@@ static void deinit_submodule(const cha
                if (!(flags & OPT_QUIET))
                        printf(format, displaypath);
  
 +              submodule_unset_core_worktree(sub);
 +
                strbuf_release(&sb_rm);
        }
  
@@@ -1267,20 -1265,19 +1267,20 @@@ struct submodule_alternate_setup 
        SUBMODULE_ALTERNATE_ERROR_IGNORE, NULL }
  
  static int add_possible_reference_from_superproject(
 -              struct alternate_object_database *alt, void *sas_cb)
 +              struct object_directory *odb, void *sas_cb)
  {
        struct submodule_alternate_setup *sas = sas_cb;
 +      size_t len;
  
        /*
         * If the alternate object store is another repository, try the
         * standard layout with .git/(modules/<name>)+/objects
         */
 -      if (ends_with(alt->path, "/objects")) {
 +      if (strip_suffix(odb->path, "/objects", &len)) {
                char *sm_alternate;
                struct strbuf sb = STRBUF_INIT;
                struct strbuf err = STRBUF_INIT;
 -              strbuf_add(&sb, alt->path, strlen(alt->path) - strlen("objects"));
 +              strbuf_add(&sb, odb->path, len);
  
                /*
                 * We need to end the new path with '/' to mark it as a dir,
                 * as the last part of a missing submodule reference would
                 * be taken as a file name.
                 */
 -              strbuf_addf(&sb, "modules/%s/", sas->submodule_name);
 +              strbuf_addf(&sb, "/modules/%s/", sas->submodule_name);
  
                sm_alternate = compute_alternate_path(sb.buf, &err);
                if (sm_alternate) {
@@@ -1554,7 -1551,7 +1554,7 @@@ struct submodule_update_clone 
  #define SUBMODULE_UPDATE_CLONE_INIT {0, MODULE_LIST_INIT, 0, \
        SUBMODULE_UPDATE_STRATEGY_INIT, 0, 0, -1, STRING_LIST_INIT_DUP, 0, \
        NULL, NULL, NULL, \
 -      NULL, 0, 0, 0, NULL, 0, 0, 0}
 +      NULL, 0, 0, 0, NULL, 0, 0, 1}
  
  
  static void next_submodule_warn_missing(struct submodule_update_clone *suc,
@@@ -2048,7 -2045,7 +2048,7 @@@ static int ensure_core_worktree(int arg
        struct repository subrepo;
  
        if (argc != 2)
 -              BUG("submodule--helper connect-gitdir-workingtree <name> <path>");
 +              BUG("submodule--helper ensure-core-worktree <path>");
  
        path = argv[1];
  
        if (!sub)
                BUG("We could get the submodule handle before?");
  
-       if (repo_submodule_init(&subrepo, the_repository, path))
+       if (repo_submodule_init(&subrepo, the_repository, sub))
                die(_("could not get a repository handle for submodule '%s'"), path);
  
        if (!repo_config_get_string(&subrepo, "core.worktree", &cw)) {
diff --combined repository.c
index 7b02e1dffac077d0c21ecaef915f4ea33abc57bc,aabe64ee5d9da82db68e4d4dc2072d58bd9ed4ab..20c509a9226645b1890c8433b22a7182699fc847
@@@ -63,14 -63,8 +63,14 @@@ void repo_set_gitdir(struct repository 
        free(old_gitdir);
  
        repo_set_commondir(repo, o->commondir);
 -      expand_base_dir(&repo->objects->objectdir, o->object_dir,
 +
 +      if (!repo->objects->odb) {
 +              repo->objects->odb = xcalloc(1, sizeof(*repo->objects->odb));
 +              repo->objects->odb_tail = &repo->objects->odb->next;
 +      }
 +      expand_base_dir(&repo->objects->odb->path, o->object_dir,
                        repo->commondir, "objects");
 +
        free(repo->objects->alternate_db);
        repo->objects->alternate_db = xstrdup_or_null(o->alternate_db);
        expand_base_dir(&repo->graft_file, o->graft_file,
@@@ -172,30 -166,23 +172,23 @@@ error
        return -1;
  }
  
- /*
-  * Initialize 'submodule' as the submodule given by 'path' in parent repository
-  * 'superproject'.
-  * Return 0 upon success and a non-zero value upon failure.
-  */
- int repo_submodule_init(struct repository *submodule,
+ int repo_submodule_init(struct repository *subrepo,
                        struct repository *superproject,
-                       const char *path)
+                       const struct submodule *sub)
  {
-       const struct submodule *sub;
        struct strbuf gitdir = STRBUF_INIT;
        struct strbuf worktree = STRBUF_INIT;
        int ret = 0;
  
-       sub = submodule_from_path(superproject, &null_oid, path);
        if (!sub) {
                ret = -1;
                goto out;
        }
  
-       strbuf_repo_worktree_path(&gitdir, superproject, "%s/.git", path);
-       strbuf_repo_worktree_path(&worktree, superproject, "%s", path);
+       strbuf_repo_worktree_path(&gitdir, superproject, "%s/.git", sub->path);
+       strbuf_repo_worktree_path(&worktree, superproject, "%s", sub->path);
  
-       if (repo_init(submodule, gitdir.buf, worktree.buf)) {
+       if (repo_init(subrepo, gitdir.buf, worktree.buf)) {
                /*
                 * If initilization fails then it may be due to the submodule
                 * not being populated in the superproject's worktree.  Instead
                strbuf_repo_git_path(&gitdir, superproject,
                                     "modules/%s", sub->name);
  
-               if (repo_init(submodule, gitdir.buf, NULL)) {
+               if (repo_init(subrepo, gitdir.buf, NULL)) {
                        ret = -1;
                        goto out;
                }
        }
  
-       submodule->submodule_prefix = xstrfmt("%s%s/",
-                                             superproject->submodule_prefix ?
-                                             superproject->submodule_prefix :
-                                             "", path);
+       subrepo->submodule_prefix = xstrfmt("%s%s/",
+                                           superproject->submodule_prefix ?
+                                           superproject->submodule_prefix :
+                                           "", sub->path);
  
  out:
        strbuf_release(&gitdir);
diff --combined submodule.c
index d393e947e699bb1d5764f42e94919218496a138d,b88343d977d78364b417e2015eaa352dec1501b9..7b5cea8522edb9a2b12b46a7ba2055345b64da88
@@@ -25,7 -25,6 +25,6 @@@
  #include "commit-reach.h"
  
  static int config_update_recurse_submodules = RECURSE_SUBMODULES_OFF;
- static struct string_list changed_submodule_names = STRING_LIST_INIT_DUP;
  static int initialized_fetch_ref_tips;
  static struct oid_array ref_tips_before_fetch;
  static struct oid_array ref_tips_after_fetch;
@@@ -495,6 -494,12 +494,12 @@@ void prepare_submodule_repo_env(struct 
                         DEFAULT_GIT_DIR_ENVIRONMENT);
  }
  
+ static void prepare_submodule_repo_env_in_gitdir(struct argv_array *out)
+ {
+       prepare_submodule_repo_env_no_git_dir(out);
+       argv_array_pushf(out, "%s=.", GIT_DIR_ENVIRONMENT);
+ }
  /* Helper function to display the submodule header line prior to the full
   * summary output. If it can locate the submodule objects directory it will
   * attempt to lookup both the left and right commits and put them into the
@@@ -1136,11 -1141,11 +1141,11 @@@ void check_for_new_submodule_commits(st
        oid_array_append(&ref_tips_after_fetch, oid);
  }
  
- static void calculate_changed_submodule_paths(struct repository *r)
+ static void calculate_changed_submodule_paths(struct repository *r,
+               struct string_list *changed_submodule_names)
  {
        struct argv_array argv = ARGV_ARRAY_INIT;
-       struct string_list changed_submodules = STRING_LIST_INIT_DUP;
-       const struct string_list_item *name;
+       struct string_list_item *name;
  
        /* No need to check if there are no submodules configured */
        if (!submodule_from_path(r, NULL, NULL))
         * Collect all submodules (whether checked out or not) for which new
         * commits have been recorded upstream in "changed_submodule_names".
         */
-       collect_changed_submodules(r, &changed_submodules, &argv);
+       collect_changed_submodules(r, changed_submodule_names, &argv);
  
-       for_each_string_list_item(name, &changed_submodules) {
+       for_each_string_list_item(name, changed_submodule_names) {
                struct oid_array *commits = name->util;
                const struct submodule *submodule;
                const char *path = NULL;
                if (!path)
                        continue;
  
-               if (!submodule_has_commits(r, path, commits))
-                       string_list_append(&changed_submodule_names, name->string);
+               if (submodule_has_commits(r, path, commits)) {
+                       oid_array_clear(commits);
+                       *name->string = '\0';
+               }
        }
  
-       free_submodules_oids(&changed_submodules);
+       string_list_remove_empty_items(changed_submodule_names, 1);
        argv_array_clear(&argv);
        oid_array_clear(&ref_tips_before_fetch);
        oid_array_clear(&ref_tips_after_fetch);
@@@ -1221,8 -1229,16 +1229,16 @@@ struct submodule_parallel_fetch 
        int default_option;
        int quiet;
        int result;
+       struct string_list changed_submodule_names;
+       /* Pending fetches by OIDs */
+       struct fetch_task **oid_fetch_tasks;
+       int oid_fetch_tasks_nr, oid_fetch_tasks_alloc;
  };
- #define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0}
+ #define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, \
+                 STRING_LIST_INIT_DUP, \
+                 NULL, 0, 0}
  
  static int get_fetch_recurse_config(const struct submodule *submodule,
                                    struct submodule_parallel_fetch *spf)
        return spf->default_option;
  }
  
+ /*
+  * Fetch in progress (if callback data) or
+  * pending (if in oid_fetch_tasks in struct submodule_parallel_fetch)
+  */
+ struct fetch_task {
+       struct repository *repo;
+       const struct submodule *sub;
+       unsigned free_sub : 1; /* Do we need to free the submodule? */
+       struct oid_array *commits; /* Ensure these commits are fetched */
+ };
+ /**
+  * When a submodule is not defined in .gitmodules, we cannot access it
+  * via the regular submodule-config. Create a fake submodule, which we can
+  * work on.
+  */
+ static const struct submodule *get_non_gitmodules_submodule(const char *path)
+ {
+       struct submodule *ret = NULL;
+       const char *name = default_name_or_path(path);
+       if (!name)
+               return NULL;
+       ret = xmalloc(sizeof(*ret));
+       memset(ret, 0, sizeof(*ret));
+       ret->path = name;
+       ret->name = name;
+       return (const struct submodule *) ret;
+ }
+ static struct fetch_task *fetch_task_create(struct repository *r,
+                                           const char *path)
+ {
+       struct fetch_task *task = xmalloc(sizeof(*task));
+       memset(task, 0, sizeof(*task));
+       task->sub = submodule_from_path(r, &null_oid, path);
+       if (!task->sub) {
+               /*
+                * No entry in .gitmodules? Technically not a submodule,
+                * but historically we supported repositories that happen to be
+                * in-place where a gitlink is. Keep supporting them.
+                */
+               task->sub = get_non_gitmodules_submodule(path);
+               if (!task->sub) {
+                       free(task);
+                       return NULL;
+               }
+               task->free_sub = 1;
+       }
+       return task;
+ }
+ static void fetch_task_release(struct fetch_task *p)
+ {
+       if (p->free_sub)
+               free((void*)p->sub);
+       p->free_sub = 0;
+       p->sub = NULL;
+       if (p->repo)
+               repo_clear(p->repo);
+       FREE_AND_NULL(p->repo);
+ }
+ static struct repository *get_submodule_repo_for(struct repository *r,
+                                                const struct submodule *sub)
+ {
+       struct repository *ret = xmalloc(sizeof(*ret));
+       if (repo_submodule_init(ret, r, sub)) {
+               /*
+                * No entry in .gitmodules? Technically not a submodule,
+                * but historically we supported repositories that happen to be
+                * in-place where a gitlink is. Keep supporting them.
+                */
+               struct strbuf gitdir = STRBUF_INIT;
+               strbuf_repo_worktree_path(&gitdir, r, "%s/.git", sub->path);
+               if (repo_init(ret, gitdir.buf, NULL)) {
+                       strbuf_release(&gitdir);
+                       free(ret);
+                       return NULL;
+               }
+               strbuf_release(&gitdir);
+       }
+       return ret;
+ }
  static int get_next_submodule(struct child_process *cp,
                              struct strbuf *err, void *data, void **task_cb)
  {
-       int ret = 0;
        struct submodule_parallel_fetch *spf = data;
  
        for (; spf->count < spf->r->index->cache_nr; spf->count++) {
-               struct strbuf submodule_path = STRBUF_INIT;
-               struct strbuf submodule_git_dir = STRBUF_INIT;
-               struct strbuf submodule_prefix = STRBUF_INIT;
                const struct cache_entry *ce = spf->r->index->cache[spf->count];
-               const char *git_dir, *default_argv;
-               const struct submodule *submodule;
-               struct submodule default_submodule = SUBMODULE_INIT;
+               const char *default_argv;
+               struct fetch_task *task;
  
                if (!S_ISGITLINK(ce->ce_mode))
                        continue;
  
-               submodule = submodule_from_path(spf->r, &null_oid, ce->name);
-               if (!submodule) {
-                       const char *name = default_name_or_path(ce->name);
-                       if (name) {
-                               default_submodule.path = default_submodule.name = name;
-                               submodule = &default_submodule;
-                       }
-               }
+               task = fetch_task_create(spf->r, ce->name);
+               if (!task)
+                       continue;
  
-               switch (get_fetch_recurse_config(submodule, spf))
+               switch (get_fetch_recurse_config(task->sub, spf))
                {
                default:
                case RECURSE_SUBMODULES_DEFAULT:
                case RECURSE_SUBMODULES_ON_DEMAND:
-                       if (!submodule || !unsorted_string_list_lookup(&changed_submodule_names,
-                                                        submodule->name))
+                       if (!task->sub ||
+                           !string_list_lookup(
+                                       &spf->changed_submodule_names,
+                                       task->sub->name))
                                continue;
                        default_argv = "on-demand";
                        break;
                        continue;
                }
  
-               strbuf_repo_worktree_path(&submodule_path, spf->r, "%s", ce->name);
-               strbuf_addf(&submodule_git_dir, "%s/.git", submodule_path.buf);
-               strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name);
-               git_dir = read_gitfile(submodule_git_dir.buf);
-               if (!git_dir)
-                       git_dir = submodule_git_dir.buf;
-               if (is_directory(git_dir)) {
+               task->repo = get_submodule_repo_for(spf->r, task->sub);
+               if (task->repo) {
+                       struct strbuf submodule_prefix = STRBUF_INIT;
                        child_process_init(cp);
-                       cp->dir = strbuf_detach(&submodule_path, NULL);
-                       prepare_submodule_repo_env(&cp->env_array);
+                       cp->dir = task->repo->gitdir;
+                       prepare_submodule_repo_env_in_gitdir(&cp->env_array);
                        cp->git_cmd = 1;
                        if (!spf->quiet)
                                strbuf_addf(err, "Fetching submodule %s%s\n",
                        argv_array_pushv(&cp->args, spf->args.argv);
                        argv_array_push(&cp->args, default_argv);
                        argv_array_push(&cp->args, "--submodule-prefix");
+                       strbuf_addf(&submodule_prefix, "%s%s/",
+                                                      spf->prefix,
+                                                      task->sub->path);
                        argv_array_push(&cp->args, submodule_prefix.buf);
-                       ret = 1;
-               }
-               strbuf_release(&submodule_path);
-               strbuf_release(&submodule_git_dir);
-               strbuf_release(&submodule_prefix);
-               if (ret) {
                        spf->count++;
+                       *task_cb = task;
+                       strbuf_release(&submodule_prefix);
                        return 1;
+               } else {
+                       fetch_task_release(task);
+                       free(task);
+                       /*
+                        * An empty directory is normal,
+                        * the submodule is not initialized
+                        */
+                       if (S_ISGITLINK(ce->ce_mode) &&
+                           !is_empty_dir(ce->name)) {
+                               spf->result = 1;
+                               strbuf_addf(err,
+                                           _("Could not access submodule '%s'"),
+                                           ce->name);
+                       }
                }
        }
+       if (spf->oid_fetch_tasks_nr) {
+               struct fetch_task *task =
+                       spf->oid_fetch_tasks[spf->oid_fetch_tasks_nr - 1];
+               struct strbuf submodule_prefix = STRBUF_INIT;
+               spf->oid_fetch_tasks_nr--;
+               strbuf_addf(&submodule_prefix, "%s%s/",
+                           spf->prefix, task->sub->path);
+               child_process_init(cp);
+               prepare_submodule_repo_env_in_gitdir(&cp->env_array);
+               cp->git_cmd = 1;
+               cp->dir = task->repo->gitdir;
+               argv_array_init(&cp->args);
+               argv_array_pushv(&cp->args, spf->args.argv);
+               argv_array_push(&cp->args, "on-demand");
+               argv_array_push(&cp->args, "--submodule-prefix");
+               argv_array_push(&cp->args, submodule_prefix.buf);
+               /* NEEDSWORK: have get_default_remote from submodule--helper */
+               argv_array_push(&cp->args, "origin");
+               oid_array_for_each_unique(task->commits,
+                                         append_oid_to_argv, &cp->args);
+               *task_cb = task;
+               strbuf_release(&submodule_prefix);
+               return 1;
+       }
        return 0;
  }
  
@@@ -1329,20 -1476,66 +1476,66 @@@ static int fetch_start_failure(struct s
                               void *cb, void *task_cb)
  {
        struct submodule_parallel_fetch *spf = cb;
+       struct fetch_task *task = task_cb;
  
        spf->result = 1;
  
+       fetch_task_release(task);
        return 0;
  }
  
+ static int commit_missing_in_sub(const struct object_id *oid, void *data)
+ {
+       struct repository *subrepo = data;
+       enum object_type type = oid_object_info(subrepo, oid, NULL);
+       return type != OBJ_COMMIT;
+ }
  static int fetch_finish(int retvalue, struct strbuf *err,
                        void *cb, void *task_cb)
  {
        struct submodule_parallel_fetch *spf = cb;
+       struct fetch_task *task = task_cb;
+       struct string_list_item *it;
+       struct oid_array *commits;
  
        if (retvalue)
                spf->result = 1;
  
+       if (!task || !task->sub)
+               BUG("callback cookie bogus");
+       /* Is this the second time we process this submodule? */
+       if (task->commits)
+               goto out;
+       it = string_list_lookup(&spf->changed_submodule_names, task->sub->name);
+       if (!it)
+               /* Could be an unchanged submodule, not contained in the list */
+               goto out;
+       commits = it->util;
+       oid_array_filter(commits,
+                        commit_missing_in_sub,
+                        task->repo);
+       /* Are there commits we want, but do not exist? */
+       if (commits->nr) {
+               task->commits = commits;
+               ALLOC_GROW(spf->oid_fetch_tasks,
+                          spf->oid_fetch_tasks_nr + 1,
+                          spf->oid_fetch_tasks_alloc);
+               spf->oid_fetch_tasks[spf->oid_fetch_tasks_nr] = task;
+               spf->oid_fetch_tasks_nr++;
+               return 0;
+       }
+ out:
+       fetch_task_release(task);
        return 0;
  }
  
@@@ -1373,7 -1566,8 +1566,8 @@@ int fetch_populated_submodules(struct r
        argv_array_push(&spf.args, "--recurse-submodules-default");
        /* default value, "--submodule-prefix" and its value are added later */
  
-       calculate_changed_submodule_paths(r);
+       calculate_changed_submodule_paths(r, &spf.changed_submodule_names);
+       string_list_sort(&spf.changed_submodule_names);
        run_processes_parallel(max_parallel_jobs,
                               get_next_submodule,
                               fetch_start_failure,
  
        argv_array_clear(&spf.args);
  out:
-       string_list_clear(&changed_submodule_names, 1);
+       free_submodules_oids(&spf.changed_submodule_names);
        return spf.result;
  }
  
        return ret;
  }
  
 +void submodule_unset_core_worktree(const struct submodule *sub)
 +{
 +      char *config_path = xstrfmt("%s/modules/%s/config",
 +                                  get_git_common_dir(), sub->name);
 +
 +      if (git_config_set_in_file_gently(config_path, "core.worktree", NULL))
 +              warning(_("Could not unset core.worktree setting in submodule '%s'"),
 +                        sub->path);
 +
 +      free(config_path);
 +}
 +
  static const char *get_super_prefix_or_empty(void)
  {
        const char *s = get_super_prefix();
@@@ -1738,8 -1920,6 +1932,8 @@@ int submodule_move_head(const char *pat
  
                        if (is_empty_dir(path))
                                rmdir_or_warn(path);
 +
 +                      submodule_unset_core_worktree(sub);
                }
        }
  out:
index a0317556c690262e26ed99cd902ec30c8adda733,9f8c744eb59da6237065e6eb9982dcb61ae0d300..63205dfdf962dc31c9db5ba038674e12760a9909
@@@ -524,8 -524,6 +524,8 @@@ test_expect_success 'fetching submodule
        git config fetch.recurseSubmodules true &&
        (
                cd downstream &&
 +              GIT_TRACE=$(pwd)/trace.out git fetch &&
 +              grep "1 tasks" trace.out &&
                GIT_TRACE=$(pwd)/trace.out git fetch --jobs 7 &&
                grep "7 tasks" trace.out &&
                git config submodule.fetchJobs 8 &&
@@@ -602,4 -600,121 +602,121 @@@ test_expect_success "fetch new commits 
        test_cmp expect actual
  '
  
+ test_expect_success "fetch new submodule commits on-demand outside standard refspec" '
+       # add a second submodule and ensure it is around in downstream first
+       git clone submodule sub1 &&
+       git submodule add ./sub1 &&
+       git commit -m "adding a second submodule" &&
+       git -C downstream pull &&
+       git -C downstream submodule update --init --recursive &&
+       git checkout --detach &&
+       C=$(git -C submodule commit-tree -m "new change outside refs/heads" HEAD^{tree}) &&
+       git -C submodule update-ref refs/changes/1 $C &&
+       git update-index --cacheinfo 160000 $C submodule &&
+       test_tick &&
+       D=$(git -C sub1 commit-tree -m "new change outside refs/heads" HEAD^{tree}) &&
+       git -C sub1 update-ref refs/changes/2 $D &&
+       git update-index --cacheinfo 160000 $D sub1 &&
+       git commit -m "updated submodules outside of refs/heads" &&
+       E=$(git rev-parse HEAD) &&
+       git update-ref refs/changes/3 $E &&
+       (
+               cd downstream &&
+               git fetch --recurse-submodules origin refs/changes/3:refs/heads/my_branch &&
+               git -C submodule cat-file -t $C &&
+               git -C sub1 cat-file -t $D &&
+               git checkout --recurse-submodules FETCH_HEAD
+       )
+ '
+ test_expect_success 'fetch new submodule commit on-demand in FETCH_HEAD' '
+       # depends on the previous test for setup
+       C=$(git -C submodule commit-tree -m "another change outside refs/heads" HEAD^{tree}) &&
+       git -C submodule update-ref refs/changes/4 $C &&
+       git update-index --cacheinfo 160000 $C submodule &&
+       test_tick &&
+       D=$(git -C sub1 commit-tree -m "another change outside refs/heads" HEAD^{tree}) &&
+       git -C sub1 update-ref refs/changes/5 $D &&
+       git update-index --cacheinfo 160000 $D sub1 &&
+       git commit -m "updated submodules outside of refs/heads" &&
+       E=$(git rev-parse HEAD) &&
+       git update-ref refs/changes/6 $E &&
+       (
+               cd downstream &&
+               git fetch --recurse-submodules origin refs/changes/6 &&
+               git -C submodule cat-file -t $C &&
+               git -C sub1 cat-file -t $D &&
+               git checkout --recurse-submodules FETCH_HEAD
+       )
+ '
+ test_expect_success 'fetch new submodule commits on-demand without .gitmodules entry' '
+       # depends on the previous test for setup
+       git config -f .gitmodules --remove-section submodule.sub1 &&
+       git add .gitmodules &&
+       git commit -m "delete gitmodules file" &&
+       git checkout -B master &&
+       git -C downstream fetch &&
+       git -C downstream checkout origin/master &&
+       C=$(git -C submodule commit-tree -m "yet another change outside refs/heads" HEAD^{tree}) &&
+       git -C submodule update-ref refs/changes/7 $C &&
+       git update-index --cacheinfo 160000 $C submodule &&
+       test_tick &&
+       D=$(git -C sub1 commit-tree -m "yet another change outside refs/heads" HEAD^{tree}) &&
+       git -C sub1 update-ref refs/changes/8 $D &&
+       git update-index --cacheinfo 160000 $D sub1 &&
+       git commit -m "updated submodules outside of refs/heads" &&
+       E=$(git rev-parse HEAD) &&
+       git update-ref refs/changes/9 $E &&
+       (
+               cd downstream &&
+               git fetch --recurse-submodules origin refs/changes/9 &&
+               git -C submodule cat-file -t $C &&
+               git -C sub1 cat-file -t $D &&
+               git checkout --recurse-submodules FETCH_HEAD
+       )
+ '
+ test_expect_success 'fetch new submodule commit intermittently referenced by superproject' '
+       # depends on the previous test for setup
+       D=$(git -C sub1 commit-tree -m "change 10 outside refs/heads" HEAD^{tree}) &&
+       E=$(git -C sub1 commit-tree -m "change 11 outside refs/heads" HEAD^{tree}) &&
+       F=$(git -C sub1 commit-tree -m "change 12 outside refs/heads" HEAD^{tree}) &&
+       git -C sub1 update-ref refs/changes/10 $D &&
+       git update-index --cacheinfo 160000 $D sub1 &&
+       git commit -m "updated submodules outside of refs/heads" &&
+       git -C sub1 update-ref refs/changes/11 $E &&
+       git update-index --cacheinfo 160000 $E sub1 &&
+       git commit -m "updated submodules outside of refs/heads" &&
+       git -C sub1 update-ref refs/changes/12 $F &&
+       git update-index --cacheinfo 160000 $F sub1 &&
+       git commit -m "updated submodules outside of refs/heads" &&
+       G=$(git rev-parse HEAD) &&
+       git update-ref refs/changes/13 $G &&
+       (
+               cd downstream &&
+               git fetch --recurse-submodules origin refs/changes/13 &&
+               git -C sub1 cat-file -t $D &&
+               git -C sub1 cat-file -t $E &&
+               git -C sub1 cat-file -t $F
+       )
+ '
  test_done