From: Junio C Hamano Date: Tue, 29 Jan 2019 20:47:54 +0000 (-0800) Subject: Merge branch 'sb/submodule-recursive-fetch-gets-the-tip' X-Git-Tag: v2.21.0-rc0~66 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/5d3635db19c6dff4fb063fabfa4161fd3b8285f0?ds=inline;hp=-c Merge branch 'sb/submodule-recursive-fetch-gets-the-tip' "git fetch --recurse-submodules" may not fetch the necessary commit that is bound to the superproject, which is getting corrected. * sb/submodule-recursive-fetch-gets-the-tip: fetch: ensure submodule objects fetched submodule.c: fetch in submodules git directory instead of in worktree submodule: migrate get_next_submodule to use repository structs repository: repo_submodule_init to take a submodule struct submodule: store OIDs in changed_submodule_names submodule.c: tighten scope of changed_submodule_names struct submodule.c: sort changed_submodule_names before searching it submodule.c: fix indentation sha1-array: provide oid_array_filter --- 5d3635db19c6dff4fb063fabfa4161fd3b8285f0 diff --combined builtin/fetch.c index c0ade48f5d,91f9b7d9c8..c316c03ba2 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@@ -763,9 -763,6 +763,6 @@@ static int update_local_ref(struct ref what = _("[new ref]"); } - if ((recurse_submodules != RECURSE_SUBMODULES_OFF) && - (recurse_submodules != RECURSE_SUBMODULES_ON)) - check_for_new_submodule_commits(&ref->new_oid); r = s_update_ref(msg, ref, 0); format_display(display, r ? '!' : '*', what, r ? _("unable to update local ref") : NULL, @@@ -779,9 -776,6 +776,6 @@@ strbuf_add_unique_abbrev(&quickref, ¤t->object.oid, DEFAULT_ABBREV); strbuf_addstr(&quickref, ".."); strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV); - if ((recurse_submodules != RECURSE_SUBMODULES_OFF) && - (recurse_submodules != RECURSE_SUBMODULES_ON)) - check_for_new_submodule_commits(&ref->new_oid); r = s_update_ref("fast-forward", ref, 1); format_display(display, r ? '!' : ' ', quickref.buf, r ? _("unable to update local ref") : NULL, @@@ -794,9 -788,6 +788,6 @@@ strbuf_add_unique_abbrev(&quickref, ¤t->object.oid, DEFAULT_ABBREV); strbuf_addstr(&quickref, "..."); strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV); - if ((recurse_submodules != RECURSE_SUBMODULES_OFF) && - (recurse_submodules != RECURSE_SUBMODULES_ON)) - check_for_new_submodule_commits(&ref->new_oid); r = s_update_ref("forced-update", ref, 1); format_display(display, r ? '!' : '+', quickref.buf, r ? _("unable to update local ref") : _("forced update"), @@@ -892,6 -883,8 +883,8 @@@ static int store_updated_refs(const cha ref->force = rm->peer_ref->force; } + if (recurse_submodules != RECURSE_SUBMODULES_OFF) + check_for_new_submodule_commits(&rm->old_oid); if (!strcmp(rm->name, "HEAD")) { kind = ""; @@@ -1478,8 -1471,7 +1471,8 @@@ static inline void fetch_one_setup_part */ if (strcmp(remote->name, repository_format_partial_clone)) { if (filter_options.choice) - die(_("--filter can only be used with the remote configured in core.partialClone")); + die(_("--filter can only be used with the remote " + "configured in extensions.partialclone")); return; } @@@ -1647,8 -1639,7 +1640,8 @@@ int cmd_fetch(int argc, const char **ar result = fetch_one(remote, argc, argv, prune_tags_ok); } else { if (filter_options.choice) - die(_("--filter can only be used with the remote configured in core.partialClone")); + die(_("--filter can only be used with the remote " + "configured in extensions.partialclone")); /* TODO should this also die if we have a previous partial-clone? */ result = fetch_multiple(&list); } diff --combined builtin/grep.c index 4748195ae1,d6bd887b2d..dc91838764 --- a/builtin/grep.c +++ b/builtin/grep.c @@@ -404,7 -404,10 +404,10 @@@ static int grep_submodule(struct grep_o const struct object_id *oid, const char *filename, const char *path) { - struct repository submodule; + struct repository subrepo; + const struct submodule *sub = submodule_from_path(superproject, + &null_oid, path); + int hit; /* @@@ -420,12 -423,12 +423,12 @@@ return 0; } - if (repo_submodule_init(&submodule, superproject, path)) { + if (repo_submodule_init(&subrepo, superproject, sub)) { grep_read_unlock(); return 0; } - repo_read_gitmodules(&submodule); + repo_read_gitmodules(&subrepo); /* * NEEDSWORK: This adds the submodule's object directory to the list of @@@ -437,7 -440,7 +440,7 @@@ * store is no longer global and instead is a member of the repository * object. */ - add_to_alternates_memory(submodule.objects->odb->path); - add_to_alternates_memory(subrepo.objects->objectdir); ++ add_to_alternates_memory(subrepo.objects->odb->path); grep_read_unlock(); if (oid) { @@@ -462,14 -465,14 +465,14 @@@ init_tree_desc(&tree, data, size); hit = grep_tree(opt, pathspec, &tree, &base, base.len, - object->type == OBJ_COMMIT, &submodule); + object->type == OBJ_COMMIT, &subrepo); strbuf_release(&base); free(data); } else { - hit = grep_cache(opt, &submodule, pathspec, 1); + hit = grep_cache(opt, &subrepo, pathspec, 1); } - repo_clear(&submodule); + repo_clear(&subrepo); return hit; } @@@ -553,8 -556,7 +556,8 @@@ static int grep_tree(struct grep_opt *o if (match != all_entries_interesting) { strbuf_addstr(&name, base->buf + tn_len); - match = tree_entry_interesting(&entry, &name, + match = tree_entry_interesting(repo->index, + &entry, &name, 0, pathspec); strbuf_setlen(&name, name_base_len); diff --combined builtin/ls-files.c index 7e0dcaa359,583a0e1ca2..cde87cbeeb --- a/builtin/ls-files.c +++ b/builtin/ls-files.c @@@ -206,17 -206,19 +206,19 @@@ static void show_files(struct repositor static void show_submodule(struct repository *superproject, struct dir_struct *dir, const char *path) { - struct repository submodule; + struct repository subrepo; + const struct submodule *sub = submodule_from_path(superproject, + &null_oid, path); - if (repo_submodule_init(&submodule, superproject, path)) + if (repo_submodule_init(&subrepo, superproject, sub)) return; - if (repo_read_index(&submodule) < 0) + if (repo_read_index(&subrepo) < 0) die("index file corrupt"); - show_files(&submodule, dir); + show_files(&subrepo, dir); - repo_clear(&submodule); + repo_clear(&subrepo); } static void show_ce(struct repository *repo, struct dir_struct *dir, @@@ -441,7 -443,7 +443,7 @@@ void overlay_tree_on_index(struct index PATHSPEC_PREFER_CWD, prefix, matchbuf); } else memset(&pathspec, 0, sizeof(pathspec)); - if (read_tree(tree, 1, &pathspec, istate)) + if (read_tree(the_repository, tree, 1, &pathspec, istate)) die("unable to read tree entries %s", tree_name); for (i = 0; i < istate->cache_nr; i++) { diff --combined builtin/submodule--helper.c index 6881b6a9cb,4eceb8f040..0e140f176c --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@@ -1131,8 -1131,6 +1131,8 @@@ static void deinit_submodule(const cha if (!(flags & OPT_QUIET)) printf(format, displaypath); + submodule_unset_core_worktree(sub); + strbuf_release(&sb_rm); } @@@ -1267,20 -1265,19 +1267,20 @@@ struct submodule_alternate_setup SUBMODULE_ALTERNATE_ERROR_IGNORE, NULL } static int add_possible_reference_from_superproject( - struct alternate_object_database *alt, void *sas_cb) + struct object_directory *odb, void *sas_cb) { struct submodule_alternate_setup *sas = sas_cb; + size_t len; /* * If the alternate object store is another repository, try the * standard layout with .git/(modules/)+/objects */ - if (ends_with(alt->path, "/objects")) { + if (strip_suffix(odb->path, "/objects", &len)) { char *sm_alternate; struct strbuf sb = STRBUF_INIT; struct strbuf err = STRBUF_INIT; - strbuf_add(&sb, alt->path, strlen(alt->path) - strlen("objects")); + strbuf_add(&sb, odb->path, len); /* * We need to end the new path with '/' to mark it as a dir, @@@ -1288,7 -1285,7 +1288,7 @@@ * as the last part of a missing submodule reference would * be taken as a file name. */ - strbuf_addf(&sb, "modules/%s/", sas->submodule_name); + strbuf_addf(&sb, "/modules/%s/", sas->submodule_name); sm_alternate = compute_alternate_path(sb.buf, &err); if (sm_alternate) { @@@ -1554,7 -1551,7 +1554,7 @@@ struct submodule_update_clone #define SUBMODULE_UPDATE_CLONE_INIT {0, MODULE_LIST_INIT, 0, \ SUBMODULE_UPDATE_STRATEGY_INIT, 0, 0, -1, STRING_LIST_INIT_DUP, 0, \ NULL, NULL, NULL, \ - NULL, 0, 0, 0, NULL, 0, 0, 0} + NULL, 0, 0, 0, NULL, 0, 0, 1} static void next_submodule_warn_missing(struct submodule_update_clone *suc, @@@ -2048,7 -2045,7 +2048,7 @@@ static int ensure_core_worktree(int arg struct repository subrepo; if (argc != 2) - BUG("submodule--helper connect-gitdir-workingtree "); + BUG("submodule--helper ensure-core-worktree "); path = argv[1]; @@@ -2056,7 -2053,7 +2056,7 @@@ if (!sub) BUG("We could get the submodule handle before?"); - if (repo_submodule_init(&subrepo, the_repository, path)) + if (repo_submodule_init(&subrepo, the_repository, sub)) die(_("could not get a repository handle for submodule '%s'"), path); if (!repo_config_get_string(&subrepo, "core.worktree", &cw)) { diff --combined repository.c index 7b02e1dffa,aabe64ee5d..20c509a922 --- a/repository.c +++ b/repository.c @@@ -63,14 -63,8 +63,14 @@@ void repo_set_gitdir(struct repository free(old_gitdir); repo_set_commondir(repo, o->commondir); - expand_base_dir(&repo->objects->objectdir, o->object_dir, + + if (!repo->objects->odb) { + repo->objects->odb = xcalloc(1, sizeof(*repo->objects->odb)); + repo->objects->odb_tail = &repo->objects->odb->next; + } + expand_base_dir(&repo->objects->odb->path, o->object_dir, repo->commondir, "objects"); + free(repo->objects->alternate_db); repo->objects->alternate_db = xstrdup_or_null(o->alternate_db); expand_base_dir(&repo->graft_file, o->graft_file, @@@ -172,30 -166,23 +172,23 @@@ error return -1; } - /* - * Initialize 'submodule' as the submodule given by 'path' in parent repository - * 'superproject'. - * Return 0 upon success and a non-zero value upon failure. - */ - int repo_submodule_init(struct repository *submodule, + int repo_submodule_init(struct repository *subrepo, struct repository *superproject, - const char *path) + const struct submodule *sub) { - const struct submodule *sub; struct strbuf gitdir = STRBUF_INIT; struct strbuf worktree = STRBUF_INIT; int ret = 0; - sub = submodule_from_path(superproject, &null_oid, path); if (!sub) { ret = -1; goto out; } - strbuf_repo_worktree_path(&gitdir, superproject, "%s/.git", path); - strbuf_repo_worktree_path(&worktree, superproject, "%s", path); + strbuf_repo_worktree_path(&gitdir, superproject, "%s/.git", sub->path); + strbuf_repo_worktree_path(&worktree, superproject, "%s", sub->path); - if (repo_init(submodule, gitdir.buf, worktree.buf)) { + if (repo_init(subrepo, gitdir.buf, worktree.buf)) { /* * If initilization fails then it may be due to the submodule * not being populated in the superproject's worktree. Instead @@@ -207,16 -194,16 +200,16 @@@ strbuf_repo_git_path(&gitdir, superproject, "modules/%s", sub->name); - if (repo_init(submodule, gitdir.buf, NULL)) { + if (repo_init(subrepo, gitdir.buf, NULL)) { ret = -1; goto out; } } - submodule->submodule_prefix = xstrfmt("%s%s/", - superproject->submodule_prefix ? - superproject->submodule_prefix : - "", path); + subrepo->submodule_prefix = xstrfmt("%s%s/", + superproject->submodule_prefix ? + superproject->submodule_prefix : + "", sub->path); out: strbuf_release(&gitdir); diff --combined submodule.c index d393e947e6,b88343d977..7b5cea8522 --- a/submodule.c +++ b/submodule.c @@@ -25,7 -25,6 +25,6 @@@ #include "commit-reach.h" static int config_update_recurse_submodules = RECURSE_SUBMODULES_OFF; - static struct string_list changed_submodule_names = STRING_LIST_INIT_DUP; static int initialized_fetch_ref_tips; static struct oid_array ref_tips_before_fetch; static struct oid_array ref_tips_after_fetch; @@@ -495,6 -494,12 +494,12 @@@ void prepare_submodule_repo_env(struct DEFAULT_GIT_DIR_ENVIRONMENT); } + static void prepare_submodule_repo_env_in_gitdir(struct argv_array *out) + { + prepare_submodule_repo_env_no_git_dir(out); + argv_array_pushf(out, "%s=.", GIT_DIR_ENVIRONMENT); + } + /* Helper function to display the submodule header line prior to the full * summary output. If it can locate the submodule objects directory it will * attempt to lookup both the left and right commits and put them into the @@@ -1136,11 -1141,11 +1141,11 @@@ void check_for_new_submodule_commits(st oid_array_append(&ref_tips_after_fetch, oid); } - static void calculate_changed_submodule_paths(struct repository *r) + static void calculate_changed_submodule_paths(struct repository *r, + struct string_list *changed_submodule_names) { struct argv_array argv = ARGV_ARRAY_INIT; - struct string_list changed_submodules = STRING_LIST_INIT_DUP; - const struct string_list_item *name; + struct string_list_item *name; /* No need to check if there are no submodules configured */ if (!submodule_from_path(r, NULL, NULL)) @@@ -1157,9 -1162,9 +1162,9 @@@ * Collect all submodules (whether checked out or not) for which new * commits have been recorded upstream in "changed_submodule_names". */ - collect_changed_submodules(r, &changed_submodules, &argv); + collect_changed_submodules(r, changed_submodule_names, &argv); - for_each_string_list_item(name, &changed_submodules) { + for_each_string_list_item(name, changed_submodule_names) { struct oid_array *commits = name->util; const struct submodule *submodule; const char *path = NULL; @@@ -1173,11 -1178,14 +1178,14 @@@ if (!path) continue; - if (!submodule_has_commits(r, path, commits)) - string_list_append(&changed_submodule_names, name->string); + if (submodule_has_commits(r, path, commits)) { + oid_array_clear(commits); + *name->string = '\0'; + } } - free_submodules_oids(&changed_submodules); + string_list_remove_empty_items(changed_submodule_names, 1); + argv_array_clear(&argv); oid_array_clear(&ref_tips_before_fetch); oid_array_clear(&ref_tips_after_fetch); @@@ -1221,8 -1229,16 +1229,16 @@@ struct submodule_parallel_fetch int default_option; int quiet; int result; + + struct string_list changed_submodule_names; + + /* Pending fetches by OIDs */ + struct fetch_task **oid_fetch_tasks; + int oid_fetch_tasks_nr, oid_fetch_tasks_alloc; }; - #define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0} + #define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0, \ + STRING_LIST_INIT_DUP, \ + NULL, 0, 0} static int get_fetch_recurse_config(const struct submodule *submodule, struct submodule_parallel_fetch *spf) @@@ -1249,40 -1265,126 +1265,126 @@@ return spf->default_option; } + /* + * Fetch in progress (if callback data) or + * pending (if in oid_fetch_tasks in struct submodule_parallel_fetch) + */ + struct fetch_task { + struct repository *repo; + const struct submodule *sub; + unsigned free_sub : 1; /* Do we need to free the submodule? */ + + struct oid_array *commits; /* Ensure these commits are fetched */ + }; + + /** + * When a submodule is not defined in .gitmodules, we cannot access it + * via the regular submodule-config. Create a fake submodule, which we can + * work on. + */ + static const struct submodule *get_non_gitmodules_submodule(const char *path) + { + struct submodule *ret = NULL; + const char *name = default_name_or_path(path); + + if (!name) + return NULL; + + ret = xmalloc(sizeof(*ret)); + memset(ret, 0, sizeof(*ret)); + ret->path = name; + ret->name = name; + + return (const struct submodule *) ret; + } + + static struct fetch_task *fetch_task_create(struct repository *r, + const char *path) + { + struct fetch_task *task = xmalloc(sizeof(*task)); + memset(task, 0, sizeof(*task)); + + task->sub = submodule_from_path(r, &null_oid, path); + if (!task->sub) { + /* + * No entry in .gitmodules? Technically not a submodule, + * but historically we supported repositories that happen to be + * in-place where a gitlink is. Keep supporting them. + */ + task->sub = get_non_gitmodules_submodule(path); + if (!task->sub) { + free(task); + return NULL; + } + + task->free_sub = 1; + } + + return task; + } + + static void fetch_task_release(struct fetch_task *p) + { + if (p->free_sub) + free((void*)p->sub); + p->free_sub = 0; + p->sub = NULL; + + if (p->repo) + repo_clear(p->repo); + FREE_AND_NULL(p->repo); + } + + static struct repository *get_submodule_repo_for(struct repository *r, + const struct submodule *sub) + { + struct repository *ret = xmalloc(sizeof(*ret)); + + if (repo_submodule_init(ret, r, sub)) { + /* + * No entry in .gitmodules? Technically not a submodule, + * but historically we supported repositories that happen to be + * in-place where a gitlink is. Keep supporting them. + */ + struct strbuf gitdir = STRBUF_INIT; + strbuf_repo_worktree_path(&gitdir, r, "%s/.git", sub->path); + if (repo_init(ret, gitdir.buf, NULL)) { + strbuf_release(&gitdir); + free(ret); + return NULL; + } + strbuf_release(&gitdir); + } + + return ret; + } + static int get_next_submodule(struct child_process *cp, struct strbuf *err, void *data, void **task_cb) { - int ret = 0; struct submodule_parallel_fetch *spf = data; for (; spf->count < spf->r->index->cache_nr; spf->count++) { - struct strbuf submodule_path = STRBUF_INIT; - struct strbuf submodule_git_dir = STRBUF_INIT; - struct strbuf submodule_prefix = STRBUF_INIT; const struct cache_entry *ce = spf->r->index->cache[spf->count]; - const char *git_dir, *default_argv; - const struct submodule *submodule; - struct submodule default_submodule = SUBMODULE_INIT; + const char *default_argv; + struct fetch_task *task; if (!S_ISGITLINK(ce->ce_mode)) continue; - submodule = submodule_from_path(spf->r, &null_oid, ce->name); - if (!submodule) { - const char *name = default_name_or_path(ce->name); - if (name) { - default_submodule.path = default_submodule.name = name; - submodule = &default_submodule; - } - } + task = fetch_task_create(spf->r, ce->name); + if (!task) + continue; - switch (get_fetch_recurse_config(submodule, spf)) + switch (get_fetch_recurse_config(task->sub, spf)) { default: case RECURSE_SUBMODULES_DEFAULT: case RECURSE_SUBMODULES_ON_DEMAND: - if (!submodule || !unsorted_string_list_lookup(&changed_submodule_names, - submodule->name)) + if (!task->sub || + !string_list_lookup( + &spf->changed_submodule_names, + task->sub->name)) continue; default_argv = "on-demand"; break; @@@ -1293,16 -1395,12 +1395,12 @@@ continue; } - strbuf_repo_worktree_path(&submodule_path, spf->r, "%s", ce->name); - strbuf_addf(&submodule_git_dir, "%s/.git", submodule_path.buf); - strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name); - git_dir = read_gitfile(submodule_git_dir.buf); - if (!git_dir) - git_dir = submodule_git_dir.buf; - if (is_directory(git_dir)) { + task->repo = get_submodule_repo_for(spf->r, task->sub); + if (task->repo) { + struct strbuf submodule_prefix = STRBUF_INIT; child_process_init(cp); - cp->dir = strbuf_detach(&submodule_path, NULL); - prepare_submodule_repo_env(&cp->env_array); + cp->dir = task->repo->gitdir; + prepare_submodule_repo_env_in_gitdir(&cp->env_array); cp->git_cmd = 1; if (!spf->quiet) strbuf_addf(err, "Fetching submodule %s%s\n", @@@ -1311,17 -1409,66 +1409,66 @@@ argv_array_pushv(&cp->args, spf->args.argv); argv_array_push(&cp->args, default_argv); argv_array_push(&cp->args, "--submodule-prefix"); + + strbuf_addf(&submodule_prefix, "%s%s/", + spf->prefix, + task->sub->path); argv_array_push(&cp->args, submodule_prefix.buf); - ret = 1; - } - strbuf_release(&submodule_path); - strbuf_release(&submodule_git_dir); - strbuf_release(&submodule_prefix); - if (ret) { + spf->count++; + *task_cb = task; + + strbuf_release(&submodule_prefix); return 1; + } else { + + fetch_task_release(task); + free(task); + + /* + * An empty directory is normal, + * the submodule is not initialized + */ + if (S_ISGITLINK(ce->ce_mode) && + !is_empty_dir(ce->name)) { + spf->result = 1; + strbuf_addf(err, + _("Could not access submodule '%s'"), + ce->name); + } } } + + if (spf->oid_fetch_tasks_nr) { + struct fetch_task *task = + spf->oid_fetch_tasks[spf->oid_fetch_tasks_nr - 1]; + struct strbuf submodule_prefix = STRBUF_INIT; + spf->oid_fetch_tasks_nr--; + + strbuf_addf(&submodule_prefix, "%s%s/", + spf->prefix, task->sub->path); + + child_process_init(cp); + prepare_submodule_repo_env_in_gitdir(&cp->env_array); + cp->git_cmd = 1; + cp->dir = task->repo->gitdir; + + argv_array_init(&cp->args); + argv_array_pushv(&cp->args, spf->args.argv); + argv_array_push(&cp->args, "on-demand"); + argv_array_push(&cp->args, "--submodule-prefix"); + argv_array_push(&cp->args, submodule_prefix.buf); + + /* NEEDSWORK: have get_default_remote from submodule--helper */ + argv_array_push(&cp->args, "origin"); + oid_array_for_each_unique(task->commits, + append_oid_to_argv, &cp->args); + + *task_cb = task; + strbuf_release(&submodule_prefix); + return 1; + } + return 0; } @@@ -1329,20 -1476,66 +1476,66 @@@ static int fetch_start_failure(struct s void *cb, void *task_cb) { struct submodule_parallel_fetch *spf = cb; + struct fetch_task *task = task_cb; spf->result = 1; + fetch_task_release(task); return 0; } + static int commit_missing_in_sub(const struct object_id *oid, void *data) + { + struct repository *subrepo = data; + + enum object_type type = oid_object_info(subrepo, oid, NULL); + + return type != OBJ_COMMIT; + } + static int fetch_finish(int retvalue, struct strbuf *err, void *cb, void *task_cb) { struct submodule_parallel_fetch *spf = cb; + struct fetch_task *task = task_cb; + + struct string_list_item *it; + struct oid_array *commits; if (retvalue) spf->result = 1; + if (!task || !task->sub) + BUG("callback cookie bogus"); + + /* Is this the second time we process this submodule? */ + if (task->commits) + goto out; + + it = string_list_lookup(&spf->changed_submodule_names, task->sub->name); + if (!it) + /* Could be an unchanged submodule, not contained in the list */ + goto out; + + commits = it->util; + oid_array_filter(commits, + commit_missing_in_sub, + task->repo); + + /* Are there commits we want, but do not exist? */ + if (commits->nr) { + task->commits = commits; + ALLOC_GROW(spf->oid_fetch_tasks, + spf->oid_fetch_tasks_nr + 1, + spf->oid_fetch_tasks_alloc); + spf->oid_fetch_tasks[spf->oid_fetch_tasks_nr] = task; + spf->oid_fetch_tasks_nr++; + return 0; + } + + out: + fetch_task_release(task); + return 0; } @@@ -1373,7 -1566,8 +1566,8 @@@ int fetch_populated_submodules(struct r argv_array_push(&spf.args, "--recurse-submodules-default"); /* default value, "--submodule-prefix" and its value are added later */ - calculate_changed_submodule_paths(r); + calculate_changed_submodule_paths(r, &spf.changed_submodule_names); + string_list_sort(&spf.changed_submodule_names); run_processes_parallel(max_parallel_jobs, get_next_submodule, fetch_start_failure, @@@ -1382,7 -1576,7 +1576,7 @@@ argv_array_clear(&spf.args); out: - string_list_clear(&changed_submodule_names, 1); + free_submodules_oids(&spf.changed_submodule_names); return spf.result; } @@@ -1561,18 -1755,6 +1755,18 @@@ out return ret; } +void submodule_unset_core_worktree(const struct submodule *sub) +{ + char *config_path = xstrfmt("%s/modules/%s/config", + get_git_common_dir(), sub->name); + + if (git_config_set_in_file_gently(config_path, "core.worktree", NULL)) + warning(_("Could not unset core.worktree setting in submodule '%s'"), + sub->path); + + free(config_path); +} + static const char *get_super_prefix_or_empty(void) { const char *s = get_super_prefix(); @@@ -1738,8 -1920,6 +1932,8 @@@ int submodule_move_head(const char *pat if (is_empty_dir(path)) rmdir_or_warn(path); + + submodule_unset_core_worktree(sub); } } out: diff --combined t/t5526-fetch-submodules.sh index a0317556c6,9f8c744eb5..63205dfdf9 --- a/t/t5526-fetch-submodules.sh +++ b/t/t5526-fetch-submodules.sh @@@ -524,8 -524,6 +524,8 @@@ test_expect_success 'fetching submodule git config fetch.recurseSubmodules true && ( cd downstream && + GIT_TRACE=$(pwd)/trace.out git fetch && + grep "1 tasks" trace.out && GIT_TRACE=$(pwd)/trace.out git fetch --jobs 7 && grep "7 tasks" trace.out && git config submodule.fetchJobs 8 && @@@ -602,4 -600,121 +602,121 @@@ test_expect_success "fetch new commits test_cmp expect actual ' + test_expect_success "fetch new submodule commits on-demand outside standard refspec" ' + # add a second submodule and ensure it is around in downstream first + git clone submodule sub1 && + git submodule add ./sub1 && + git commit -m "adding a second submodule" && + git -C downstream pull && + git -C downstream submodule update --init --recursive && + + git checkout --detach && + + C=$(git -C submodule commit-tree -m "new change outside refs/heads" HEAD^{tree}) && + git -C submodule update-ref refs/changes/1 $C && + git update-index --cacheinfo 160000 $C submodule && + test_tick && + + D=$(git -C sub1 commit-tree -m "new change outside refs/heads" HEAD^{tree}) && + git -C sub1 update-ref refs/changes/2 $D && + git update-index --cacheinfo 160000 $D sub1 && + + git commit -m "updated submodules outside of refs/heads" && + E=$(git rev-parse HEAD) && + git update-ref refs/changes/3 $E && + ( + cd downstream && + git fetch --recurse-submodules origin refs/changes/3:refs/heads/my_branch && + git -C submodule cat-file -t $C && + git -C sub1 cat-file -t $D && + git checkout --recurse-submodules FETCH_HEAD + ) + ' + + test_expect_success 'fetch new submodule commit on-demand in FETCH_HEAD' ' + # depends on the previous test for setup + + C=$(git -C submodule commit-tree -m "another change outside refs/heads" HEAD^{tree}) && + git -C submodule update-ref refs/changes/4 $C && + git update-index --cacheinfo 160000 $C submodule && + test_tick && + + D=$(git -C sub1 commit-tree -m "another change outside refs/heads" HEAD^{tree}) && + git -C sub1 update-ref refs/changes/5 $D && + git update-index --cacheinfo 160000 $D sub1 && + + git commit -m "updated submodules outside of refs/heads" && + E=$(git rev-parse HEAD) && + git update-ref refs/changes/6 $E && + ( + cd downstream && + git fetch --recurse-submodules origin refs/changes/6 && + git -C submodule cat-file -t $C && + git -C sub1 cat-file -t $D && + git checkout --recurse-submodules FETCH_HEAD + ) + ' + + test_expect_success 'fetch new submodule commits on-demand without .gitmodules entry' ' + # depends on the previous test for setup + + git config -f .gitmodules --remove-section submodule.sub1 && + git add .gitmodules && + git commit -m "delete gitmodules file" && + git checkout -B master && + git -C downstream fetch && + git -C downstream checkout origin/master && + + C=$(git -C submodule commit-tree -m "yet another change outside refs/heads" HEAD^{tree}) && + git -C submodule update-ref refs/changes/7 $C && + git update-index --cacheinfo 160000 $C submodule && + test_tick && + + D=$(git -C sub1 commit-tree -m "yet another change outside refs/heads" HEAD^{tree}) && + git -C sub1 update-ref refs/changes/8 $D && + git update-index --cacheinfo 160000 $D sub1 && + + git commit -m "updated submodules outside of refs/heads" && + E=$(git rev-parse HEAD) && + git update-ref refs/changes/9 $E && + ( + cd downstream && + git fetch --recurse-submodules origin refs/changes/9 && + git -C submodule cat-file -t $C && + git -C sub1 cat-file -t $D && + git checkout --recurse-submodules FETCH_HEAD + ) + ' + + test_expect_success 'fetch new submodule commit intermittently referenced by superproject' ' + # depends on the previous test for setup + + D=$(git -C sub1 commit-tree -m "change 10 outside refs/heads" HEAD^{tree}) && + E=$(git -C sub1 commit-tree -m "change 11 outside refs/heads" HEAD^{tree}) && + F=$(git -C sub1 commit-tree -m "change 12 outside refs/heads" HEAD^{tree}) && + + git -C sub1 update-ref refs/changes/10 $D && + git update-index --cacheinfo 160000 $D sub1 && + git commit -m "updated submodules outside of refs/heads" && + + git -C sub1 update-ref refs/changes/11 $E && + git update-index --cacheinfo 160000 $E sub1 && + git commit -m "updated submodules outside of refs/heads" && + + git -C sub1 update-ref refs/changes/12 $F && + git update-index --cacheinfo 160000 $F sub1 && + git commit -m "updated submodules outside of refs/heads" && + + G=$(git rev-parse HEAD) && + git update-ref refs/changes/13 $G && + ( + cd downstream && + git fetch --recurse-submodules origin refs/changes/13 && + + git -C sub1 cat-file -t $D && + git -C sub1 cat-file -t $E && + git -C sub1 cat-file -t $F + ) + ' + test_done