From: Junio C Hamano Date: Tue, 17 May 2016 21:38:25 +0000 (-0700) Subject: Merge branch 'jk/submodule-c-credential' X-Git-Tag: v2.9.0-rc0~34 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/e059388fb200fb2b5281ce211242ef3525a9c2c7?ds=inline;hp=-c Merge branch 'jk/submodule-c-credential' An earlier addition of "sanitize_submodule_env" with 14111fc4 (git: submodule honor -c credential.* from command line, 2016-02-29) turned out to be a convoluted no-op; implement what it wanted to do correctly, and stop filtering settings given via "git -c var=val". * jk/submodule-c-credential: submodule: stop sanitizing config options submodule: use prepare_submodule_repo_env consistently submodule--helper: move config-sanitizing to submodule.c submodule: export sanitized GIT_CONFIG_PARAMETERS t5550: break submodule config test into multiple sub-tests t5550: fix typo in $HTTPD_URL --- e059388fb200fb2b5281ce211242ef3525a9c2c7 diff --combined builtin/submodule--helper.c index 825a421fc9,89250f0b78..8da263f0b0 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@@ -9,211 -9,6 +9,211 @@@ #include "submodule-config.h" #include "string-list.h" #include "run-command.h" +#include "remote.h" +#include "refs.h" +#include "connect.h" + +static char *get_default_remote(void) +{ + char *dest = NULL, *ret; + unsigned char sha1[20]; + struct strbuf sb = STRBUF_INIT; + const char *refname = resolve_ref_unsafe("HEAD", 0, sha1, NULL); + + if (!refname) + die(_("No such ref: %s"), "HEAD"); + + /* detached HEAD */ + if (!strcmp(refname, "HEAD")) + return xstrdup("origin"); + + if (!skip_prefix(refname, "refs/heads/", &refname)) + die(_("Expecting a full ref name, got %s"), refname); + + strbuf_addf(&sb, "branch.%s.remote", refname); + if (git_config_get_string(sb.buf, &dest)) + ret = xstrdup("origin"); + else + ret = dest; + + strbuf_release(&sb); + return ret; +} + +static int starts_with_dot_slash(const char *str) +{ + return str[0] == '.' && is_dir_sep(str[1]); +} + +static int starts_with_dot_dot_slash(const char *str) +{ + return str[0] == '.' && str[1] == '.' && is_dir_sep(str[2]); +} + +/* + * Returns 1 if it was the last chop before ':'. + */ +static int chop_last_dir(char **remoteurl, int is_relative) +{ + char *rfind = find_last_dir_sep(*remoteurl); + if (rfind) { + *rfind = '\0'; + return 0; + } + + rfind = strrchr(*remoteurl, ':'); + if (rfind) { + *rfind = '\0'; + return 1; + } + + if (is_relative || !strcmp(".", *remoteurl)) + die(_("cannot strip one component off url '%s'"), + *remoteurl); + + free(*remoteurl); + *remoteurl = xstrdup("."); + return 0; +} + +/* + * The `url` argument is the URL that navigates to the submodule origin + * repo. When relative, this URL is relative to the superproject origin + * URL repo. The `up_path` argument, if specified, is the relative + * path that navigates from the submodule working tree to the superproject + * working tree. Returns the origin URL of the submodule. + * + * Return either an absolute URL or filesystem path (if the superproject + * origin URL is an absolute URL or filesystem path, respectively) or a + * relative file system path (if the superproject origin URL is a relative + * file system path). + * + * When the output is a relative file system path, the path is either + * relative to the submodule working tree, if up_path is specified, or to + * the superproject working tree otherwise. + * + * NEEDSWORK: This works incorrectly on the domain and protocol part. + * remote_url url outcome expectation + * http://a.com/b ../c http://a.com/c as is + * http://a.com/b ../../c http://c error out + * http://a.com/b ../../../c http:/c error out + * http://a.com/b ../../../../c http:c error out + * http://a.com/b ../../../../../c .:c error out + * NEEDSWORK: Given how chop_last_dir() works, this function is broken + * when a local part has a colon in its path component, too. + */ +static char *relative_url(const char *remote_url, + const char *url, + const char *up_path) +{ + int is_relative = 0; + int colonsep = 0; + char *out; + char *remoteurl = xstrdup(remote_url); + struct strbuf sb = STRBUF_INIT; + size_t len = strlen(remoteurl); + + if (is_dir_sep(remoteurl[len])) + remoteurl[len] = '\0'; + + if (!url_is_local_not_ssh(remoteurl) || is_absolute_path(remoteurl)) + is_relative = 0; + else { + is_relative = 1; + /* + * Prepend a './' to ensure all relative + * remoteurls start with './' or '../' + */ + if (!starts_with_dot_slash(remoteurl) && + !starts_with_dot_dot_slash(remoteurl)) { + strbuf_reset(&sb); + strbuf_addf(&sb, "./%s", remoteurl); + free(remoteurl); + remoteurl = strbuf_detach(&sb, NULL); + } + } + /* + * When the url starts with '../', remove that and the + * last directory in remoteurl. + */ + while (url) { + if (starts_with_dot_dot_slash(url)) { + url += 3; + colonsep |= chop_last_dir(&remoteurl, is_relative); + } else if (starts_with_dot_slash(url)) + url += 2; + else + break; + } + strbuf_reset(&sb); + strbuf_addf(&sb, "%s%s%s", remoteurl, colonsep ? ":" : "/", url); + free(remoteurl); + + if (starts_with_dot_slash(sb.buf)) + out = xstrdup(sb.buf + 2); + else + out = xstrdup(sb.buf); + strbuf_reset(&sb); + + if (!up_path || !is_relative) + return out; + + strbuf_addf(&sb, "%s%s", up_path, out); + free(out); + return strbuf_detach(&sb, NULL); +} + +static int resolve_relative_url(int argc, const char **argv, const char *prefix) +{ + char *remoteurl = NULL; + char *remote = get_default_remote(); + const char *up_path = NULL; + char *res; + const char *url; + struct strbuf sb = STRBUF_INIT; + + if (argc != 2 && argc != 3) + die("resolve-relative-url only accepts one or two arguments"); + + url = argv[1]; + strbuf_addf(&sb, "remote.%s.url", remote); + free(remote); + + if (git_config_get_string(sb.buf, &remoteurl)) + /* the repository is its own authoritative upstream */ + remoteurl = xgetcwd(); + + if (argc == 3) + up_path = argv[2]; + + res = relative_url(remoteurl, url, up_path); + puts(res); + free(res); + free(remoteurl); + return 0; +} + +static int resolve_relative_url_test(int argc, const char **argv, const char *prefix) +{ + char *remoteurl, *res; + const char *up_path, *url; + + if (argc != 4) + die("resolve-relative-url-test only accepts three arguments: "); + + up_path = argv[1]; + remoteurl = xstrdup(argv[2]); + url = argv[3]; + + if (!strcmp(up_path, "(null)")) + up_path = NULL; + + res = relative_url(remoteurl, url, up_path); + puts(res); + free(res); + free(remoteurl); + return 0; +} struct module_list { const struct cache_entry **entries; @@@ -227,12 -22,17 +227,12 @@@ static int module_list_compute(int argc struct module_list *list) { int i, result = 0; - char *max_prefix, *ps_matched = NULL; - int max_prefix_len; + char *ps_matched = NULL; parse_pathspec(pathspec, 0, PATHSPEC_PREFER_FULL | PATHSPEC_STRIP_SUBMODULE_SLASH_CHEAP, prefix, argv); - /* Find common prefix for all pathspec's */ - max_prefix = common_prefix(pathspec); - max_prefix_len = max_prefix ? strlen(max_prefix) : 0; - if (pathspec->nr) ps_matched = xcalloc(pathspec->nr, 1); @@@ -242,9 -42,9 +242,9 @@@ for (i = 0; i < active_nr; i++) { const struct cache_entry *ce = active_cache[i]; - if (!S_ISGITLINK(ce->ce_mode) || - !match_pathspec(pathspec, ce->name, ce_namelen(ce), - max_prefix_len, ps_matched, 1)) + if (!match_pathspec(pathspec, ce->name, ce_namelen(ce), + 0, ps_matched, 1) || + !S_ISGITLINK(ce->ce_mode)) continue; ALLOC_GROW(list->entries, list->nr + 1, list->alloc); @@@ -257,6 -57,7 +257,6 @@@ */ i++; } - free(max_prefix); if (ps_matched && report_path_error(ps_matched, pathspec, prefix)) result = -1; @@@ -305,125 -106,6 +305,125 @@@ static int module_list(int argc, const return 0; } +static void init_submodule(const char *path, const char *prefix, int quiet) +{ + const struct submodule *sub; + struct strbuf sb = STRBUF_INIT; + char *upd = NULL, *url = NULL, *displaypath; + + /* Only loads from .gitmodules, no overlay with .git/config */ + gitmodules_config(); + + if (prefix) { + strbuf_addf(&sb, "%s%s", prefix, path); + displaypath = strbuf_detach(&sb, NULL); + } else + displaypath = xstrdup(path); + + sub = submodule_from_path(null_sha1, path); + + if (!sub) + die(_("No url found for submodule path '%s' in .gitmodules"), + displaypath); + + /* + * Copy url setting when it is not set yet. + * To look up the url in .git/config, we must not fall back to + * .gitmodules, so look it up directly. + */ + strbuf_reset(&sb); + strbuf_addf(&sb, "submodule.%s.url", sub->name); + if (git_config_get_string(sb.buf, &url)) { + url = xstrdup(sub->url); + + if (!url) + die(_("No url found for submodule path '%s' in .gitmodules"), + displaypath); + + /* Possibly a url relative to parent */ + if (starts_with_dot_dot_slash(url) || + starts_with_dot_slash(url)) { + char *remoteurl, *relurl; + char *remote = get_default_remote(); + struct strbuf remotesb = STRBUF_INIT; + strbuf_addf(&remotesb, "remote.%s.url", remote); + free(remote); + + if (git_config_get_string(remotesb.buf, &remoteurl)) + /* + * The repository is its own + * authoritative upstream + */ + remoteurl = xgetcwd(); + relurl = relative_url(remoteurl, url, NULL); + strbuf_release(&remotesb); + free(remoteurl); + free(url); + url = relurl; + } + + if (git_config_set_gently(sb.buf, url)) + die(_("Failed to register url for submodule path '%s'"), + displaypath); + if (!quiet) + fprintf(stderr, + _("Submodule '%s' (%s) registered for path '%s'\n"), + sub->name, url, displaypath); + } + + /* Copy "update" setting when it is not set yet */ + strbuf_reset(&sb); + strbuf_addf(&sb, "submodule.%s.update", sub->name); + if (git_config_get_string(sb.buf, &upd) && + sub->update_strategy.type != SM_UPDATE_UNSPECIFIED) { + if (sub->update_strategy.type == SM_UPDATE_COMMAND) { + fprintf(stderr, _("warning: command update mode suggested for submodule '%s'\n"), + sub->name); + upd = xstrdup("none"); + } else + upd = xstrdup(submodule_strategy_to_string(&sub->update_strategy)); + + if (git_config_set_gently(sb.buf, upd)) + die(_("Failed to register update mode for submodule path '%s'"), displaypath); + } + strbuf_release(&sb); + free(displaypath); + free(url); + free(upd); +} + +static int module_init(int argc, const char **argv, const char *prefix) +{ + struct pathspec pathspec; + struct module_list list = MODULE_LIST_INIT; + int quiet = 0; + int i; + + struct option module_init_options[] = { + OPT_STRING(0, "prefix", &prefix, + N_("path"), + N_("alternative anchor for relative paths")), + OPT__QUIET(&quiet, N_("Suppress output for initializing a submodule")), + OPT_END() + }; + + const char *const git_submodule_helper_usage[] = { + N_("git submodule--helper init []"), + NULL + }; + + argc = parse_options(argc, argv, prefix, module_init_options, + git_submodule_helper_usage, 0); + + if (module_list_compute(argc, argv, prefix, &pathspec, &list) < 0) + return 1; + + for (i = 0; i < list.nr; i++) + init_submodule(list.entries[i]->name, prefix, quiet); + + return 0; +} + static int module_name(int argc, const char **argv, const char *prefix) { const struct submodule *sub; @@@ -443,54 -125,6 +443,6 @@@ return 0; } - /* - * Rules to sanitize configuration variables that are Ok to be passed into - * submodule operations from the parent project using "-c". Should only - * include keys which are both (a) safe and (b) necessary for proper - * operation. - */ - static int submodule_config_ok(const char *var) - { - if (starts_with(var, "credential.")) - return 1; - return 0; - } - - static int sanitize_submodule_config(const char *var, const char *value, void *data) - { - struct strbuf *out = data; - - if (submodule_config_ok(var)) { - if (out->len) - strbuf_addch(out, ' '); - - if (value) - sq_quotef(out, "%s=%s", var, value); - else - sq_quote_buf(out, var); - } - - return 0; - } - - static void prepare_submodule_repo_env(struct argv_array *out) - { - const char * const *var; - - for (var = local_repo_env; *var; var++) { - if (!strcmp(*var, CONFIG_DATA_ENVIRONMENT)) { - struct strbuf sanitized_config = STRBUF_INIT; - git_config_from_parameters(sanitize_submodule_config, - &sanitized_config); - argv_array_pushf(out, "%s=%s", *var, sanitized_config.buf); - strbuf_release(&sanitized_config); - } else { - argv_array_push(out, *var); - } - } - - } - static int clone_submodule(const char *path, const char *gitdir, const char *url, const char *depth, const char *reference, int quiet) { @@@ -520,11 -154,11 +472,11 @@@ static int module_clone(int argc, const char **argv, const char *prefix) { - const char *path = NULL, *name = NULL, *url = NULL; + const char *name = NULL, *url = NULL; const char *reference = NULL, *depth = NULL; int quiet = 0; FILE *submodule_dot_git; - char *sm_gitdir, *cwd, *p; + char *p, *path = NULL, *sm_gitdir; struct strbuf rel_path = STRBUF_INIT; struct strbuf sb = STRBUF_INIT; @@@ -561,19 -195,12 +513,19 @@@ argc = parse_options(argc, argv, prefix, module_clone_options, git_submodule_helper_usage, 0); - if (argc || !url || !path) + if (argc || !url || !path || !*path) usage_with_options(git_submodule_helper_usage, module_clone_options); strbuf_addf(&sb, "%s/modules/%s", get_git_dir(), name); - sm_gitdir = strbuf_detach(&sb, NULL); + sm_gitdir = xstrdup(absolute_path(sb.buf)); + strbuf_reset(&sb); + + if (!is_absolute_path(path)) { + strbuf_addf(&sb, "%s/%s", get_git_work_tree(), path); + path = strbuf_detach(&sb, NULL); + } else + path = xstrdup(path); if (!file_exists(sm_gitdir)) { if (safe_create_leading_directories_const(sm_gitdir) < 0) @@@ -590,313 -217,49 +542,297 @@@ } /* Write a .git file in the submodule to redirect to the superproject. */ - if (safe_create_leading_directories_const(path) < 0) - die(_("could not create directory '%s'"), path); - - if (path && *path) - strbuf_addf(&sb, "%s/.git", path); - else - strbuf_addstr(&sb, ".git"); - + strbuf_addf(&sb, "%s/.git", path); if (safe_create_leading_directories_const(sb.buf) < 0) die(_("could not create leading directories of '%s'"), sb.buf); submodule_dot_git = fopen(sb.buf, "w"); if (!submodule_dot_git) die_errno(_("cannot open file '%s'"), sb.buf); - fprintf(submodule_dot_git, "gitdir: %s\n", - relative_path(sm_gitdir, path, &rel_path)); + fprintf_or_die(submodule_dot_git, "gitdir: %s\n", + relative_path(sm_gitdir, path, &rel_path)); if (fclose(submodule_dot_git)) die(_("could not close file %s"), sb.buf); strbuf_reset(&sb); strbuf_reset(&rel_path); - cwd = xgetcwd(); /* Redirect the worktree of the submodule in the superproject's config */ - if (!is_absolute_path(sm_gitdir)) { - strbuf_addf(&sb, "%s/%s", cwd, sm_gitdir); - free(sm_gitdir); - sm_gitdir = strbuf_detach(&sb, NULL); - } - - strbuf_addf(&sb, "%s/%s", cwd, path); p = git_pathdup_submodule(path, "config"); if (!p) die(_("could not get submodule directory for '%s'"), path); git_config_set_in_file(p, "core.worktree", - relative_path(sb.buf, sm_gitdir, &rel_path)); + relative_path(path, sm_gitdir, &rel_path)); strbuf_release(&sb); strbuf_release(&rel_path); free(sm_gitdir); - free(cwd); + free(path); free(p); return 0; } - static int module_sanitize_config(int argc, const char **argv, const char *prefix) - { - struct strbuf sanitized_config = STRBUF_INIT; - - if (argc > 1) - usage(_("git submodule--helper sanitize-config")); - - git_config_from_parameters(sanitize_submodule_config, &sanitized_config); - if (sanitized_config.len) - printf("%s\n", sanitized_config.buf); - - strbuf_release(&sanitized_config); - - return 0; - } - +struct submodule_update_clone { + /* index into 'list', the list of submodules to look into for cloning */ + int current; + struct module_list list; + unsigned warn_if_uninitialized : 1; + + /* update parameter passed via commandline */ + struct submodule_update_strategy update; + + /* configuration parameters which are passed on to the children */ + int quiet; + const char *reference; + const char *depth; + const char *recursive_prefix; + const char *prefix; + + /* Machine-readable status lines to be consumed by git-submodule.sh */ + struct string_list projectlines; + + /* If we want to stop as fast as possible and return an error */ + unsigned quickstop : 1; +}; +#define SUBMODULE_UPDATE_CLONE_INIT {0, MODULE_LIST_INIT, 0, \ + SUBMODULE_UPDATE_STRATEGY_INIT, 0, NULL, NULL, NULL, NULL, \ + STRING_LIST_INIT_DUP, 0} + + +static void next_submodule_warn_missing(struct submodule_update_clone *suc, + struct strbuf *out, const char *displaypath) +{ + /* + * Only mention uninitialized submodules when their + * paths have been specified. + */ + if (suc->warn_if_uninitialized) { + strbuf_addf(out, + _("Submodule path '%s' not initialized"), + displaypath); + strbuf_addch(out, '\n'); + strbuf_addstr(out, + _("Maybe you want to use 'update --init'?")); + strbuf_addch(out, '\n'); + } +} + +/** + * Determine whether 'ce' needs to be cloned. If so, prepare the 'child' to + * run the clone. Returns 1 if 'ce' needs to be cloned, 0 otherwise. + */ +static int prepare_to_clone_next_submodule(const struct cache_entry *ce, + struct child_process *child, + struct submodule_update_clone *suc, + struct strbuf *out) +{ + const struct submodule *sub = NULL; + struct strbuf displaypath_sb = STRBUF_INIT; + struct strbuf sb = STRBUF_INIT; + const char *displaypath = NULL; + char *url = NULL; + int needs_cloning = 0; + + if (ce_stage(ce)) { + if (suc->recursive_prefix) + strbuf_addf(&sb, "%s/%s", suc->recursive_prefix, ce->name); + else + strbuf_addf(&sb, "%s", ce->name); + strbuf_addf(out, _("Skipping unmerged submodule %s"), sb.buf); + strbuf_addch(out, '\n'); + goto cleanup; + } + + sub = submodule_from_path(null_sha1, ce->name); + + if (suc->recursive_prefix) + displaypath = relative_path(suc->recursive_prefix, + ce->name, &displaypath_sb); + else + displaypath = ce->name; + + if (!sub) { + next_submodule_warn_missing(suc, out, displaypath); + goto cleanup; + } + + if (suc->update.type == SM_UPDATE_NONE + || (suc->update.type == SM_UPDATE_UNSPECIFIED + && sub->update_strategy.type == SM_UPDATE_NONE)) { + strbuf_addf(out, _("Skipping submodule '%s'"), displaypath); + strbuf_addch(out, '\n'); + goto cleanup; + } + + /* + * Looking up the url in .git/config. + * We must not fall back to .gitmodules as we only want + * to process configured submodules. + */ + strbuf_reset(&sb); + strbuf_addf(&sb, "submodule.%s.url", sub->name); + git_config_get_string(sb.buf, &url); + if (!url) { + next_submodule_warn_missing(suc, out, displaypath); + goto cleanup; + } + + strbuf_reset(&sb); + strbuf_addf(&sb, "%s/.git", ce->name); + needs_cloning = !file_exists(sb.buf); + + strbuf_reset(&sb); + strbuf_addf(&sb, "%06o %s %d %d\t%s\n", ce->ce_mode, + sha1_to_hex(ce->sha1), ce_stage(ce), + needs_cloning, ce->name); + string_list_append(&suc->projectlines, sb.buf); + + if (!needs_cloning) + goto cleanup; + + child->git_cmd = 1; + child->no_stdin = 1; + child->stdout_to_stderr = 1; + child->err = -1; + argv_array_push(&child->args, "submodule--helper"); + argv_array_push(&child->args, "clone"); + if (suc->quiet) + argv_array_push(&child->args, "--quiet"); + if (suc->prefix) + argv_array_pushl(&child->args, "--prefix", suc->prefix, NULL); + argv_array_pushl(&child->args, "--path", sub->path, NULL); + argv_array_pushl(&child->args, "--name", sub->name, NULL); + argv_array_pushl(&child->args, "--url", url, NULL); + if (suc->reference) + argv_array_push(&child->args, suc->reference); + if (suc->depth) + argv_array_push(&child->args, suc->depth); + +cleanup: + free(url); + strbuf_reset(&displaypath_sb); + strbuf_reset(&sb); + + return needs_cloning; +} + +static int update_clone_get_next_task(struct child_process *child, + struct strbuf *err, + void *suc_cb, + void **void_task_cb) +{ + struct submodule_update_clone *suc = suc_cb; + + for (; suc->current < suc->list.nr; suc->current++) { + const struct cache_entry *ce = suc->list.entries[suc->current]; + if (prepare_to_clone_next_submodule(ce, child, suc, err)) { + suc->current++; + return 1; + } + } + return 0; +} + +static int update_clone_start_failure(struct strbuf *err, + void *suc_cb, + void *void_task_cb) +{ + struct submodule_update_clone *suc = suc_cb; + suc->quickstop = 1; + return 1; +} + +static int update_clone_task_finished(int result, + struct strbuf *err, + void *suc_cb, + void *void_task_cb) +{ + struct submodule_update_clone *suc = suc_cb; + + if (!result) + return 0; + + suc->quickstop = 1; + return 1; +} + +static int update_clone(int argc, const char **argv, const char *prefix) +{ + const char *update = NULL; + int max_jobs = -1; + struct string_list_item *item; + struct pathspec pathspec; + struct submodule_update_clone suc = SUBMODULE_UPDATE_CLONE_INIT; + + struct option module_update_clone_options[] = { + OPT_STRING(0, "prefix", &prefix, + N_("path"), + N_("path into the working tree")), + OPT_STRING(0, "recursive-prefix", &suc.recursive_prefix, + N_("path"), + N_("path into the working tree, across nested " + "submodule boundaries")), + OPT_STRING(0, "update", &update, + N_("string"), + N_("rebase, merge, checkout or none")), + OPT_STRING(0, "reference", &suc.reference, N_("repo"), + N_("reference repository")), + OPT_STRING(0, "depth", &suc.depth, "", + N_("Create a shallow clone truncated to the " + "specified number of revisions")), + OPT_INTEGER('j', "jobs", &max_jobs, + N_("parallel jobs")), + OPT__QUIET(&suc.quiet, N_("don't print cloning progress")), + OPT_END() + }; + + const char *const git_submodule_helper_usage[] = { + N_("git submodule--helper update_clone [--prefix=] [...]"), + NULL + }; + suc.prefix = prefix; + + argc = parse_options(argc, argv, prefix, module_update_clone_options, + git_submodule_helper_usage, 0); + + if (update) + if (parse_submodule_update_strategy(update, &suc.update) < 0) + die(_("bad value for update parameter")); + + if (module_list_compute(argc, argv, prefix, &pathspec, &suc.list) < 0) + return 1; + + if (pathspec.nr) + suc.warn_if_uninitialized = 1; + + /* Overlay the parsed .gitmodules file with .git/config */ + gitmodules_config(); + git_config(submodule_config, NULL); + + if (max_jobs < 0) + max_jobs = parallel_submodules(); + + run_processes_parallel(max_jobs, + update_clone_get_next_task, + update_clone_start_failure, + update_clone_task_finished, + &suc); + + /* + * We saved the output and put it out all at once now. + * That means: + * - the listener does not have to interleave their (checkout) + * work with our fetching. The writes involved in a + * checkout involve more straightforward sequential I/O. + * - the listener can avoid doing any work if fetching failed. + */ + if (suc.quickstop) + return 1; + + for_each_string_list_item(item, &suc.projectlines) + utf8_fprintf(stdout, "%s", item->string); + + return 0; +} + struct cmd_struct { const char *cmd; int (*fn)(int, const char **, const char *); @@@ -906,24 -269,19 +842,23 @@@ static struct cmd_struct commands[] = {"list", module_list}, {"name", module_name}, {"clone", module_clone}, - {"sanitize-config", module_sanitize_config}, + {"update-clone", update_clone}, + {"resolve-relative-url", resolve_relative_url}, + {"resolve-relative-url-test", resolve_relative_url_test}, + {"init", module_init} }; int cmd_submodule__helper(int argc, const char **argv, const char *prefix) { int i; if (argc < 2) - die(_("fatal: submodule--helper subcommand must be " + die(_("submodule--helper subcommand must be " "called with a subcommand")); for (i = 0; i < ARRAY_SIZE(commands); i++) if (!strcmp(argv[1], commands[i].cmd)) return commands[i].fn(argc - 1, argv + 1, prefix); - die(_("fatal: '%s' is not a valid submodule--helper " + die(_("'%s' is not a valid submodule--helper " "subcommand"), argv[1]); } diff --combined git-submodule.sh index 72fa391283,b1c056c715..5a4dec050b --- a/git-submodule.sh +++ b/git-submodule.sh @@@ -8,7 -8,7 +8,7 @@@ dashless=$(basename "$0" | sed -e 's/- USAGE="[--quiet] add [-b ] [-f|--force] [--name ] [--reference ] [--] [] or: $dashless [--quiet] status [--cached] [--recursive] [--] [...] or: $dashless [--quiet] init [--] [...] - or: $dashless [--quiet] deinit [-f|--force] [--] ... + or: $dashless [--quiet] deinit [-f|--force] (--all| [--] ...) or: $dashless [--quiet] update [--init] [--remote] [-N|--no-fetch] [-f|--force] [--checkout|--merge|--rebase] [--reference ] [--recursive] [--] [...] or: $dashless [--quiet] summary [--cached|--files] [--summary-limit ] [commit] [--] [...] or: $dashless [--quiet] foreach [--recursive] @@@ -46,6 -46,79 +46,6 @@@ prefix custom_name= depth= -# The function takes at most 2 arguments. The first argument is the -# URL that navigates to the submodule origin repo. When relative, this URL -# is relative to the superproject origin URL repo. The second up_path -# argument, if specified, is the relative path that navigates -# from the submodule working tree to the superproject working tree. -# -# The output of the function is the origin URL of the submodule. -# -# The output will either be an absolute URL or filesystem path (if the -# superproject origin URL is an absolute URL or filesystem path, -# respectively) or a relative file system path (if the superproject -# origin URL is a relative file system path). -# -# When the output is a relative file system path, the path is either -# relative to the submodule working tree, if up_path is specified, or to -# the superproject working tree otherwise. -resolve_relative_url () -{ - remote=$(get_default_remote) - remoteurl=$(git config "remote.$remote.url") || - remoteurl=$(pwd) # the repository is its own authoritative upstream - url="$1" - remoteurl=${remoteurl%/} - sep=/ - up_path="$2" - - case "$remoteurl" in - *:*|/*) - is_relative= - ;; - ./*|../*) - is_relative=t - ;; - *) - is_relative=t - remoteurl="./$remoteurl" - ;; - esac - - while test -n "$url" - do - case "$url" in - ../*) - url="${url#../}" - case "$remoteurl" in - */*) - remoteurl="${remoteurl%/*}" - ;; - *:*) - remoteurl="${remoteurl%:*}" - sep=: - ;; - *) - if test -z "$is_relative" || test "." = "$remoteurl" - then - die "$(eval_gettext "cannot strip one component off url '\$remoteurl'")" - else - remoteurl=. - fi - ;; - esac - ;; - ./*) - url="${url#./}" - ;; - *) - break;; - esac - done - remoteurl="$remoteurl$sep${url%/}" - echo "${is_relative:+${up_path}}${remoteurl#./}" -} - # Resolve a path to be relative to another path. This is intended for # converting submodule paths when git-submodule is run in a subdirectory # and only handles paths where the directory separator is '/'. @@@ -124,9 -197,10 +124,10 @@@ isnumber( # of the settings from GIT_CONFIG_PARAMETERS. sanitize_submodule_env() { - sanitized_config=$(git submodule--helper sanitize-config) + save_config=$GIT_CONFIG_PARAMETERS clear_local_git_env - GIT_CONFIG_PARAMETERS=$sanitized_config + GIT_CONFIG_PARAMETERS=$save_config + export GIT_CONFIG_PARAMETERS } # @@@ -218,7 -292,7 +219,7 @@@ cmd_add( die "$(gettext "Relative path can only be used from the toplevel of the working tree")" # dereference source url relative to parent's url - realrepo=$(resolve_relative_url "$repo") || exit + realrepo=$(git submodule--helper resolve-relative-url "$repo") || exit ;; *:*|/*) # absolute url @@@ -350,8 -424,8 +351,8 @@@ cmd_foreach( die_if_unmatched "$mode" if test -e "$sm_path"/.git then - displaypath=$(relative_path "$sm_path") - say "$(eval_gettext "Entering '\$prefix\$displaypath'")" + displaypath=$(relative_path "$prefix$sm_path") + say "$(eval_gettext "Entering '\$displaypath'")" name=$(git submodule--helper name "$sm_path") ( prefix="$prefix$sm_path/" @@@ -371,7 -445,7 +372,7 @@@ cmd_foreach "--recursive" "$@" fi ) <&3 3<&- || - die "$(eval_gettext "Stopping at '\$prefix\$displaypath'; script returned non-zero status.")" + die "$(eval_gettext "Stopping at '\$displaypath'; script returned non-zero status.")" fi done } @@@ -404,7 -478,50 +405,7 @@@ cmd_init( shift done - git submodule--helper list --prefix "$wt_prefix" "$@" | - while read mode sha1 stage sm_path - do - die_if_unmatched "$mode" - name=$(git submodule--helper name "$sm_path") || exit - - displaypath=$(relative_path "$sm_path") - - # Copy url setting when it is not set yet - if test -z "$(git config "submodule.$name.url")" - then - url=$(git config -f .gitmodules submodule."$name".url) - test -z "$url" && - die "$(eval_gettext "No url found for submodule path '\$displaypath' in .gitmodules")" - - # Possibly a url relative to parent - case "$url" in - ./*|../*) - url=$(resolve_relative_url "$url") || exit - ;; - esac - git config submodule."$name".url "$url" || - die "$(eval_gettext "Failed to register url for submodule path '\$displaypath'")" - - say "$(eval_gettext "Submodule '\$name' (\$url) registered for path '\$displaypath'")" - fi - - # Copy "update" setting when it is not set yet - if upd="$(git config -f .gitmodules submodule."$name".update)" && - test -n "$upd" && - test -z "$(git config submodule."$name".update)" - then - case "$upd" in - checkout | rebase | merge | none) - ;; # known modes of updating - *) - echo >&2 "warning: unknown update mode '$upd' suggested for submodule '$name'" - upd=none - ;; - esac - git config submodule."$name".update "$upd" || - die "$(eval_gettext "Failed to register update mode for submodule path '\$displaypath'")" - fi - done + git ${wt_prefix:+-C "$wt_prefix"} submodule--helper init ${GIT_QUIET:+--quiet} ${prefix:+--prefix "$prefix"} "$@" } # @@@ -415,7 -532,6 +416,7 @@@ cmd_deinit() { # parse $args after "submodule ... deinit". + deinit_all= while test $# -ne 0 do case "$1" in @@@ -425,9 -541,6 +426,9 @@@ -q|--quiet) GIT_QUIET=1 ;; + --all) + deinit_all=t + ;; --) shift break @@@ -442,14 -555,9 +443,14 @@@ shift done - if test $# = 0 + if test -n "$deinit_all" && test "$#" -ne 0 then - die "$(eval_gettext "Use '.' if you really want to deinitialize all submodules")" + echo >&2 "$(eval_gettext "pathspec and --all are incompatible")" + usage + fi + if test $# = 0 && test -z "$deinit_all" + then + die "$(eval_gettext "Use '--all' if you really want to deinitialize all submodules")" fi git submodule--helper list --prefix "$wt_prefix" "$@" | @@@ -494,24 -602,6 +495,24 @@@ done } +is_tip_reachable () ( + sanitize_submodule_env && + cd "$1" && + rev=$(git rev-list -n 1 "$2" --not --all 2>/dev/null) && + test -z "$rev" +) + +fetch_in_submodule () ( + sanitize_submodule_env && + cd "$1" && + case "$2" in + '') + git fetch ;; + *) + git fetch $(get_default_remote) "$2" ;; + esac +) + # # Update each submodule path to correct revision, using clone and checkout as needed # @@@ -566,14 -656,6 +567,14 @@@ cmd_update( --depth=*) depth=$1 ;; + -j|--jobs) + case "$2" in '') usage ;; esac + jobs="--jobs=$2" + shift + ;; + --jobs=*) + jobs=$1 + ;; --) shift break @@@ -593,21 -675,17 +594,21 @@@ cmd_init "--" "$@" || return fi - cloned_modules= - git submodule--helper list --prefix "$wt_prefix" "$@" | { + { + git submodule--helper update-clone ${GIT_QUIET:+--quiet} \ + ${wt_prefix:+--prefix "$wt_prefix"} \ + ${prefix:+--recursive-prefix "$prefix"} \ + ${update:+--update "$update"} \ + ${reference:+--reference "$reference"} \ + ${depth:+--depth "$depth"} \ + ${jobs:+$jobs} \ + "$@" || echo "#unmatched" + } | { err= - while read mode sha1 stage sm_path + while read mode sha1 stage just_cloned sm_path do die_if_unmatched "$mode" - if test "$stage" = U - then - echo >&2 "Skipping unmerged submodule $prefix$sm_path" - continue - fi + name=$(git submodule--helper name "$sm_path") || exit url=$(git config submodule."$name".url) branch=$(get_submodule_config "$name" branch master) @@@ -624,10 -702,27 +625,10 @@@ displaypath=$(relative_path "$prefix$sm_path") - if test "$update_module" = "none" + if test $just_cloned -eq 1 then - echo "Skipping submodule '$displaypath'" - continue - fi - - if test -z "$url" - then - # Only mention uninitialized submodules when its - # path have been specified - test "$#" != "0" && - say "$(eval_gettext "Submodule path '\$displaypath' not initialized -Maybe you want to use 'update --init'?")" - continue - fi - - if ! test -d "$sm_path"/.git && ! test -f "$sm_path"/.git - then - git submodule--helper clone ${GIT_QUIET:+--quiet} --prefix "$prefix" --path "$sm_path" --name "$name" --url "$url" ${reference:+"$reference"} ${depth:+"$depth"} || exit - cloned_modules="$cloned_modules;$name" subsha1= + update_module=checkout else subsha1=$(sanitize_submodule_env; cd "$sm_path" && git rev-parse --verify HEAD) || @@@ -661,16 -756,18 +662,16 @@@ then # Run fetch only if $sha1 isn't present or it # is not reachable from a ref. - (sanitize_submodule_env; cd "$sm_path" && - ( (rev=$(git rev-list -n 1 $sha1 --not --all 2>/dev/null) && - test -z "$rev") || git-fetch)) || + is_tip_reachable "$sm_path" "$sha1" || + fetch_in_submodule "$sm_path" || die "$(eval_gettext "Unable to fetch in submodule path '\$displaypath'")" - fi - # Is this something we just cloned? - case ";$cloned_modules;" in - *";$name;"*) - # then there is no local change to integrate - update_module=checkout ;; - esac + # Now we tried the usual fetch, but $sha1 may + # not be reachable from any of the refs + is_tip_reachable "$sm_path" "$sha1" || + fetch_in_submodule "$sm_path" "$sha1" || + die "$(eval_gettext "Fetched in submodule path '\$displaypath', but it did not contain $sha1. Direct fetching of that commit failed.")" + fi must_die_on_failure= case "$update_module" in @@@ -693,8 -790,8 +694,8 @@@ ;; !*) command="${update_module#!}" - die_msg="$(eval_gettext "Execution of '\$command \$sha1' failed in submodule path '\$prefix\$sm_path'")" - say_msg="$(eval_gettext "Submodule path '\$prefix\$sm_path': '\$command \$sha1'")" + die_msg="$(eval_gettext "Execution of '\$command \$sha1' failed in submodule path '\$displaypath'")" + say_msg="$(eval_gettext "Submodule path '\$displaypath': '\$command \$sha1'")" must_die_on_failure=yes ;; *) @@@ -716,8 -813,7 +717,8 @@@ if test -n "$recursive" then ( - prefix="$prefix$sm_path/" + prefix=$(relative_path "$prefix$sm_path/") + wt_prefix= sanitize_submodule_env cd "$sm_path" && eval cmd_update @@@ -1051,7 -1147,6 +1052,7 @@@ cmd_status( ( prefix="$displaypath/" sanitize_submodule_env + wt_prefix= cd "$sm_path" && eval cmd_status ) || @@@ -1106,9 -1201,9 +1107,9 @@@ cmd_sync( # guarantee a trailing / up_path=${up_path%/}/ && # path from submodule work tree to submodule origin repo - sub_origin_url=$(resolve_relative_url "$url" "$up_path") && + sub_origin_url=$(git submodule--helper resolve-relative-url "$url" "$up_path") && # path from superproject work tree to submodule origin repo - super_config_url=$(resolve_relative_url "$url") || exit + super_config_url=$(git submodule--helper resolve-relative-url "$url") || exit ;; *) sub_origin_url="$url" diff --combined submodule.c index 4cc1c27931,d598881114..4532b11d66 --- a/submodule.c +++ b/submodule.c @@@ -13,9 -13,9 +13,10 @@@ #include "argv-array.h" #include "blob.h" #include "thread-utils.h" + #include "quote.h" static int config_fetch_recurse_submodules = RECURSE_SUBMODULES_ON_DEMAND; +static int parallel_jobs = 1; static struct string_list changed_submodule_paths; static int initialized_fetch_ref_tips; static struct sha1_array ref_tips_before_fetch; @@@ -70,7 -70,7 +71,7 @@@ int update_path_in_gitmodules(const cha strbuf_addstr(&entry, "submodule."); strbuf_addstr(&entry, submodule->name); strbuf_addstr(&entry, ".path"); - if (git_config_set_in_file(".gitmodules", entry.buf, newpath) < 0) { + if (git_config_set_in_file_gently(".gitmodules", entry.buf, newpath) < 0) { /* Maybe the user already did that, don't error out here */ warning(_("Could not update .gitmodules entry %s"), entry.buf); strbuf_release(&entry); @@@ -124,7 -124,7 +125,7 @@@ static int add_submodule_odb(const cha struct strbuf objects_directory = STRBUF_INIT; struct alternate_object_database *alt_odb; int ret = 0; - int alloc; + size_t alloc; strbuf_git_path_submodule(&objects_directory, path, "objects/"); if (!is_directory(objects_directory.buf)) { @@@ -139,8 -139,8 +140,8 @@@ objects_directory.len)) goto done; - alloc = objects_directory.len + 42; /* for "12/345..." sha1 */ - alt_odb = xmalloc(sizeof(*alt_odb) + alloc); + alloc = st_add(objects_directory.len, 42); /* for "12/345..." sha1 */ + alt_odb = xmalloc(st_add(sizeof(*alt_odb), alloc)); alt_odb->next = alt_odb_list; xsnprintf(alt_odb->base, alloc, "%s", objects_directory.buf); alt_odb->name = alt_odb->base + objects_directory.len; @@@ -170,12 -170,7 +171,12 @@@ void set_diffopt_flags_from_submodule_c int submodule_config(const char *var, const char *value, void *cb) { - if (starts_with(var, "submodule.")) + if (!strcmp(var, "submodule.fetchjobs")) { + parallel_jobs = git_config_int(var, value); + if (parallel_jobs < 0) + die(_("negative values not allowed for submodule.fetchJobs")); + return 0; + } else if (starts_with(var, "submodule.")) return parse_submodule_config_option(var, value); else if (!strcmp(var, "fetch.recursesubmodules")) { config_fetch_recurse_submodules = parse_fetch_recurse_submodules_arg(var, value); @@@ -216,48 -211,6 +217,48 @@@ void gitmodules_config(void } } +int parse_submodule_update_strategy(const char *value, + struct submodule_update_strategy *dst) +{ + free((void*)dst->command); + dst->command = NULL; + if (!strcmp(value, "none")) + dst->type = SM_UPDATE_NONE; + else if (!strcmp(value, "checkout")) + dst->type = SM_UPDATE_CHECKOUT; + else if (!strcmp(value, "rebase")) + dst->type = SM_UPDATE_REBASE; + else if (!strcmp(value, "merge")) + dst->type = SM_UPDATE_MERGE; + else if (skip_prefix(value, "!", &value)) { + dst->type = SM_UPDATE_COMMAND; + dst->command = xstrdup(value); + } else + return -1; + return 0; +} + +const char *submodule_strategy_to_string(const struct submodule_update_strategy *s) +{ + struct strbuf sb = STRBUF_INIT; + switch (s->type) { + case SM_UPDATE_CHECKOUT: + return "checkout"; + case SM_UPDATE_MERGE: + return "merge"; + case SM_UPDATE_REBASE: + return "rebase"; + case SM_UPDATE_NONE: + return "none"; + case SM_UPDATE_UNSPECIFIED: + return NULL; + case SM_UPDATE_COMMAND: + strbuf_addf(&sb, "!%s", s->command); + return strbuf_detach(&sb, NULL); + } + return NULL; +} + void handle_ignore_submodules_arg(struct diff_options *diffopt, const char *arg) { @@@ -414,7 -367,7 +415,7 @@@ static int submodule_needs_pushing(cons argv[1] = sha1_to_hex(sha1); cp.argv = argv; - cp.env = local_repo_env; + prepare_submodule_repo_env(&cp.env_array); cp.git_cmd = 1; cp.no_stdin = 1; cp.out = -1; @@@ -501,7 -454,7 +502,7 @@@ static int push_submodule(const char *p const char *argv[] = {"push", NULL}; cp.argv = argv; - cp.env = local_repo_env; + prepare_submodule_repo_env(&cp.env_array); cp.git_cmd = 1; cp.no_stdin = 1; cp.dir = path; @@@ -547,7 -500,7 +548,7 @@@ static int is_submodule_commit_present( argv[3] = sha1_to_hex(sha1); cp.argv = argv; - cp.env = local_repo_env; + prepare_submodule_repo_env(&cp.env_array); cp.git_cmd = 1; cp.no_stdin = 1; cp.dir = path; @@@ -730,7 -683,7 +731,7 @@@ static int get_next_submodule(struct ch if (is_directory(git_dir)) { child_process_init(cp); cp->dir = strbuf_detach(&submodule_path, NULL); - cp->env = local_repo_env; + prepare_submodule_repo_env(&cp->env_array); cp->git_cmd = 1; if (!spf->quiet) strbuf_addf(err, "Fetching submodule %s%s\n", @@@ -753,7 -706,8 +754,7 @@@ return 0; } -static int fetch_start_failure(struct child_process *cp, - struct strbuf *err, +static int fetch_start_failure(struct strbuf *err, void *cb, void *task_cb) { struct submodule_parallel_fetch *spf = cb; @@@ -763,8 -717,8 +764,8 @@@ return 0; } -static int fetch_finish(int retvalue, struct child_process *cp, - struct strbuf *err, void *cb, void *task_cb) +static int fetch_finish(int retvalue, struct strbuf *err, + void *cb, void *task_cb) { struct submodule_parallel_fetch *spf = cb; @@@ -798,9 -752,6 +799,9 @@@ int fetch_populated_submodules(const st argv_array_push(&spf.args, "--recurse-submodules-default"); /* default value, "--submodule-prefix" and its value are added later */ + if (max_parallel_jobs < 0) + max_parallel_jobs = parallel_jobs; + calculate_changed_submodule_paths(); run_processes_parallel(max_parallel_jobs, get_next_submodule, @@@ -845,7 -796,7 +846,7 @@@ unsigned is_submodule_modified(const ch argv[2] = "-uno"; cp.argv = argv; - cp.env = local_repo_env; + prepare_submodule_repo_env(&cp.env_array); cp.git_cmd = 1; cp.no_stdin = 1; cp.out = -1; @@@ -906,7 -857,7 +907,7 @@@ int submodule_uses_gitfile(const char * /* Now test that all nested submodules use a gitfile too */ cp.argv = argv; - cp.env = local_repo_env; + prepare_submodule_repo_env(&cp.env_array); cp.git_cmd = 1; cp.no_stdin = 1; cp.no_stderr = 1; @@@ -939,7 -890,7 +940,7 @@@ int ok_to_remove_submodule(const char * return 0; cp.argv = argv; - cp.env = local_repo_env; + prepare_submodule_repo_env(&cp.env_array); cp.git_cmd = 1; cp.no_stdin = 1; cp.out = -1; @@@ -1137,16 -1088,23 +1138,26 @@@ void connect_work_tree_and_git_dir(cons /* Update core.worktree setting */ strbuf_reset(&file_name); strbuf_addf(&file_name, "%s/config", git_dir); - if (git_config_set_in_file(file_name.buf, "core.worktree", - relative_path(real_work_tree, git_dir, - &rel_path))) - die(_("Could not set core.worktree in %s"), - file_name.buf); + git_config_set_in_file(file_name.buf, "core.worktree", + relative_path(real_work_tree, git_dir, + &rel_path)); strbuf_release(&file_name); strbuf_release(&rel_path); free((void *)real_work_tree); } +int parallel_submodules(void) +{ + return parallel_jobs; +} ++ + void prepare_submodule_repo_env(struct argv_array *out) + { + const char * const *var; + + for (var = local_repo_env; *var; var++) { + if (strcmp(*var, CONFIG_DATA_ENVIRONMENT)) + argv_array_push(out, *var); + } + } diff --combined submodule.h index ff4c4f33a5,869d259fac..2af9390998 --- a/submodule.h +++ b/submodule.h @@@ -14,21 -14,6 +14,21 @@@ enum RECURSE_SUBMODULES_ON = 2 }; +enum submodule_update_type { + SM_UPDATE_UNSPECIFIED = 0, + SM_UPDATE_CHECKOUT, + SM_UPDATE_REBASE, + SM_UPDATE_MERGE, + SM_UPDATE_NONE, + SM_UPDATE_COMMAND +}; + +struct submodule_update_strategy { + enum submodule_update_type type; + const char *command; +}; +#define SUBMODULE_UPDATE_STRATEGY_INIT {SM_UPDATE_UNSPECIFIED, NULL} + int is_staging_gitmodules_ok(void); int update_path_in_gitmodules(const char *oldpath, const char *newpath); int remove_path_from_gitmodules(const char *path); @@@ -37,9 -22,6 +37,9 @@@ void set_diffopt_flags_from_submodule_c const char *path); int submodule_config(const char *var, const char *value, void *cb); void gitmodules_config(void); +int parse_submodule_update_strategy(const char *value, + struct submodule_update_strategy *dst); +const char *submodule_strategy_to_string(const struct submodule_update_strategy *s); void handle_ignore_submodules_arg(struct diff_options *diffopt, const char *); void show_submodule_summary(FILE *f, const char *path, const char *line_prefix, @@@ -60,6 -42,12 +60,13 @@@ int find_unpushed_submodules(unsigned c struct string_list *needs_pushing); int push_unpushed_submodules(unsigned char new_sha1[20], const char *remotes_name); void connect_work_tree_and_git_dir(const char *work_tree, const char *git_dir); +int parallel_submodules(void); + /* + * Prepare the "env_array" parameter of a "struct child_process" for executing + * a submodule by clearing any repo-specific envirionment variables, but + * retaining any config in the environment. + */ + void prepare_submodule_repo_env(struct argv_array *out); + #endif