From: Junio C Hamano Date: Thu, 25 Jul 2019 20:59:22 +0000 (-0700) Subject: Merge branch 'mt/dir-iterator-updates' X-Git-Tag: v2.23.0-rc0~16 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/080af915a3ee4d9511dc288b29143b9958ac0adc?hp=-c Merge branch 'mt/dir-iterator-updates' Adjust the dir-iterator API and apply it to the local clone optimization codepath. * mt/dir-iterator-updates: clone: replace strcmp by fspathcmp clone: use dir-iterator to avoid explicit dir traversal clone: extract function from copy_or_link_directory clone: copy hidden paths at local clone dir-iterator: add flags parameter to dir_iterator_begin dir-iterator: refactor state machine model dir-iterator: use warning_errno when possible dir-iterator: add tests for dir-iterator API clone: better handle symlinked files at .git/objects/ clone: test for our behavior on odd objects/* content --- 080af915a3ee4d9511dc288b29143b9958ac0adc diff --combined Makefile index b11cdd4fe7,61df07f488..bd246f2989 --- a/Makefile +++ b/Makefile @@@ -265,6 -265,10 +265,6 @@@ all: # # Define NO_DEFLATE_BOUND if your zlib does not have deflateBound. # -# Define NO_R_TO_GCC_LINKER if your gcc does not like "-R/path/lib" -# that tells runtime paths to dynamic libraries; -# "-Wl,-rpath=/path/lib" is used instead. -# # Define NO_NORETURN if using buggy versions of gcc 4.6+ and profile feedback, # as the compiler can crash (http://gcc.gnu.org/bugzilla/show_bug.cgi?id=49299) # @@@ -620,6 -624,8 +620,6 @@@ SCRIPT_SH += git-web--browse.s SCRIPT_LIB += git-mergetool--lib SCRIPT_LIB += git-parse-remote -SCRIPT_LIB += git-rebase--am -SCRIPT_LIB += git-rebase--common SCRIPT_LIB += git-rebase--preserve-merges SCRIPT_LIB += git-sh-setup SCRIPT_LIB += git-sh-i18n @@@ -704,6 -710,7 +704,7 @@@ TEST_BUILTINS_OBJS += test-config. TEST_BUILTINS_OBJS += test-ctype.o TEST_BUILTINS_OBJS += test-date.o TEST_BUILTINS_OBJS += test-delta.o + TEST_BUILTINS_OBJS += test-dir-iterator.o TEST_BUILTINS_OBJS += test-drop-caches.o TEST_BUILTINS_OBJS += test-dump-cache-tree.o TEST_BUILTINS_OBJS += test-dump-fsmonitor.o @@@ -721,7 -728,6 +722,7 @@@ TEST_BUILTINS_OBJS += test-lazy-init-na TEST_BUILTINS_OBJS += test-match-trees.o TEST_BUILTINS_OBJS += test-mergesort.o TEST_BUILTINS_OBJS += test-mktemp.o +TEST_BUILTINS_OBJS += test-oidmap.o TEST_BUILTINS_OBJS += test-online-cpus.o TEST_BUILTINS_OBJS += test-parse-options.o TEST_BUILTINS_OBJS += test-path-utils.o @@@ -772,11 -778,9 +773,11 @@@ BUILT_INS += git-format-patch$ BUILT_INS += git-fsck-objects$X BUILT_INS += git-init$X BUILT_INS += git-merge-subtree$X +BUILT_INS += git-restore$X BUILT_INS += git-show$X BUILT_INS += git-stage$X BUILT_INS += git-status$X +BUILT_INS += git-switch$X BUILT_INS += git-whatchanged$X # what 'all' will build and 'install' will install in gitexecdir, @@@ -1062,7 -1066,6 +1063,7 @@@ BUILTIN_OBJS += builtin/diff-index. BUILTIN_OBJS += builtin/diff-tree.o BUILTIN_OBJS += builtin/diff.o BUILTIN_OBJS += builtin/difftool.o +BUILTIN_OBJS += builtin/env--helper.o BUILTIN_OBJS += builtin/fast-export.o BUILTIN_OBJS += builtin/fetch-pack.o BUILTIN_OBJS += builtin/fetch.o @@@ -1158,7 -1161,6 +1159,7 @@@ endi # which'll override these defaults. CFLAGS = -g -O2 -Wall LDFLAGS = +CC_LD_DYNPATH = -Wl,-rpath, BASIC_CFLAGS = -I. BASIC_LDFLAGS = @@@ -1239,7 -1241,7 +1240,7 @@@ endi ifdef SANE_TOOL_PATH SANE_TOOL_PATH_SQ = $(subst ','\'',$(SANE_TOOL_PATH)) -BROKEN_PATH_FIX = 's|^\# @@BROKEN_PATH_FIX@@$$|git_broken_path_fix $(SANE_TOOL_PATH_SQ)|' +BROKEN_PATH_FIX = 's|^\# @@BROKEN_PATH_FIX@@$$|git_broken_path_fix "$(SANE_TOOL_PATH_SQ)"|' PATH := $(SANE_TOOL_PATH):${PATH} else BROKEN_PATH_FIX = '/^\# @@BROKEN_PATH_FIX@@$$/d' @@@ -1289,6 -1291,16 +1290,6 @@@ ifeq ($(uname_S),Darwin PTHREAD_LIBS = endif -ifndef CC_LD_DYNPATH - ifdef NO_R_TO_GCC_LINKER - # Some gcc does not accept and pass -R to the linker to specify - # the runtime dynamic library path. - CC_LD_DYNPATH = -Wl,-rpath, - else - CC_LD_DYNPATH = -R - endif -endif - ifdef NO_LIBGEN_H COMPAT_CFLAGS += -DNO_LIBGEN_H COMPAT_OBJS += compat/basename.o @@@ -2862,33 -2874,6 +2863,33 @@@ install: al $(INSTALL) $(ALL_PROGRAMS) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) -m 644 $(SCRIPT_LIB) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) $(install_bindir_programs) '$(DESTDIR_SQ)$(bindir_SQ)' +ifdef MSVC + # We DO NOT install the individual foo.o.pdb files because they + # have already been rolled up into the exe's pdb file. + # We DO NOT have pdb files for the builtin commands (like git-status.exe) + # because it is just a copy/hardlink of git.exe, rather than a unique binary. + $(INSTALL) git.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-shell.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-upload-pack.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-credential-store.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-daemon.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-fast-import.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-backend.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-fetch.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-push.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-imap-send.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-remote-http.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-remote-testsvn.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-sh-i18n--envsubst.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-show-index.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' +ifndef DEBUG + $(INSTALL) $(vcpkg_rel_bin)/*.dll '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) $(vcpkg_rel_bin)/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)' +else + $(INSTALL) $(vcpkg_dbg_bin)/*.dll '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) $(vcpkg_dbg_bin)/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)' +endif +endif $(MAKE) -C templates DESTDIR='$(DESTDIR_SQ)' install $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(mergetools_instdir_SQ)' $(INSTALL) -m 644 mergetools/* '$(DESTDIR_SQ)$(mergetools_instdir_SQ)' @@@ -3101,19 -3086,6 +3102,19 @@@ endi $(RM) GIT-VERSION-FILE GIT-CFLAGS GIT-LDFLAGS GIT-BUILD-OPTIONS $(RM) GIT-USER-AGENT GIT-PREFIX $(RM) GIT-SCRIPT-DEFINES GIT-PERL-DEFINES GIT-PERL-HEADER GIT-PYTHON-VARS +ifdef MSVC + $(RM) $(patsubst %.o,%.o.pdb,$(OBJECTS)) + $(RM) $(patsubst %.exe,%.pdb,$(OTHER_PROGRAMS)) + $(RM) $(patsubst %.exe,%.iobj,$(OTHER_PROGRAMS)) + $(RM) $(patsubst %.exe,%.ipdb,$(OTHER_PROGRAMS)) + $(RM) $(patsubst %.exe,%.pdb,$(PROGRAMS)) + $(RM) $(patsubst %.exe,%.iobj,$(PROGRAMS)) + $(RM) $(patsubst %.exe,%.ipdb,$(PROGRAMS)) + $(RM) $(patsubst %.exe,%.pdb,$(TEST_PROGRAMS)) + $(RM) $(patsubst %.exe,%.iobj,$(TEST_PROGRAMS)) + $(RM) $(patsubst %.exe,%.ipdb,$(TEST_PROGRAMS)) + $(RM) compat/vcbuild/MSVC-DEFS-GEN +endif .PHONY: all install profile-clean cocciclean clean strip .PHONY: shell_compatibility_test please_set_SHELL_PATH_to_a_more_modern_shell diff --combined builtin/clone.c index a4fe72879d,62b6a0a352..f665b28ccc --- a/builtin/clone.c +++ b/builtin/clone.c @@@ -23,6 -23,8 +23,8 @@@ #include "transport.h" #include "strbuf.h" #include "dir.h" + #include "dir-iterator.h" + #include "iterator.h" #include "sigchain.h" #include "branch.h" #include "remote.h" @@@ -67,7 -69,6 +69,7 @@@ static int max_jobs = -1 static struct string_list option_recurse_submodules = STRING_LIST_INIT_NODUP; static struct list_objects_filter_options filter_options; static struct string_list server_options = STRING_LIST_INIT_NODUP; +static int option_remote_submodules; static int recurse_submodules_cb(const struct option *opt, const char *arg, int unset) @@@ -143,8 -144,6 +145,8 @@@ static struct option builtin_clone_opti OPT_SET_INT('6', "ipv6", &family, N_("use IPv6 addresses only"), TRANSPORT_FAMILY_IPV6), OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options), + OPT_BOOL(0, "remote-submodules", &option_remote_submodules, + N_("any cloned submodules will use their remote-tracking branch")), OPT_END() }; @@@ -394,50 -393,55 +396,55 @@@ static void copy_alternates(struct strb fclose(in); } + static void mkdir_if_missing(const char *pathname, mode_t mode) + { + struct stat st; + + if (!mkdir(pathname, mode)) + return; + + if (errno != EEXIST) + die_errno(_("failed to create directory '%s'"), pathname); + else if (stat(pathname, &st)) + die_errno(_("failed to stat '%s'"), pathname); + else if (!S_ISDIR(st.st_mode)) + die(_("%s exists and is not a directory"), pathname); + } + static void copy_or_link_directory(struct strbuf *src, struct strbuf *dest, - const char *src_repo, int src_baselen) + const char *src_repo) { - struct dirent *de; - struct stat buf; int src_len, dest_len; - DIR *dir; - - dir = opendir(src->buf); - if (!dir) - die_errno(_("failed to open '%s'"), src->buf); - - if (mkdir(dest->buf, 0777)) { - if (errno != EEXIST) - die_errno(_("failed to create directory '%s'"), dest->buf); - else if (stat(dest->buf, &buf)) - die_errno(_("failed to stat '%s'"), dest->buf); - else if (!S_ISDIR(buf.st_mode)) - die(_("%s exists and is not a directory"), dest->buf); - } + struct dir_iterator *iter; + int iter_status; + unsigned int flags; + + mkdir_if_missing(dest->buf, 0777); + + flags = DIR_ITERATOR_PEDANTIC | DIR_ITERATOR_FOLLOW_SYMLINKS; + iter = dir_iterator_begin(src->buf, flags); + + if (!iter) + die_errno(_("failed to start iterator over '%s'"), src->buf); strbuf_addch(src, '/'); src_len = src->len; strbuf_addch(dest, '/'); dest_len = dest->len; - while ((de = readdir(dir)) != NULL) { + while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) { strbuf_setlen(src, src_len); - strbuf_addstr(src, de->d_name); + strbuf_addstr(src, iter->relative_path); strbuf_setlen(dest, dest_len); - strbuf_addstr(dest, de->d_name); - if (stat(src->buf, &buf)) { - warning (_("failed to stat %s\n"), src->buf); - continue; - } - if (S_ISDIR(buf.st_mode)) { - if (de->d_name[0] != '.') - copy_or_link_directory(src, dest, - src_repo, src_baselen); + strbuf_addstr(dest, iter->relative_path); + + if (S_ISDIR(iter->st.st_mode)) { + mkdir_if_missing(dest->buf, 0777); continue; } /* Files that cannot be copied bit-for-bit... */ - if (!strcmp(src->buf + src_baselen, "/info/alternates")) { + if (!fspathcmp(iter->relative_path, "info/alternates")) { copy_alternates(src, src_repo); continue; } @@@ -445,7 -449,7 +452,7 @@@ if (unlink(dest->buf) && errno != ENOENT) die_errno(_("failed to unlink '%s'"), dest->buf); if (!option_no_hardlinks) { - if (!link(src->buf, dest->buf)) + if (!link(real_path(src->buf), dest->buf)) continue; if (option_local > 0) die_errno(_("failed to create link '%s'"), dest->buf); @@@ -454,7 -458,11 +461,11 @@@ if (copy_file_with_time(dest->buf, src->buf, 0666)) die_errno(_("failed to copy file to '%s'"), dest->buf); } - closedir(dir); + + if (iter_status != ITER_DONE) { + strbuf_setlen(src, src_len); + die(_("failed to iterate over '%s'"), src->buf); + } } static void clone_local(const char *src_repo, const char *dest_repo) @@@ -472,7 -480,7 +483,7 @@@ get_common_dir(&dest, dest_repo); strbuf_addstr(&src, "/objects"); strbuf_addstr(&dest, "/objects"); - copy_or_link_directory(&src, &dest, src_repo, src.len); + copy_or_link_directory(&src, &dest, src_repo); strbuf_release(&src); strbuf_release(&dest); } @@@ -494,7 -502,7 +505,7 @@@ static enum static const char junk_leave_repo_msg[] = N_("Clone succeeded, but checkout failed.\n" "You can inspect what was checked out with 'git status'\n" - "and retry the checkout with 'git checkout -f HEAD'\n"); + "and retry with 'git restore --source=HEAD :/'\n"); static void remove_junk(void) { @@@ -793,11 -801,6 +804,11 @@@ static int checkout(int submodule_progr if (option_verbosity < 0) argv_array_push(&args, "--quiet"); + if (option_remote_submodules) { + argv_array_push(&args, "--remote"); + argv_array_push(&args, "--no-fetch"); + } + err = run_command_v_opt(args.argv, RUN_GIT_CMD); argv_array_clear(&args); } @@@ -1227,7 -1230,7 +1238,7 @@@ int cmd_clone(int argc, const char **ar remote_head_points_at, &branch_top); if (filter_options.choice) - partial_clone_register("origin", &filter_options); + partial_clone_register(option_origin, &filter_options); if (is_local) clone_local(path, git_dir); @@@ -1252,7 -1255,7 +1263,7 @@@ transport_disconnect(transport); if (option_dissociate) { - close_all_packs(the_repository->objects); + close_object_store(the_repository->objects); dissociate_from_references(); } diff --combined t/helper/test-tool.c index 1eac25233f,7bc9bb231e..ce7e89028c --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@@ -19,6 -19,7 +19,7 @@@ static struct test_cmd cmds[] = { "ctype", cmd__ctype }, { "date", cmd__date }, { "delta", cmd__delta }, + { "dir-iterator", cmd__dir_iterator }, { "drop-caches", cmd__drop_caches }, { "dump-cache-tree", cmd__dump_cache_tree }, { "dump-fsmonitor", cmd__dump_fsmonitor }, @@@ -35,7 -36,6 +36,7 @@@ { "match-trees", cmd__match_trees }, { "mergesort", cmd__mergesort }, { "mktemp", cmd__mktemp }, + { "oidmap", cmd__oidmap }, { "online-cpus", cmd__online_cpus }, { "parse-options", cmd__parse_options }, { "path-utils", cmd__path_utils }, diff --combined t/helper/test-tool.h index c7a46dc320,ec0ffbd0cb..f805bb39ae --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@@ -9,6 -9,7 +9,7 @@@ int cmd__config(int argc, const char ** int cmd__ctype(int argc, const char **argv); int cmd__date(int argc, const char **argv); int cmd__delta(int argc, const char **argv); + int cmd__dir_iterator(int argc, const char **argv); int cmd__drop_caches(int argc, const char **argv); int cmd__dump_cache_tree(int argc, const char **argv); int cmd__dump_fsmonitor(int argc, const char **argv); @@@ -25,7 -26,6 +26,7 @@@ int cmd__lazy_init_name_hash(int argc, int cmd__match_trees(int argc, const char **argv); int cmd__mergesort(int argc, const char **argv); int cmd__mktemp(int argc, const char **argv); +int cmd__oidmap(int argc, const char **argv); int cmd__online_cpus(int argc, const char **argv); int cmd__parse_options(int argc, const char **argv); int cmd__path_utils(int argc, const char **argv);