Merge branch 'mt/dir-iterator-updates'
authorJunio C Hamano <gitster@pobox.com>
Thu, 25 Jul 2019 20:59:22 +0000 (13:59 -0700)
committerJunio C Hamano <gitster@pobox.com>
Thu, 25 Jul 2019 20:59:22 +0000 (13:59 -0700)
Adjust the dir-iterator API and apply it to the local clone
optimization codepath.

* mt/dir-iterator-updates:
clone: replace strcmp by fspathcmp
clone: use dir-iterator to avoid explicit dir traversal
clone: extract function from copy_or_link_directory
clone: copy hidden paths at local clone
dir-iterator: add flags parameter to dir_iterator_begin
dir-iterator: refactor state machine model
dir-iterator: use warning_errno when possible
dir-iterator: add tests for dir-iterator API
clone: better handle symlinked files at .git/objects/
clone: test for our behavior on odd objects/* content

1  2 
Makefile
builtin/clone.c
t/helper/test-tool.c
t/helper/test-tool.h
diff --combined Makefile
index b11cdd4fe79234b57211d236930b0798416b8e6c,61df07f48815d166e36b269863418597edb92185..bd246f2989575a72758fd3765659f61a8aa58c8c
+++ b/Makefile
@@@ -265,6 -265,10 +265,6 @@@ all:
  #
  # Define NO_DEFLATE_BOUND if your zlib does not have deflateBound.
  #
 -# Define NO_R_TO_GCC_LINKER if your gcc does not like "-R/path/lib"
 -# that tells runtime paths to dynamic libraries;
 -# "-Wl,-rpath=/path/lib" is used instead.
 -#
  # Define NO_NORETURN if using buggy versions of gcc 4.6+ and profile feedback,
  # as the compiler can crash (http://gcc.gnu.org/bugzilla/show_bug.cgi?id=49299)
  #
@@@ -620,6 -624,8 +620,6 @@@ SCRIPT_SH += git-web--browse.s
  
  SCRIPT_LIB += git-mergetool--lib
  SCRIPT_LIB += git-parse-remote
 -SCRIPT_LIB += git-rebase--am
 -SCRIPT_LIB += git-rebase--common
  SCRIPT_LIB += git-rebase--preserve-merges
  SCRIPT_LIB += git-sh-setup
  SCRIPT_LIB += git-sh-i18n
@@@ -704,6 -710,7 +704,7 @@@ TEST_BUILTINS_OBJS += test-config.
  TEST_BUILTINS_OBJS += test-ctype.o
  TEST_BUILTINS_OBJS += test-date.o
  TEST_BUILTINS_OBJS += test-delta.o
+ TEST_BUILTINS_OBJS += test-dir-iterator.o
  TEST_BUILTINS_OBJS += test-drop-caches.o
  TEST_BUILTINS_OBJS += test-dump-cache-tree.o
  TEST_BUILTINS_OBJS += test-dump-fsmonitor.o
@@@ -721,7 -728,6 +722,7 @@@ TEST_BUILTINS_OBJS += test-lazy-init-na
  TEST_BUILTINS_OBJS += test-match-trees.o
  TEST_BUILTINS_OBJS += test-mergesort.o
  TEST_BUILTINS_OBJS += test-mktemp.o
 +TEST_BUILTINS_OBJS += test-oidmap.o
  TEST_BUILTINS_OBJS += test-online-cpus.o
  TEST_BUILTINS_OBJS += test-parse-options.o
  TEST_BUILTINS_OBJS += test-path-utils.o
@@@ -772,11 -778,9 +773,11 @@@ BUILT_INS += git-format-patch$
  BUILT_INS += git-fsck-objects$X
  BUILT_INS += git-init$X
  BUILT_INS += git-merge-subtree$X
 +BUILT_INS += git-restore$X
  BUILT_INS += git-show$X
  BUILT_INS += git-stage$X
  BUILT_INS += git-status$X
 +BUILT_INS += git-switch$X
  BUILT_INS += git-whatchanged$X
  
  # what 'all' will build and 'install' will install in gitexecdir,
@@@ -1062,7 -1066,6 +1063,7 @@@ BUILTIN_OBJS += builtin/diff-index.
  BUILTIN_OBJS += builtin/diff-tree.o
  BUILTIN_OBJS += builtin/diff.o
  BUILTIN_OBJS += builtin/difftool.o
 +BUILTIN_OBJS += builtin/env--helper.o
  BUILTIN_OBJS += builtin/fast-export.o
  BUILTIN_OBJS += builtin/fetch-pack.o
  BUILTIN_OBJS += builtin/fetch.o
@@@ -1158,7 -1161,6 +1159,7 @@@ endi
  # which'll override these defaults.
  CFLAGS = -g -O2 -Wall
  LDFLAGS =
 +CC_LD_DYNPATH = -Wl,-rpath,
  BASIC_CFLAGS = -I.
  BASIC_LDFLAGS =
  
@@@ -1239,7 -1241,7 +1240,7 @@@ endi
  
  ifdef SANE_TOOL_PATH
  SANE_TOOL_PATH_SQ = $(subst ','\'',$(SANE_TOOL_PATH))
 -BROKEN_PATH_FIX = 's|^\# @@BROKEN_PATH_FIX@@$$|git_broken_path_fix $(SANE_TOOL_PATH_SQ)|'
 +BROKEN_PATH_FIX = 's|^\# @@BROKEN_PATH_FIX@@$$|git_broken_path_fix "$(SANE_TOOL_PATH_SQ)"|'
  PATH := $(SANE_TOOL_PATH):${PATH}
  else
  BROKEN_PATH_FIX = '/^\# @@BROKEN_PATH_FIX@@$$/d'
@@@ -1289,6 -1291,16 +1290,6 @@@ ifeq ($(uname_S),Darwin
        PTHREAD_LIBS =
  endif
  
 -ifndef CC_LD_DYNPATH
 -      ifdef NO_R_TO_GCC_LINKER
 -              # Some gcc does not accept and pass -R to the linker to specify
 -              # the runtime dynamic library path.
 -              CC_LD_DYNPATH = -Wl,-rpath,
 -      else
 -              CC_LD_DYNPATH = -R
 -      endif
 -endif
 -
  ifdef NO_LIBGEN_H
        COMPAT_CFLAGS += -DNO_LIBGEN_H
        COMPAT_OBJS += compat/basename.o
@@@ -2862,33 -2874,6 +2863,33 @@@ install: al
        $(INSTALL) $(ALL_PROGRAMS) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
        $(INSTALL) -m 644 $(SCRIPT_LIB) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
        $(INSTALL) $(install_bindir_programs) '$(DESTDIR_SQ)$(bindir_SQ)'
 +ifdef MSVC
 +      # We DO NOT install the individual foo.o.pdb files because they
 +      # have already been rolled up into the exe's pdb file.
 +      # We DO NOT have pdb files for the builtin commands (like git-status.exe)
 +      # because it is just a copy/hardlink of git.exe, rather than a unique binary.
 +      $(INSTALL) git.pdb '$(DESTDIR_SQ)$(bindir_SQ)'
 +      $(INSTALL) git-shell.pdb '$(DESTDIR_SQ)$(bindir_SQ)'
 +      $(INSTALL) git-upload-pack.pdb '$(DESTDIR_SQ)$(bindir_SQ)'
 +      $(INSTALL) git-credential-store.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 +      $(INSTALL) git-daemon.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 +      $(INSTALL) git-fast-import.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 +      $(INSTALL) git-http-backend.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 +      $(INSTALL) git-http-fetch.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 +      $(INSTALL) git-http-push.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 +      $(INSTALL) git-imap-send.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 +      $(INSTALL) git-remote-http.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 +      $(INSTALL) git-remote-testsvn.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 +      $(INSTALL) git-sh-i18n--envsubst.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 +      $(INSTALL) git-show-index.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 +ifndef DEBUG
 +      $(INSTALL) $(vcpkg_rel_bin)/*.dll '$(DESTDIR_SQ)$(bindir_SQ)'
 +      $(INSTALL) $(vcpkg_rel_bin)/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)'
 +else
 +      $(INSTALL) $(vcpkg_dbg_bin)/*.dll '$(DESTDIR_SQ)$(bindir_SQ)'
 +      $(INSTALL) $(vcpkg_dbg_bin)/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)'
 +endif
 +endif
        $(MAKE) -C templates DESTDIR='$(DESTDIR_SQ)' install
        $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(mergetools_instdir_SQ)'
        $(INSTALL) -m 644 mergetools/* '$(DESTDIR_SQ)$(mergetools_instdir_SQ)'
@@@ -3101,19 -3086,6 +3102,19 @@@ endi
        $(RM) GIT-VERSION-FILE GIT-CFLAGS GIT-LDFLAGS GIT-BUILD-OPTIONS
        $(RM) GIT-USER-AGENT GIT-PREFIX
        $(RM) GIT-SCRIPT-DEFINES GIT-PERL-DEFINES GIT-PERL-HEADER GIT-PYTHON-VARS
 +ifdef MSVC
 +      $(RM) $(patsubst %.o,%.o.pdb,$(OBJECTS))
 +      $(RM) $(patsubst %.exe,%.pdb,$(OTHER_PROGRAMS))
 +      $(RM) $(patsubst %.exe,%.iobj,$(OTHER_PROGRAMS))
 +      $(RM) $(patsubst %.exe,%.ipdb,$(OTHER_PROGRAMS))
 +      $(RM) $(patsubst %.exe,%.pdb,$(PROGRAMS))
 +      $(RM) $(patsubst %.exe,%.iobj,$(PROGRAMS))
 +      $(RM) $(patsubst %.exe,%.ipdb,$(PROGRAMS))
 +      $(RM) $(patsubst %.exe,%.pdb,$(TEST_PROGRAMS))
 +      $(RM) $(patsubst %.exe,%.iobj,$(TEST_PROGRAMS))
 +      $(RM) $(patsubst %.exe,%.ipdb,$(TEST_PROGRAMS))
 +      $(RM) compat/vcbuild/MSVC-DEFS-GEN
 +endif
  
  .PHONY: all install profile-clean cocciclean clean strip
  .PHONY: shell_compatibility_test please_set_SHELL_PATH_to_a_more_modern_shell
diff --combined builtin/clone.c
index a4fe72879d43e4e42d6cbb5a4dada2f7a111b872,62b6a0a352e8f42b83c03b4a3bd4f445ce280e8a..f665b28ccccfacaf5dfe84b7f94081e1afacdd49
@@@ -23,6 -23,8 +23,8 @@@
  #include "transport.h"
  #include "strbuf.h"
  #include "dir.h"
+ #include "dir-iterator.h"
+ #include "iterator.h"
  #include "sigchain.h"
  #include "branch.h"
  #include "remote.h"
@@@ -67,7 -69,6 +69,7 @@@ static int max_jobs = -1
  static struct string_list option_recurse_submodules = STRING_LIST_INIT_NODUP;
  static struct list_objects_filter_options filter_options;
  static struct string_list server_options = STRING_LIST_INIT_NODUP;
 +static int option_remote_submodules;
  
  static int recurse_submodules_cb(const struct option *opt,
                                 const char *arg, int unset)
@@@ -143,8 -144,6 +145,8 @@@ static struct option builtin_clone_opti
        OPT_SET_INT('6', "ipv6", &family, N_("use IPv6 addresses only"),
                        TRANSPORT_FAMILY_IPV6),
        OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),
 +      OPT_BOOL(0, "remote-submodules", &option_remote_submodules,
 +                  N_("any cloned submodules will use their remote-tracking branch")),
        OPT_END()
  };
  
@@@ -394,50 -393,55 +396,55 @@@ static void copy_alternates(struct strb
        fclose(in);
  }
  
+ static void mkdir_if_missing(const char *pathname, mode_t mode)
+ {
+       struct stat st;
+       if (!mkdir(pathname, mode))
+               return;
+       if (errno != EEXIST)
+               die_errno(_("failed to create directory '%s'"), pathname);
+       else if (stat(pathname, &st))
+               die_errno(_("failed to stat '%s'"), pathname);
+       else if (!S_ISDIR(st.st_mode))
+               die(_("%s exists and is not a directory"), pathname);
+ }
  static void copy_or_link_directory(struct strbuf *src, struct strbuf *dest,
-                                  const char *src_repo, int src_baselen)
+                                  const char *src_repo)
  {
-       struct dirent *de;
-       struct stat buf;
        int src_len, dest_len;
-       DIR *dir;
-       dir = opendir(src->buf);
-       if (!dir)
-               die_errno(_("failed to open '%s'"), src->buf);
-       if (mkdir(dest->buf, 0777)) {
-               if (errno != EEXIST)
-                       die_errno(_("failed to create directory '%s'"), dest->buf);
-               else if (stat(dest->buf, &buf))
-                       die_errno(_("failed to stat '%s'"), dest->buf);
-               else if (!S_ISDIR(buf.st_mode))
-                       die(_("%s exists and is not a directory"), dest->buf);
-       }
+       struct dir_iterator *iter;
+       int iter_status;
+       unsigned int flags;
+       mkdir_if_missing(dest->buf, 0777);
+       flags = DIR_ITERATOR_PEDANTIC | DIR_ITERATOR_FOLLOW_SYMLINKS;
+       iter = dir_iterator_begin(src->buf, flags);
+       if (!iter)
+               die_errno(_("failed to start iterator over '%s'"), src->buf);
  
        strbuf_addch(src, '/');
        src_len = src->len;
        strbuf_addch(dest, '/');
        dest_len = dest->len;
  
-       while ((de = readdir(dir)) != NULL) {
+       while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) {
                strbuf_setlen(src, src_len);
-               strbuf_addstr(src, de->d_name);
+               strbuf_addstr(src, iter->relative_path);
                strbuf_setlen(dest, dest_len);
-               strbuf_addstr(dest, de->d_name);
-               if (stat(src->buf, &buf)) {
-                       warning (_("failed to stat %s\n"), src->buf);
-                       continue;
-               }
-               if (S_ISDIR(buf.st_mode)) {
-                       if (de->d_name[0] != '.')
-                               copy_or_link_directory(src, dest,
-                                                      src_repo, src_baselen);
+               strbuf_addstr(dest, iter->relative_path);
+               if (S_ISDIR(iter->st.st_mode)) {
+                       mkdir_if_missing(dest->buf, 0777);
                        continue;
                }
  
                /* Files that cannot be copied bit-for-bit... */
-               if (!strcmp(src->buf + src_baselen, "/info/alternates")) {
+               if (!fspathcmp(iter->relative_path, "info/alternates")) {
                        copy_alternates(src, src_repo);
                        continue;
                }
                if (unlink(dest->buf) && errno != ENOENT)
                        die_errno(_("failed to unlink '%s'"), dest->buf);
                if (!option_no_hardlinks) {
-                       if (!link(src->buf, dest->buf))
+                       if (!link(real_path(src->buf), dest->buf))
                                continue;
                        if (option_local > 0)
                                die_errno(_("failed to create link '%s'"), dest->buf);
                if (copy_file_with_time(dest->buf, src->buf, 0666))
                        die_errno(_("failed to copy file to '%s'"), dest->buf);
        }
-       closedir(dir);
+       if (iter_status != ITER_DONE) {
+               strbuf_setlen(src, src_len);
+               die(_("failed to iterate over '%s'"), src->buf);
+       }
  }
  
  static void clone_local(const char *src_repo, const char *dest_repo)
                get_common_dir(&dest, dest_repo);
                strbuf_addstr(&src, "/objects");
                strbuf_addstr(&dest, "/objects");
-               copy_or_link_directory(&src, &dest, src_repo, src.len);
+               copy_or_link_directory(&src, &dest, src_repo);
                strbuf_release(&src);
                strbuf_release(&dest);
        }
@@@ -494,7 -502,7 +505,7 @@@ static enum 
  static const char junk_leave_repo_msg[] =
  N_("Clone succeeded, but checkout failed.\n"
     "You can inspect what was checked out with 'git status'\n"
 -   "and retry the checkout with 'git checkout -f HEAD'\n");
 +   "and retry with 'git restore --source=HEAD :/'\n");
  
  static void remove_junk(void)
  {
@@@ -793,11 -801,6 +804,11 @@@ static int checkout(int submodule_progr
                if (option_verbosity < 0)
                        argv_array_push(&args, "--quiet");
  
 +              if (option_remote_submodules) {
 +                      argv_array_push(&args, "--remote");
 +                      argv_array_push(&args, "--no-fetch");
 +              }
 +
                err = run_command_v_opt(args.argv, RUN_GIT_CMD);
                argv_array_clear(&args);
        }
@@@ -1227,7 -1230,7 +1238,7 @@@ int cmd_clone(int argc, const char **ar
                        remote_head_points_at, &branch_top);
  
        if (filter_options.choice)
 -              partial_clone_register("origin", &filter_options);
 +              partial_clone_register(option_origin, &filter_options);
  
        if (is_local)
                clone_local(path, git_dir);
        transport_disconnect(transport);
  
        if (option_dissociate) {
 -              close_all_packs(the_repository->objects);
 +              close_object_store(the_repository->objects);
                dissociate_from_references();
        }
  
diff --combined t/helper/test-tool.c
index 1eac25233f7ce62ecb00b2d1e3d06d1423c3581f,7bc9bb231ef40c13cec4671d4a5fc8cedc51e0d7..ce7e89028c4a7cc328998b0acfd42ec983cd7a74
@@@ -19,6 -19,7 +19,7 @@@ static struct test_cmd cmds[] = 
        { "ctype", cmd__ctype },
        { "date", cmd__date },
        { "delta", cmd__delta },
+       { "dir-iterator", cmd__dir_iterator },
        { "drop-caches", cmd__drop_caches },
        { "dump-cache-tree", cmd__dump_cache_tree },
        { "dump-fsmonitor", cmd__dump_fsmonitor },
@@@ -35,7 -36,6 +36,7 @@@
        { "match-trees", cmd__match_trees },
        { "mergesort", cmd__mergesort },
        { "mktemp", cmd__mktemp },
 +      { "oidmap", cmd__oidmap },
        { "online-cpus", cmd__online_cpus },
        { "parse-options", cmd__parse_options },
        { "path-utils", cmd__path_utils },
diff --combined t/helper/test-tool.h
index c7a46dc320e93b3bb5aef5f7fce3697c4558f814,ec0ffbd0cb674db09bd0242a5f2e66ce962ee432..f805bb39ae9e2ebe83ecb4e52f953fbcb43ce444
@@@ -9,6 -9,7 +9,7 @@@ int cmd__config(int argc, const char **
  int cmd__ctype(int argc, const char **argv);
  int cmd__date(int argc, const char **argv);
  int cmd__delta(int argc, const char **argv);
+ int cmd__dir_iterator(int argc, const char **argv);
  int cmd__drop_caches(int argc, const char **argv);
  int cmd__dump_cache_tree(int argc, const char **argv);
  int cmd__dump_fsmonitor(int argc, const char **argv);
@@@ -25,7 -26,6 +26,7 @@@ int cmd__lazy_init_name_hash(int argc, 
  int cmd__match_trees(int argc, const char **argv);
  int cmd__mergesort(int argc, const char **argv);
  int cmd__mktemp(int argc, const char **argv);
 +int cmd__oidmap(int argc, const char **argv);
  int cmd__online_cpus(int argc, const char **argv);
  int cmd__parse_options(int argc, const char **argv);
  int cmd__path_utils(int argc, const char **argv);