Merge branch 'nd/index-pack-one-fd-per-thread' into maint
authorJunio C Hamano <gitster@pobox.com>
Wed, 25 Jun 2014 18:47:58 +0000 (11:47 -0700)
committerJunio C Hamano <gitster@pobox.com>
Wed, 25 Jun 2014 18:47:58 +0000 (11:47 -0700)
We used to disable threaded "git index-pack" on platforms without
thread-safe pread(); use a different workaround for such
platforms to allow threaded "git index-pack".

* nd/index-pack-one-fd-per-thread:
index-pack: work around thread-unsafe pread()

1  2 
Makefile
builtin/index-pack.c
config.mak.uname
diff --combined Makefile
index a53f3a8326c2e62dc79bae7169d64137ac3dab20,00a70840eb17d70601e55ad654e4ca75e2b26883..e0caec3f679089b6643e129fbdb2423dfd8f6d94
+++ b/Makefile
@@@ -59,9 -59,9 +59,9 @@@ all:
  # FreeBSD can use either, but MinGW and some others need to use
  # libcharset.h's locale_charset() instead.
  #
 -# Define CHARSET_LIB to you need to link with library other than -liconv to
 +# Define CHARSET_LIB to the library you need to link with in order to
  # use locale_charset() function.  On some platforms this needs to set to
 -# -lcharset
 +# -lcharset, on others to -liconv .
  #
  # Define LIBC_CONTAINS_LIBINTL if your gettext implementation doesn't
  # need -lintl when linking.
  #
  # Define NO_MKSTEMPS if you don't have mkstemps in the C library.
  #
 -# Define NO_FNMATCH if you don't have fnmatch in the C library.
 -#
 -# Define NO_FNMATCH_CASEFOLD if your fnmatch function doesn't have the
 -# FNM_CASEFOLD GNU extension.
 -#
 -# Define NO_WILDMATCH if you do not want to use Git's wildmatch
 -# implementation as fnmatch
 -#
  # Define NO_GECOS_IN_PWENT if you don't have pw_gecos in struct passwd
  # in the C library.
  #
  #
  # Define NEEDS_LIBINTL_BEFORE_LIBICONV if you need libintl before libiconv.
  #
 -# Define NO_INTPTR_T if you don't have intptr_t nor uintptr_t.
 +# Define NO_INTPTR_T if you don't have intptr_t or uintptr_t.
  #
  # Define NO_UINTMAX_T if you don't have uintmax_t.
  #
  # Define NO_STRUCT_ITIMERVAL if you don't have struct itimerval
  # This also implies NO_SETITIMER
  #
- # Define NO_THREAD_SAFE_PREAD if your pread() implementation is not
- # thread-safe. (e.g. compat/pread.c or cygwin)
- #
  # Define NO_FAST_WORKING_DIRECTORY if accessing objects in pack files is
  # generally faster on your platform than accessing the working directory.
  #
  # Define DEFAULT_HELP_FORMAT to "man", "info" or "html"
  # (defaults to "man") if you want to have a different default when
  # "git help" is called without a parameter specifying the format.
 +#
 +# Define TEST_GIT_INDEX_VERSION to 2, 3 or 4 to run the test suite
 +# with a different indexfile format version.  If it isn't set the index
 +# file format used is index-v[23].
 +#
 +# Define GMTIME_UNRELIABLE_ERRORS if your gmtime() function does not
 +# return NULL when it receives a bogus time_t.
  
  GIT-VERSION-FILE: FORCE
        @$(SHELL_PATH) ./GIT-VERSION-GEN
@@@ -554,7 -552,6 +551,7 @@@ TEST_PROGRAMS_NEED_X += test-dat
  TEST_PROGRAMS_NEED_X += test-delta
  TEST_PROGRAMS_NEED_X += test-dump-cache-tree
  TEST_PROGRAMS_NEED_X += test-genrandom
 +TEST_PROGRAMS_NEED_X += test-hashmap
  TEST_PROGRAMS_NEED_X += test-index-version
  TEST_PROGRAMS_NEED_X += test-line-buffer
  TEST_PROGRAMS_NEED_X += test-match-trees
@@@ -663,8 -660,6 +660,8 @@@ LIB_H += diff.
  LIB_H += diffcore.h
  LIB_H += dir.h
  LIB_H += exec_cmd.h
 +LIB_H += ewah/ewok.h
 +LIB_H += ewah/ewok_rlw.h
  LIB_H += fetch-pack.h
  LIB_H += fmt-merge-msg.h
  LIB_H += fsck.h
@@@ -673,7 -668,7 +670,7 @@@ LIB_H += git-compat-util.
  LIB_H += gpg-interface.h
  LIB_H += graph.h
  LIB_H += grep.h
 -LIB_H += hash.h
 +LIB_H += hashmap.h
  LIB_H += help.h
  LIB_H += http.h
  LIB_H += kwset.h
@@@ -692,10 -687,8 +689,10 @@@ LIB_H += notes-merge.
  LIB_H += notes-utils.h
  LIB_H += notes.h
  LIB_H += object.h
 +LIB_H += pack-objects.h
  LIB_H += pack-revindex.h
  LIB_H += pack.h
 +LIB_H += pack-bitmap.h
  LIB_H += parse-options.h
  LIB_H += patch-ids.h
  LIB_H += pathspec.h
@@@ -799,10 -792,6 +796,10 @@@ LIB_OBJS += dir.
  LIB_OBJS += editor.o
  LIB_OBJS += entry.o
  LIB_OBJS += environment.o
 +LIB_OBJS += ewah/bitmap.o
 +LIB_OBJS += ewah/ewah_bitmap.o
 +LIB_OBJS += ewah/ewah_io.o
 +LIB_OBJS += ewah/ewah_rlw.o
  LIB_OBJS += exec_cmd.o
  LIB_OBJS += fetch-pack.o
  LIB_OBJS += fsck.o
@@@ -810,7 -799,7 +807,7 @@@ LIB_OBJS += gettext.
  LIB_OBJS += gpg-interface.o
  LIB_OBJS += graph.o
  LIB_OBJS += grep.o
 -LIB_OBJS += hash.o
 +LIB_OBJS += hashmap.o
  LIB_OBJS += help.o
  LIB_OBJS += hex.o
  LIB_OBJS += ident.o
@@@ -834,10 -823,7 +831,10 @@@ LIB_OBJS += notes-cache.
  LIB_OBJS += notes-merge.o
  LIB_OBJS += notes-utils.o
  LIB_OBJS += object.o
 +LIB_OBJS += pack-bitmap.o
 +LIB_OBJS += pack-bitmap-write.o
  LIB_OBJS += pack-check.o
 +LIB_OBJS += pack-objects.o
  LIB_OBJS += pack-revindex.o
  LIB_OBJS += pack-write.o
  LIB_OBJS += pager.o
@@@ -895,7 -881,6 +892,7 @@@ LIB_OBJS += userdiff.
  LIB_OBJS += utf8.o
  LIB_OBJS += varint.o
  LIB_OBJS += version.o
 +LIB_OBJS += versioncmp.o
  LIB_OBJS += walker.o
  LIB_OBJS += wildmatch.o
  LIB_OBJS += wrapper.o
@@@ -1283,6 -1268,20 +1280,6 @@@ endi
  ifdef NO_STRTOULL
        COMPAT_CFLAGS += -DNO_STRTOULL
  endif
 -ifdef NO_FNMATCH
 -      COMPAT_CFLAGS += -Icompat/fnmatch
 -      COMPAT_CFLAGS += -DNO_FNMATCH
 -      COMPAT_OBJS += compat/fnmatch/fnmatch.o
 -else
 -ifdef NO_FNMATCH_CASEFOLD
 -      COMPAT_CFLAGS += -Icompat/fnmatch
 -      COMPAT_CFLAGS += -DNO_FNMATCH_CASEFOLD
 -      COMPAT_OBJS += compat/fnmatch/fnmatch.o
 -endif
 -endif
 -ifndef NO_WILDMATCH
 -      COMPAT_CFLAGS += -DUSE_WILDMATCH
 -endif
  ifdef NO_SETENV
        COMPAT_CFLAGS += -DNO_SETENV
        COMPAT_OBJS += compat/setenv.o
@@@ -1339,10 -1338,6 +1336,6 @@@ endi
  ifdef NO_PREAD
        COMPAT_CFLAGS += -DNO_PREAD
        COMPAT_OBJS += compat/pread.o
-       NO_THREAD_SAFE_PREAD = YesPlease
- endif
- ifdef NO_THREAD_SAFE_PREAD
-       BASIC_CFLAGS += -DNO_THREAD_SAFE_PREAD
  endif
  ifdef NO_FAST_WORKING_DIRECTORY
        BASIC_CFLAGS += -DNO_FAST_WORKING_DIRECTORY
@@@ -1492,11 -1487,6 +1485,11 @@@ ifneq (,$(XDL_FAST_HASH)
        BASIC_CFLAGS += -DXDL_FAST_HASH
  endif
  
 +ifdef GMTIME_UNRELIABLE_ERRORS
 +      COMPAT_OBJS += compat/gmtime.o
 +      BASIC_CFLAGS += -DGMTIME_UNRELIABLE_ERRORS
 +endif
 +
  ifeq ($(TCLTK_PATH),)
  NO_TCLTK = NoThanks
  endif
@@@ -2102,7 -2092,7 +2095,7 @@@ pdf
  
  XGETTEXT_FLAGS = \
        --force-po \
 -      --add-comments \
 +      --add-comments=TRANSLATORS: \
        --msgid-bugs-address="Git Mailing List <git@vger.kernel.org>" \
        --from-code=UTF-8
  XGETTEXT_FLAGS_C = $(XGETTEXT_FLAGS) --language=C \
@@@ -2225,9 -2215,6 +2218,9 @@@ endi
  ifdef GIT_PERF_MAKE_OPTS
        @echo GIT_PERF_MAKE_OPTS=\''$(subst ','\'',$(subst ','\'',$(GIT_PERF_MAKE_OPTS)))'\' >>$@
  endif
 +ifdef TEST_GIT_INDEX_VERSION
 +      @echo TEST_GIT_INDEX_VERSION=\''$(subst ','\'',$(subst ','\'',$(TEST_GIT_INDEX_VERSION)))'\' >>$@
 +endif
  
  ### Detect Python interpreter path changes
  ifndef NO_PYTHON
@@@ -2485,9 -2472,8 +2478,9 @@@ profile-clean
        $(RM) $(addsuffix *.gcno,$(addprefix $(PROFILE_DIR)/, $(object_dirs)))
  
  clean: profile-clean coverage-clean
 -      $(RM) *.o *.res block-sha1/*.o ppc/*.o compat/*.o compat/*/*.o xdiff/*.o vcs-svn/*.o \
 -              builtin/*.o $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB)
 +      $(RM) *.o *.res block-sha1/*.o ppc/*.o compat/*.o compat/*/*.o
 +      $(RM) xdiff/*.o vcs-svn/*.o ewah/*.o builtin/*.o
 +      $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB)
        $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X
        $(RM) $(TEST_PROGRAMS) $(NO_INSTALL)
        $(RM) -r bin-wrappers $(dep_dirs)
diff --combined builtin/index-pack.c
index b9f6e12c0e91635eafd8510dc7cf20d6c3e58433,ed489bbe9aab3ae7268f139f2d34c737880be49c..de35960d99bb6c92c0f57c588aa6f43e3c227d2c
@@@ -40,17 -40,13 +40,13 @@@ struct base_data 
        int ofs_first, ofs_last;
  };
  
- #if !defined(NO_PTHREADS) && defined(NO_THREAD_SAFE_PREAD)
- /* pread() emulation is not thread-safe. Disable threading. */
- #define NO_PTHREADS
- #endif
  struct thread_local {
  #ifndef NO_PTHREADS
        pthread_t thread;
  #endif
        struct base_data *base_cache;
        size_t base_cache_used;
+       int pack_fd;
  };
  
  /*
@@@ -91,7 -87,8 +87,8 @@@ static off_t consumed_bytes
  static unsigned deepest_delta;
  static git_SHA_CTX input_ctx;
  static uint32_t input_crc32;
- static int input_fd, output_fd, pack_fd;
+ static int input_fd, output_fd;
+ static const char *curr_pack;
  
  #ifndef NO_PTHREADS
  
@@@ -134,6 -131,7 +131,7 @@@ static inline void unlock_mutex(pthread
   */
  static void init_thread(void)
  {
+       int i;
        init_recursive_mutex(&read_mutex);
        pthread_mutex_init(&counter_mutex, NULL);
        pthread_mutex_init(&work_mutex, NULL);
                pthread_mutex_init(&deepest_delta_mutex, NULL);
        pthread_key_create(&key, NULL);
        thread_data = xcalloc(nr_threads, sizeof(*thread_data));
+       for (i = 0; i < nr_threads; i++) {
+               thread_data[i].pack_fd = open(curr_pack, O_RDONLY);
+               if (thread_data[i].pack_fd == -1)
+                       die_errno(_("unable to open %s"), curr_pack);
+       }
        threads_active = 1;
  }
  
  static void cleanup_thread(void)
  {
+       int i;
        if (!threads_active)
                return;
        threads_active = 0;
        pthread_mutex_destroy(&work_mutex);
        if (show_stat)
                pthread_mutex_destroy(&deepest_delta_mutex);
+       for (i = 0; i < nr_threads; i++)
+               close(thread_data[i].pack_fd);
        pthread_key_delete(key);
        free(thread_data);
  }
@@@ -288,13 -295,13 +295,13 @@@ static const char *open_pack_file(cons
                        output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600);
                if (output_fd < 0)
                        die_errno(_("unable to create '%s'"), pack_name);
-               pack_fd = output_fd;
+               nothread_data.pack_fd = output_fd;
        } else {
                input_fd = open(pack_name, O_RDONLY);
                if (input_fd < 0)
                        die_errno(_("cannot open packfile '%s'"), pack_name);
                output_fd = -1;
-               pack_fd = input_fd;
+               nothread_data.pack_fd = input_fd;
        }
        git_SHA1_Init(&input_ctx);
        return pack_name;
@@@ -542,7 -549,7 +549,7 @@@ static void *unpack_data(struct object_
  
        do {
                ssize_t n = (len < 64*1024) ? len : 64*1024;
-               n = pread(pack_fd, inbuf, n, from);
+               n = pread(get_thread_data()->pack_fd, inbuf, n, from);
                if (n < 0)
                        die_errno(_("cannot pread pack file"));
                if (!n)
@@@ -1291,7 -1298,7 +1298,7 @@@ static void final(const char *final_pac
                if (keep_fd < 0) {
                        if (errno != EEXIST)
                                die_errno(_("cannot write keep file '%s'"),
 -                                        keep_name);
 +                                        keep_name ? keep_name : name);
                } else {
                        if (keep_msg_len > 0) {
                                write_or_die(keep_fd, keep_msg, keep_msg_len);
                        }
                        if (close(keep_fd) != 0)
                                die_errno(_("cannot close written keep file '%s'"),
 -                                  keep_name);
 +                                        keep_name ? keep_name : name);
                        report = "keep";
                }
        }
@@@ -1490,7 -1497,7 +1497,7 @@@ static void show_pack_info(int stat_onl
  int cmd_index_pack(int argc, const char **argv, const char *prefix)
  {
        int i, fix_thin_pack = 0, verify = 0, stat_only = 0;
-       const char *curr_pack, *curr_index;
+       const char *curr_index;
        const char *index_name = NULL, *pack_name = NULL;
        const char *keep_name = NULL, *keep_msg = NULL;
        char *index_name_buf = NULL, *keep_name_buf = NULL;
        if (argc == 2 && !strcmp(argv[1], "-h"))
                usage(index_pack_usage);
  
 -      read_replace_refs = 0;
 +      check_replace_refs = 0;
  
        reset_pack_idx_option(&opts);
        git_config(git_index_pack_config, &opts);
diff --combined config.mak.uname
index 23a880365616c34ea84d8a3194b98163864a2cd1,db5c249b4ca291f9b3ba27b9619430fccbf7d1ff..7846bd76573fe2513f08a824e89938deae6543de
@@@ -108,6 -108,7 +108,6 @@@ ifeq ($(uname_S),SunOS
        NO_MKDTEMP = YesPlease
        NO_MKSTEMPS = YesPlease
        NO_REGEX = YesPlease
 -      NO_FNMATCH_CASEFOLD = YesPlease
        NO_MSGFMT_EXTENDED_OPTIONS = YesPlease
        HAVE_DEV_TTY = YesPlease
        ifeq ($(uname_R),5.6)
@@@ -157,7 -158,6 +157,6 @@@ ifeq ($(uname_O),Cygwin
                NO_SYMLINK_HEAD = YesPlease
                NO_IPV6 = YesPlease
                OLD_ICONV = UnfortunatelyYes
-               NO_THREAD_SAFE_PREAD = YesPlease
                # There are conflicting reports about this.
                # On some boxes NO_MMAP is needed, and not so elsewhere.
                # Try commenting this out if you suspect MMAP is more efficient
@@@ -187,7 -187,6 +186,7 @@@ ifeq ($(uname_S),FreeBSD
        endif
        PYTHON_PATH = /usr/local/bin/python
        HAVE_PATHS_H = YesPlease
 +      GMTIME_UNRELIABLE_ERRORS = UnfortunatelyYes
  endif
  ifeq ($(uname_S),OpenBSD)
        NO_STRCASESTR = YesPlease
@@@ -259,6 -258,7 +258,6 @@@ ifeq ($(uname_S),IRIX
        # issue, comment out the NO_MMAP statement.
        NO_MMAP = YesPlease
        NO_REGEX = YesPlease
 -      NO_FNMATCH_CASEFOLD = YesPlease
        SNPRINTF_RETURNS_BOGUS = YesPlease
        SHELL_PATH = /usr/gnu/bin/bash
        NEEDS_LIBGEN = YesPlease
@@@ -278,6 -278,7 +277,6 @@@ ifeq ($(uname_S),IRIX64
        # issue, comment out the NO_MMAP statement.
        NO_MMAP = YesPlease
        NO_REGEX = YesPlease
 -      NO_FNMATCH_CASEFOLD = YesPlease
        SNPRINTF_RETURNS_BOGUS = YesPlease
        SHELL_PATH = /usr/gnu/bin/bash
        NEEDS_LIBGEN = YesPlease
@@@ -294,6 -295,7 +293,6 @@@ ifeq ($(uname_S),HP-UX
        NO_UNSETENV = YesPlease
        NO_HSTRERROR = YesPlease
        NO_SYS_SELECT_H = YesPlease
 -      NO_FNMATCH_CASEFOLD = YesPlease
        SNPRINTF_RETURNS_BOGUS = YesPlease
        NO_NSEC = YesPlease
        ifeq ($(uname_R),B.11.00)
@@@ -324,6 -326,7 +323,6 @@@ ifeq ($(uname_S),Windows
        NO_UNSETENV = YesPlease
        NO_STRCASESTR = YesPlease
        NO_STRLCPY = YesPlease
 -      NO_FNMATCH = YesPlease
        NO_MEMMEM = YesPlease
        # NEEDS_LIBICONV = YesPlease
        NO_ICONV = YesPlease
        NO_MKSTEMPS = YesPlease
        SNPRINTF_RETURNS_BOGUS = YesPlease
        NO_SVN_TESTS = YesPlease
 -      NO_PERL_MAKEMAKER = YesPlease
        RUNTIME_PREFIX = YesPlease
        NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease
        NO_NSEC = YesPlease
        UNRELIABLE_FSTAT = UnfortunatelyYes
        OBJECT_CREATION_USES_RENAMES = UnfortunatelyNeedsTo
        NO_REGEX = YesPlease
 -      NO_CURL = YesPlease
        NO_GETTEXT = YesPlease
        NO_PYTHON = YesPlease
        BLK_SHA1 = YesPlease
                compat/win32/dirent.o
        COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DNOGDI -DHAVE_STRING_H -DHAVE_ALLOCA_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\"
        BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -SUBSYSTEM:CONSOLE -NODEFAULTLIB:MSVCRT.lib
 -      EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib
 +      EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj
        PTHREAD_LIBS =
        lib =
  ifndef DEBUG
@@@ -383,11 -388,13 +382,11 @@@ ifeq ($(uname_S),Interix
                NO_INET_NTOP = YesPlease
                NO_INET_PTON = YesPlease
                NO_SOCKADDR_STORAGE = YesPlease
 -              NO_FNMATCH_CASEFOLD = YesPlease
        endif
        ifeq ($(uname_R),5.2)
                NO_INET_NTOP = YesPlease
                NO_INET_PTON = YesPlease
                NO_SOCKADDR_STORAGE = YesPlease
 -              NO_FNMATCH_CASEFOLD = YesPlease
        endif
  endif
  ifeq ($(uname_S),Minix)
@@@ -432,6 -439,7 +431,6 @@@ ifeq ($(uname_S),NONSTOP_KERNEL
        NO_D_TYPE_IN_DIRENT = YesPlease
        NO_HSTRERROR = YesPlease
        NO_STRCASESTR = YesPlease
 -      NO_FNMATCH_CASEFOLD = YesPlease
        NO_MEMMEM = YesPlease
        NO_STRLCPY = YesPlease
        NO_SETENV = YesPlease
@@@ -475,6 -483,7 +474,6 @@@ ifneq (,$(findstring MINGW,$(uname_S))
        NO_UNSETENV = YesPlease
        NO_STRCASESTR = YesPlease
        NO_STRLCPY = YesPlease
 -      NO_FNMATCH = YesPlease
        NO_MEMMEM = YesPlease
        NEEDS_LIBICONV = YesPlease
        NO_STRTOUMAX = YesPlease
@@@ -528,6 -537,7 +527,6 @@@ ifeq ($(uname_S),QNX
        EXPAT_NEEDS_XMLPARSE_H = YesPlease
        HAVE_STRINGS_H = YesPlease
        NEEDS_SOCKET = YesPlease
 -      NO_FNMATCH_CASEFOLD = YesPlease
        NO_GETPAGESIZE = YesPlease
        NO_ICONV = YesPlease
        NO_MEMMEM = YesPlease