From: Junio C Hamano Date: Wed, 25 Jun 2014 18:47:58 +0000 (-0700) Subject: Merge branch 'nd/index-pack-one-fd-per-thread' into maint X-Git-Tag: v2.0.1~13 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/a9041df7ab712be1bfad9e6b67bf4cbf0f7c9c9b?ds=inline;hp=-c Merge branch 'nd/index-pack-one-fd-per-thread' into maint We used to disable threaded "git index-pack" on platforms without thread-safe pread(); use a different workaround for such platforms to allow threaded "git index-pack". * nd/index-pack-one-fd-per-thread: index-pack: work around thread-unsafe pread() --- a9041df7ab712be1bfad9e6b67bf4cbf0f7c9c9b diff --combined Makefile index a53f3a8326,00a70840eb..e0caec3f67 --- a/Makefile +++ b/Makefile @@@ -59,9 -59,9 +59,9 @@@ all: # FreeBSD can use either, but MinGW and some others need to use # libcharset.h's locale_charset() instead. # -# Define CHARSET_LIB to you need to link with library other than -liconv to +# Define CHARSET_LIB to the library you need to link with in order to # use locale_charset() function. On some platforms this needs to set to -# -lcharset +# -lcharset, on others to -liconv . # # Define LIBC_CONTAINS_LIBINTL if your gettext implementation doesn't # need -lintl when linking. @@@ -101,6 -101,14 +101,6 @@@ # # Define NO_MKSTEMPS if you don't have mkstemps in the C library. # -# Define NO_FNMATCH if you don't have fnmatch in the C library. -# -# Define NO_FNMATCH_CASEFOLD if your fnmatch function doesn't have the -# FNM_CASEFOLD GNU extension. -# -# Define NO_WILDMATCH if you do not want to use Git's wildmatch -# implementation as fnmatch -# # Define NO_GECOS_IN_PWENT if you don't have pw_gecos in struct passwd # in the C library. # @@@ -151,7 -159,7 +151,7 @@@ # # Define NEEDS_LIBINTL_BEFORE_LIBICONV if you need libintl before libiconv. # -# Define NO_INTPTR_T if you don't have intptr_t nor uintptr_t. +# Define NO_INTPTR_T if you don't have intptr_t or uintptr_t. # # Define NO_UINTMAX_T if you don't have uintmax_t. # @@@ -183,9 -191,6 +183,6 @@@ # Define NO_STRUCT_ITIMERVAL if you don't have struct itimerval # This also implies NO_SETITIMER # - # Define NO_THREAD_SAFE_PREAD if your pread() implementation is not - # thread-safe. (e.g. compat/pread.c or cygwin) - # # Define NO_FAST_WORKING_DIRECTORY if accessing objects in pack files is # generally faster on your platform than accessing the working directory. # @@@ -334,13 -339,6 +331,13 @@@ # Define DEFAULT_HELP_FORMAT to "man", "info" or "html" # (defaults to "man") if you want to have a different default when # "git help" is called without a parameter specifying the format. +# +# Define TEST_GIT_INDEX_VERSION to 2, 3 or 4 to run the test suite +# with a different indexfile format version. If it isn't set the index +# file format used is index-v[23]. +# +# Define GMTIME_UNRELIABLE_ERRORS if your gmtime() function does not +# return NULL when it receives a bogus time_t. GIT-VERSION-FILE: FORCE @$(SHELL_PATH) ./GIT-VERSION-GEN @@@ -554,7 -552,6 +551,7 @@@ TEST_PROGRAMS_NEED_X += test-dat TEST_PROGRAMS_NEED_X += test-delta TEST_PROGRAMS_NEED_X += test-dump-cache-tree TEST_PROGRAMS_NEED_X += test-genrandom +TEST_PROGRAMS_NEED_X += test-hashmap TEST_PROGRAMS_NEED_X += test-index-version TEST_PROGRAMS_NEED_X += test-line-buffer TEST_PROGRAMS_NEED_X += test-match-trees @@@ -663,8 -660,6 +660,8 @@@ LIB_H += diff. LIB_H += diffcore.h LIB_H += dir.h LIB_H += exec_cmd.h +LIB_H += ewah/ewok.h +LIB_H += ewah/ewok_rlw.h LIB_H += fetch-pack.h LIB_H += fmt-merge-msg.h LIB_H += fsck.h @@@ -673,7 -668,7 +670,7 @@@ LIB_H += git-compat-util. LIB_H += gpg-interface.h LIB_H += graph.h LIB_H += grep.h -LIB_H += hash.h +LIB_H += hashmap.h LIB_H += help.h LIB_H += http.h LIB_H += kwset.h @@@ -692,10 -687,8 +689,10 @@@ LIB_H += notes-merge. LIB_H += notes-utils.h LIB_H += notes.h LIB_H += object.h +LIB_H += pack-objects.h LIB_H += pack-revindex.h LIB_H += pack.h +LIB_H += pack-bitmap.h LIB_H += parse-options.h LIB_H += patch-ids.h LIB_H += pathspec.h @@@ -799,10 -792,6 +796,10 @@@ LIB_OBJS += dir. LIB_OBJS += editor.o LIB_OBJS += entry.o LIB_OBJS += environment.o +LIB_OBJS += ewah/bitmap.o +LIB_OBJS += ewah/ewah_bitmap.o +LIB_OBJS += ewah/ewah_io.o +LIB_OBJS += ewah/ewah_rlw.o LIB_OBJS += exec_cmd.o LIB_OBJS += fetch-pack.o LIB_OBJS += fsck.o @@@ -810,7 -799,7 +807,7 @@@ LIB_OBJS += gettext. LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o LIB_OBJS += grep.o -LIB_OBJS += hash.o +LIB_OBJS += hashmap.o LIB_OBJS += help.o LIB_OBJS += hex.o LIB_OBJS += ident.o @@@ -834,10 -823,7 +831,10 @@@ LIB_OBJS += notes-cache. LIB_OBJS += notes-merge.o LIB_OBJS += notes-utils.o LIB_OBJS += object.o +LIB_OBJS += pack-bitmap.o +LIB_OBJS += pack-bitmap-write.o LIB_OBJS += pack-check.o +LIB_OBJS += pack-objects.o LIB_OBJS += pack-revindex.o LIB_OBJS += pack-write.o LIB_OBJS += pager.o @@@ -895,7 -881,6 +892,7 @@@ LIB_OBJS += userdiff. LIB_OBJS += utf8.o LIB_OBJS += varint.o LIB_OBJS += version.o +LIB_OBJS += versioncmp.o LIB_OBJS += walker.o LIB_OBJS += wildmatch.o LIB_OBJS += wrapper.o @@@ -1283,6 -1268,20 +1280,6 @@@ endi ifdef NO_STRTOULL COMPAT_CFLAGS += -DNO_STRTOULL endif -ifdef NO_FNMATCH - COMPAT_CFLAGS += -Icompat/fnmatch - COMPAT_CFLAGS += -DNO_FNMATCH - COMPAT_OBJS += compat/fnmatch/fnmatch.o -else -ifdef NO_FNMATCH_CASEFOLD - COMPAT_CFLAGS += -Icompat/fnmatch - COMPAT_CFLAGS += -DNO_FNMATCH_CASEFOLD - COMPAT_OBJS += compat/fnmatch/fnmatch.o -endif -endif -ifndef NO_WILDMATCH - COMPAT_CFLAGS += -DUSE_WILDMATCH -endif ifdef NO_SETENV COMPAT_CFLAGS += -DNO_SETENV COMPAT_OBJS += compat/setenv.o @@@ -1339,10 -1338,6 +1336,6 @@@ endi ifdef NO_PREAD COMPAT_CFLAGS += -DNO_PREAD COMPAT_OBJS += compat/pread.o - NO_THREAD_SAFE_PREAD = YesPlease - endif - ifdef NO_THREAD_SAFE_PREAD - BASIC_CFLAGS += -DNO_THREAD_SAFE_PREAD endif ifdef NO_FAST_WORKING_DIRECTORY BASIC_CFLAGS += -DNO_FAST_WORKING_DIRECTORY @@@ -1492,11 -1487,6 +1485,11 @@@ ifneq (,$(XDL_FAST_HASH) BASIC_CFLAGS += -DXDL_FAST_HASH endif +ifdef GMTIME_UNRELIABLE_ERRORS + COMPAT_OBJS += compat/gmtime.o + BASIC_CFLAGS += -DGMTIME_UNRELIABLE_ERRORS +endif + ifeq ($(TCLTK_PATH),) NO_TCLTK = NoThanks endif @@@ -2102,7 -2092,7 +2095,7 @@@ pdf XGETTEXT_FLAGS = \ --force-po \ - --add-comments \ + --add-comments=TRANSLATORS: \ --msgid-bugs-address="Git Mailing List " \ --from-code=UTF-8 XGETTEXT_FLAGS_C = $(XGETTEXT_FLAGS) --language=C \ @@@ -2225,9 -2215,6 +2218,9 @@@ endi ifdef GIT_PERF_MAKE_OPTS @echo GIT_PERF_MAKE_OPTS=\''$(subst ','\'',$(subst ','\'',$(GIT_PERF_MAKE_OPTS)))'\' >>$@ endif +ifdef TEST_GIT_INDEX_VERSION + @echo TEST_GIT_INDEX_VERSION=\''$(subst ','\'',$(subst ','\'',$(TEST_GIT_INDEX_VERSION)))'\' >>$@ +endif ### Detect Python interpreter path changes ifndef NO_PYTHON @@@ -2485,9 -2472,8 +2478,9 @@@ profile-clean $(RM) $(addsuffix *.gcno,$(addprefix $(PROFILE_DIR)/, $(object_dirs))) clean: profile-clean coverage-clean - $(RM) *.o *.res block-sha1/*.o ppc/*.o compat/*.o compat/*/*.o xdiff/*.o vcs-svn/*.o \ - builtin/*.o $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) + $(RM) *.o *.res block-sha1/*.o ppc/*.o compat/*.o compat/*/*.o + $(RM) xdiff/*.o vcs-svn/*.o ewah/*.o builtin/*.o + $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X $(RM) $(TEST_PROGRAMS) $(NO_INSTALL) $(RM) -r bin-wrappers $(dep_dirs) diff --combined builtin/index-pack.c index b9f6e12c0e,ed489bbe9a..de35960d99 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@@ -40,17 -40,13 +40,13 @@@ struct base_data int ofs_first, ofs_last; }; - #if !defined(NO_PTHREADS) && defined(NO_THREAD_SAFE_PREAD) - /* pread() emulation is not thread-safe. Disable threading. */ - #define NO_PTHREADS - #endif - struct thread_local { #ifndef NO_PTHREADS pthread_t thread; #endif struct base_data *base_cache; size_t base_cache_used; + int pack_fd; }; /* @@@ -91,7 -87,8 +87,8 @@@ static off_t consumed_bytes static unsigned deepest_delta; static git_SHA_CTX input_ctx; static uint32_t input_crc32; - static int input_fd, output_fd, pack_fd; + static int input_fd, output_fd; + static const char *curr_pack; #ifndef NO_PTHREADS @@@ -134,6 -131,7 +131,7 @@@ static inline void unlock_mutex(pthread */ static void init_thread(void) { + int i; init_recursive_mutex(&read_mutex); pthread_mutex_init(&counter_mutex, NULL); pthread_mutex_init(&work_mutex, NULL); @@@ -141,11 -139,18 +139,18 @@@ pthread_mutex_init(&deepest_delta_mutex, NULL); pthread_key_create(&key, NULL); thread_data = xcalloc(nr_threads, sizeof(*thread_data)); + for (i = 0; i < nr_threads; i++) { + thread_data[i].pack_fd = open(curr_pack, O_RDONLY); + if (thread_data[i].pack_fd == -1) + die_errno(_("unable to open %s"), curr_pack); + } + threads_active = 1; } static void cleanup_thread(void) { + int i; if (!threads_active) return; threads_active = 0; @@@ -154,6 -159,8 +159,8 @@@ pthread_mutex_destroy(&work_mutex); if (show_stat) pthread_mutex_destroy(&deepest_delta_mutex); + for (i = 0; i < nr_threads; i++) + close(thread_data[i].pack_fd); pthread_key_delete(key); free(thread_data); } @@@ -288,13 -295,13 +295,13 @@@ static const char *open_pack_file(cons output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600); if (output_fd < 0) die_errno(_("unable to create '%s'"), pack_name); - pack_fd = output_fd; + nothread_data.pack_fd = output_fd; } else { input_fd = open(pack_name, O_RDONLY); if (input_fd < 0) die_errno(_("cannot open packfile '%s'"), pack_name); output_fd = -1; - pack_fd = input_fd; + nothread_data.pack_fd = input_fd; } git_SHA1_Init(&input_ctx); return pack_name; @@@ -542,7 -549,7 +549,7 @@@ static void *unpack_data(struct object_ do { ssize_t n = (len < 64*1024) ? len : 64*1024; - n = pread(pack_fd, inbuf, n, from); + n = pread(get_thread_data()->pack_fd, inbuf, n, from); if (n < 0) die_errno(_("cannot pread pack file")); if (!n) @@@ -1291,7 -1298,7 +1298,7 @@@ static void final(const char *final_pac if (keep_fd < 0) { if (errno != EEXIST) die_errno(_("cannot write keep file '%s'"), - keep_name); + keep_name ? keep_name : name); } else { if (keep_msg_len > 0) { write_or_die(keep_fd, keep_msg, keep_msg_len); @@@ -1299,7 -1306,7 +1306,7 @@@ } if (close(keep_fd) != 0) die_errno(_("cannot close written keep file '%s'"), - keep_name); + keep_name ? keep_name : name); report = "keep"; } } @@@ -1490,7 -1497,7 +1497,7 @@@ static void show_pack_info(int stat_onl int cmd_index_pack(int argc, const char **argv, const char *prefix) { int i, fix_thin_pack = 0, verify = 0, stat_only = 0; - const char *curr_pack, *curr_index; + const char *curr_index; const char *index_name = NULL, *pack_name = NULL; const char *keep_name = NULL, *keep_msg = NULL; char *index_name_buf = NULL, *keep_name_buf = NULL; @@@ -1502,7 -1509,7 +1509,7 @@@ if (argc == 2 && !strcmp(argv[1], "-h")) usage(index_pack_usage); - read_replace_refs = 0; + check_replace_refs = 0; reset_pack_idx_option(&opts); git_config(git_index_pack_config, &opts); diff --combined config.mak.uname index 23a8803656,db5c249b4c..7846bd7657 --- a/config.mak.uname +++ b/config.mak.uname @@@ -108,6 -108,7 +108,6 @@@ ifeq ($(uname_S),SunOS NO_MKDTEMP = YesPlease NO_MKSTEMPS = YesPlease NO_REGEX = YesPlease - NO_FNMATCH_CASEFOLD = YesPlease NO_MSGFMT_EXTENDED_OPTIONS = YesPlease HAVE_DEV_TTY = YesPlease ifeq ($(uname_R),5.6) @@@ -157,7 -158,6 +157,6 @@@ ifeq ($(uname_O),Cygwin NO_SYMLINK_HEAD = YesPlease NO_IPV6 = YesPlease OLD_ICONV = UnfortunatelyYes - NO_THREAD_SAFE_PREAD = YesPlease # There are conflicting reports about this. # On some boxes NO_MMAP is needed, and not so elsewhere. # Try commenting this out if you suspect MMAP is more efficient @@@ -187,7 -187,6 +186,7 @@@ ifeq ($(uname_S),FreeBSD endif PYTHON_PATH = /usr/local/bin/python HAVE_PATHS_H = YesPlease + GMTIME_UNRELIABLE_ERRORS = UnfortunatelyYes endif ifeq ($(uname_S),OpenBSD) NO_STRCASESTR = YesPlease @@@ -259,6 -258,7 +258,6 @@@ ifeq ($(uname_S),IRIX # issue, comment out the NO_MMAP statement. NO_MMAP = YesPlease NO_REGEX = YesPlease - NO_FNMATCH_CASEFOLD = YesPlease SNPRINTF_RETURNS_BOGUS = YesPlease SHELL_PATH = /usr/gnu/bin/bash NEEDS_LIBGEN = YesPlease @@@ -278,6 -278,7 +277,6 @@@ ifeq ($(uname_S),IRIX64 # issue, comment out the NO_MMAP statement. NO_MMAP = YesPlease NO_REGEX = YesPlease - NO_FNMATCH_CASEFOLD = YesPlease SNPRINTF_RETURNS_BOGUS = YesPlease SHELL_PATH = /usr/gnu/bin/bash NEEDS_LIBGEN = YesPlease @@@ -294,6 -295,7 +293,6 @@@ ifeq ($(uname_S),HP-UX NO_UNSETENV = YesPlease NO_HSTRERROR = YesPlease NO_SYS_SELECT_H = YesPlease - NO_FNMATCH_CASEFOLD = YesPlease SNPRINTF_RETURNS_BOGUS = YesPlease NO_NSEC = YesPlease ifeq ($(uname_R),B.11.00) @@@ -324,6 -326,7 +323,6 @@@ ifeq ($(uname_S),Windows NO_UNSETENV = YesPlease NO_STRCASESTR = YesPlease NO_STRLCPY = YesPlease - NO_FNMATCH = YesPlease NO_MEMMEM = YesPlease # NEEDS_LIBICONV = YesPlease NO_ICONV = YesPlease @@@ -332,6 -335,7 +331,6 @@@ NO_MKSTEMPS = YesPlease SNPRINTF_RETURNS_BOGUS = YesPlease NO_SVN_TESTS = YesPlease - NO_PERL_MAKEMAKER = YesPlease RUNTIME_PREFIX = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease NO_NSEC = YesPlease @@@ -340,6 -344,7 +339,6 @@@ UNRELIABLE_FSTAT = UnfortunatelyYes OBJECT_CREATION_USES_RENAMES = UnfortunatelyNeedsTo NO_REGEX = YesPlease - NO_CURL = YesPlease NO_GETTEXT = YesPlease NO_PYTHON = YesPlease BLK_SHA1 = YesPlease @@@ -359,7 -364,7 +358,7 @@@ compat/win32/dirent.o COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DNOGDI -DHAVE_STRING_H -DHAVE_ALLOCA_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -SUBSYSTEM:CONSOLE -NODEFAULTLIB:MSVCRT.lib - EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib + EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj PTHREAD_LIBS = lib = ifndef DEBUG @@@ -383,11 -388,13 +382,11 @@@ ifeq ($(uname_S),Interix NO_INET_NTOP = YesPlease NO_INET_PTON = YesPlease NO_SOCKADDR_STORAGE = YesPlease - NO_FNMATCH_CASEFOLD = YesPlease endif ifeq ($(uname_R),5.2) NO_INET_NTOP = YesPlease NO_INET_PTON = YesPlease NO_SOCKADDR_STORAGE = YesPlease - NO_FNMATCH_CASEFOLD = YesPlease endif endif ifeq ($(uname_S),Minix) @@@ -432,6 -439,7 +431,6 @@@ ifeq ($(uname_S),NONSTOP_KERNEL NO_D_TYPE_IN_DIRENT = YesPlease NO_HSTRERROR = YesPlease NO_STRCASESTR = YesPlease - NO_FNMATCH_CASEFOLD = YesPlease NO_MEMMEM = YesPlease NO_STRLCPY = YesPlease NO_SETENV = YesPlease @@@ -475,6 -483,7 +474,6 @@@ ifneq (,$(findstring MINGW,$(uname_S)) NO_UNSETENV = YesPlease NO_STRCASESTR = YesPlease NO_STRLCPY = YesPlease - NO_FNMATCH = YesPlease NO_MEMMEM = YesPlease NEEDS_LIBICONV = YesPlease NO_STRTOUMAX = YesPlease @@@ -528,6 -537,7 +527,6 @@@ ifeq ($(uname_S),QNX EXPAT_NEEDS_XMLPARSE_H = YesPlease HAVE_STRINGS_H = YesPlease NEEDS_SOCKET = YesPlease - NO_FNMATCH_CASEFOLD = YesPlease NO_GETPAGESIZE = YesPlease NO_ICONV = YesPlease NO_MEMMEM = YesPlease