From: Junio C Hamano Date: Mon, 14 May 2012 18:50:40 +0000 (-0700) Subject: Merge branch 'nd/threaded-index-pack' X-Git-Tag: v1.7.11-rc0~31 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/cc13431a49800a6a1d2b7ff0b94f67da0fecdbab?ds=inline;hp=-c Merge branch 'nd/threaded-index-pack' Enables threading in index-pack to resolve base data in parallel. By Nguyễn Thái Ngọc Duy (3) and Ramsay Jones (1) * nd/threaded-index-pack: index-pack: disable threading if NO_PREAD is defined index-pack: support multithreaded delta resolving index-pack: restructure pack processing into three main functions compat/win32/pthread.h: Add an pthread_key_delete() implementation --- cc13431a49800a6a1d2b7ff0b94f67da0fecdbab diff --combined Makefile index 3eda1e8c68,e41955ff95..96ebcf9830 --- a/Makefile +++ b/Makefile @@@ -247,9 -247,6 +247,9 @@@ all: # Define NO_CROSS_DIRECTORY_HARDLINKS if you plan to distribute the installed # programs as a tar, where bin/ and libexec/ might be on different file systems. # +# Define NO_INSTALL_HARDLINKS if you prefer to use either symbolic links or +# copies to install built-in git commands e.g. git-cat-file. +# # Define USE_NED_ALLOCATOR if you want to replace the platforms default # memory allocators with the nedmalloc allocator written by Niall Douglas. # @@@ -291,11 -288,6 +291,11 @@@ # dependency rules. # # Define NATIVE_CRLF if your platform uses CRLF for line endings. +# +# Define XDL_FAST_HASH to use an alternative line-hashing method in +# the diff algorithm. It gives a nice speedup if your processor has +# fast unaligned word loads. Does NOT work on big-endian systems! +# Enabled by default on x86_64. GIT-VERSION-FILE: FORCE @$(SHELL_PATH) ./GIT-VERSION-GEN @@@ -394,7 -386,6 +394,7 @@@ XDIFF_OBJS VCSSVN_H = VCSSVN_OBJS = VCSSVN_TEST_OBJS = +MISC_H = EXTRA_CPPFLAGS = LIB_H = LIB_OBJS = @@@ -449,7 -440,6 +449,7 @@@ SCRIPT_PERL += git-send-email.per SCRIPT_PERL += git-svn.perl SCRIPT_PYTHON += git-remote-testgit.py +SCRIPT_PYTHON += git-p4.py SCRIPTS = $(patsubst %.sh,%,$(SCRIPT_SH)) \ $(patsubst %.perl,%,$(SCRIPT_PERL)) \ @@@ -464,15 -454,15 +464,15 @@@ EXTRA_PROGRAMS # ... and all the rest that could be moved out of bindir to gitexecdir PROGRAMS += $(EXTRA_PROGRAMS) +PROGRAM_OBJS += credential-store.o PROGRAM_OBJS += daemon.o PROGRAM_OBJS += fast-import.o +PROGRAM_OBJS += http-backend.o PROGRAM_OBJS += imap-send.o +PROGRAM_OBJS += sh-i18n--envsubst.o PROGRAM_OBJS += shell.o PROGRAM_OBJS += show-index.o PROGRAM_OBJS += upload-pack.o -PROGRAM_OBJS += http-backend.o -PROGRAM_OBJS += sh-i18n--envsubst.o -PROGRAM_OBJS += credential-store.o # Binary suffix, set to .exe for Windows builds X = @@@ -485,17 -475,15 +485,17 @@@ TEST_PROGRAMS_NEED_X += test-ctyp TEST_PROGRAMS_NEED_X += test-date TEST_PROGRAMS_NEED_X += test-delta TEST_PROGRAMS_NEED_X += test-dump-cache-tree -TEST_PROGRAMS_NEED_X += test-scrap-cache-tree TEST_PROGRAMS_NEED_X += test-genrandom TEST_PROGRAMS_NEED_X += test-index-version TEST_PROGRAMS_NEED_X += test-line-buffer TEST_PROGRAMS_NEED_X += test-match-trees +TEST_PROGRAMS_NEED_X += test-mergesort TEST_PROGRAMS_NEED_X += test-mktemp TEST_PROGRAMS_NEED_X += test-parse-options TEST_PROGRAMS_NEED_X += test-path-utils +TEST_PROGRAMS_NEED_X += test-revision-walking TEST_PROGRAMS_NEED_X += test-run-command +TEST_PROGRAMS_NEED_X += test-scrap-cache-tree TEST_PROGRAMS_NEED_X += test-sha1 TEST_PROGRAMS_NEED_X += test-sigchain TEST_PROGRAMS_NEED_X += test-subprocess @@@ -555,36 -543,6 +555,36 @@@ LIB_FILE=libgit. XDIFF_LIB=xdiff/lib.a VCSSVN_LIB=vcs-svn/lib.a +XDIFF_H += xdiff/xinclude.h +XDIFF_H += xdiff/xmacros.h +XDIFF_H += xdiff/xdiff.h +XDIFF_H += xdiff/xtypes.h +XDIFF_H += xdiff/xutils.h +XDIFF_H += xdiff/xprepare.h +XDIFF_H += xdiff/xdiffi.h +XDIFF_H += xdiff/xemit.h + +VCSSVN_H += vcs-svn/line_buffer.h +VCSSVN_H += vcs-svn/sliding_window.h +VCSSVN_H += vcs-svn/repo_tree.h +VCSSVN_H += vcs-svn/fast_export.h +VCSSVN_H += vcs-svn/svndiff.h +VCSSVN_H += vcs-svn/svndump.h + +MISC_H += bisect.h +MISC_H += branch.h +MISC_H += bundle.h +MISC_H += common-cmds.h +MISC_H += fetch-pack.h +MISC_H += reachable.h +MISC_H += send-pack.h +MISC_H += shortlog.h +MISC_H += tar.h +MISC_H += thread-utils.h +MISC_H += url.h +MISC_H += walker.h +MISC_H += wt-status.h + LIB_H += advice.h LIB_H += archive.h LIB_H += argv-array.h @@@ -601,18 -559,18 +601,18 @@@ LIB_H += compat/cygwin. LIB_H += compat/mingw.h LIB_H += compat/obstack.h LIB_H += compat/terminal.h +LIB_H += compat/win32/dirent.h +LIB_H += compat/win32/poll.h LIB_H += compat/win32/pthread.h LIB_H += compat/win32/syslog.h -LIB_H += compat/win32/poll.h -LIB_H += compat/win32/dirent.h LIB_H += connected.h LIB_H += convert.h LIB_H += credential.h LIB_H += csum-file.h LIB_H += decorate.h LIB_H += delta.h -LIB_H += diffcore.h LIB_H += diff.h +LIB_H += diffcore.h LIB_H += dir.h LIB_H += exec_cmd.h LIB_H += fmt-merge-msg.h @@@ -632,7 -590,6 +632,7 @@@ LIB_H += log-tree. LIB_H += mailmap.h LIB_H += merge-file.h LIB_H += merge-recursive.h +LIB_H += mergesort.h LIB_H += notes.h LIB_H += notes-cache.h LIB_H += notes-merge.h @@@ -670,7 -627,6 +670,7 @@@ LIB_H += tree-walk. LIB_H += unpack-trees.h LIB_H += userdiff.h LIB_H += utf8.h +LIB_H += varint.h LIB_H += xdiff-interface.h LIB_H += xdiff/xdiff.h @@@ -691,7 -647,6 +691,7 @@@ LIB_OBJS += bulk-checkin. LIB_OBJS += bundle.o LIB_OBJS += cache-tree.o LIB_OBJS += color.o +LIB_OBJS += column.o LIB_OBJS += combine-diff.o LIB_OBJS += commit.o LIB_OBJS += compat/obstack.o @@@ -721,8 -676,8 +721,8 @@@ LIB_OBJS += entry. LIB_OBJS += environment.o LIB_OBJS += exec_cmd.o LIB_OBJS += fsck.o -LIB_OBJS += gpg-interface.o LIB_OBJS += gettext.o +LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o LIB_OBJS += grep.o LIB_OBJS += hash.o @@@ -739,7 -694,6 +739,7 @@@ LIB_OBJS += mailmap. LIB_OBJS += match-trees.o LIB_OBJS += merge-file.o LIB_OBJS += merge-recursive.o +LIB_OBJS += mergesort.o LIB_OBJS += name-hash.o LIB_OBJS += notes.o LIB_OBJS += notes-cache.o @@@ -771,9 -725,9 +771,9 @@@ LIB_OBJS += rerere. LIB_OBJS += resolve-undo.o LIB_OBJS += revision.o LIB_OBJS += run-command.o +LIB_OBJS += sequencer.o LIB_OBJS += server-info.o LIB_OBJS += setup.o -LIB_OBJS += sequencer.o LIB_OBJS += sha1-array.o LIB_OBJS += sha1-lookup.o LIB_OBJS += sha1_file.o @@@ -798,7 -752,6 +798,7 @@@ LIB_OBJS += url. LIB_OBJS += usage.o LIB_OBJS += userdiff.o LIB_OBJS += utf8.o +LIB_OBJS += varint.o LIB_OBJS += walker.o LIB_OBJS += wrapper.o LIB_OBJS += write_or_die.o @@@ -822,7 -775,6 +822,7 @@@ BUILTIN_OBJS += builtin/checkout-index. BUILTIN_OBJS += builtin/checkout.o BUILTIN_OBJS += builtin/clean.o BUILTIN_OBJS += builtin/clone.o +BUILTIN_OBJS += builtin/column.o BUILTIN_OBJS += builtin/commit-tree.o BUILTIN_OBJS += builtin/commit.o BUILTIN_OBJS += builtin/config.o @@@ -912,9 -864,6 +912,9 @@@ EXTLIBS # because maintaining the nesting to match is a pain. If # we had "elif" things would have been much nicer... +ifeq ($(uname_M),x86_64) + XDL_FAST_HASH = YesPlease +endif ifeq ($(uname_S),OSF1) # Need this for u_short definitions et al BASIC_CFLAGS += -D_OSF_SOURCE @@@ -1788,10 -1737,6 +1788,10 @@@ ifndef NO_MSGFMT_EXTENDED_OPTION MSGFMT += --check --statistics endif +ifneq (,$(XDL_FAST_HASH)) + BASIC_CFLAGS += -DXDL_FAST_HASH +endif + ifeq ($(TCLTK_PATH),) NO_TCLTK=NoThanks endif @@@ -1838,10 -1783,6 +1838,10 @@@ ifdef ASCIIDOC export ASCIIDOC7 endif +ifdef NO_INSTALL_HARDLINKS + export NO_INSTALL_HARDLINKS +endif + ### profile feedback build # @@@ -1908,13 -1849,6 +1908,13 @@@ DEFAULT_PAGER_CQ_SQ = $(subst ','\'',$( BASIC_CFLAGS += -DDEFAULT_PAGER='$(DEFAULT_PAGER_CQ_SQ)' endif +ifdef SHELL_PATH +SHELL_PATH_CQ = "$(subst ",\",$(subst \,\\,$(SHELL_PATH)))" +SHELL_PATH_CQ_SQ = $(subst ','\'',$(SHELL_PATH_CQ)) + +BASIC_CFLAGS += -DSHELL_PATH='$(SHELL_PATH_CQ_SQ)' +endif + ALL_CFLAGS += $(BASIC_CFLAGS) ALL_LDFLAGS += $(BASIC_LDFLAGS) @@@ -2226,19 -2160,34 +2226,19 @@@ builtin/branch.o builtin/checkout.o bui builtin/bundle.o bundle.o transport.o: bundle.h builtin/bisect--helper.o builtin/rev-list.o bisect.o: bisect.h builtin/clone.o builtin/fetch-pack.o transport.o: fetch-pack.h - builtin/grep.o builtin/pack-objects.o transport-helper.o thread-utils.o: thread-utils.h + builtin/index-pack.o builtin/grep.o builtin/pack-objects.o transport-helper.o thread-utils.o: thread-utils.h builtin/send-pack.o transport.o: send-pack.h builtin/log.o builtin/shortlog.o: shortlog.h builtin/prune.o builtin/reflog.o reachable.o: reachable.h builtin/commit.o builtin/revert.o wt-status.o: wt-status.h builtin/tar-tree.o archive-tar.o: tar.h connect.o transport.o url.o http-backend.o: url.h +builtin/branch.o builtin/commit.o builtin/tag.o column.o help.o pager.o: column.h http-fetch.o http-walker.o remote-curl.o transport.o walker.o: walker.h http.o http-walker.o http-push.o http-fetch.o remote-curl.o: http.h url.h -XDIFF_H += xdiff/xinclude.h -XDIFF_H += xdiff/xmacros.h -XDIFF_H += xdiff/xdiff.h -XDIFF_H += xdiff/xtypes.h -XDIFF_H += xdiff/xutils.h -XDIFF_H += xdiff/xprepare.h -XDIFF_H += xdiff/xdiffi.h -XDIFF_H += xdiff/xemit.h - xdiff-interface.o $(XDIFF_OBJS): $(XDIFF_H) -VCSSVN_H += vcs-svn/line_buffer.h -VCSSVN_H += vcs-svn/sliding_window.h -VCSSVN_H += vcs-svn/repo_tree.h -VCSSVN_H += vcs-svn/fast_export.h -VCSSVN_H += vcs-svn/svndiff.h -VCSSVN_H += vcs-svn/svndump.h - $(VCSSVN_OBJS) $(VCSSVN_TEST_OBJS): $(LIB_H) $(VCSSVN_H) endif @@@ -2309,8 -2258,6 +2309,8 @@@ $(XDIFF_LIB): $(XDIFF_OBJS $(VCSSVN_LIB): $(VCSSVN_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) rcs $@ $(VCSSVN_OBJS) +export DEFAULT_EDITOR DEFAULT_PAGER + doc: $(MAKE) -C Documentation all @@@ -2335,7 -2282,7 +2335,7 @@@ XGETTEXT_FLAGS_C = $(XGETTEXT_FLAGS) -- --keyword=_ --keyword=N_ --keyword="Q_:1,2" XGETTEXT_FLAGS_SH = $(XGETTEXT_FLAGS) --language=Shell XGETTEXT_FLAGS_PERL = $(XGETTEXT_FLAGS) --keyword=__ --language=Perl -LOCALIZED_C := $(C_OBJ:o=c) +LOCALIZED_C := $(C_OBJ:o=c) $(LIB_H) $(XDIFF_H) $(VCSSVN_H) $(MISC_H) LOCALIZED_SH := $(SCRIPT_SH) LOCALIZED_PERL := $(SCRIPT_PERL) @@@ -2581,21 -2528,19 +2581,21 @@@ endi { test "$$bindir/" = "$$execdir/" || \ for p in git$X $(filter $(install_bindir_programs),$(ALL_PROGRAMS)); do \ $(RM) "$$execdir/$$p" && \ - test -z "$(NO_CROSS_DIRECTORY_HARDLINKS)" && \ + test -z "$(NO_INSTALL_HARDLINKS)$(NO_CROSS_DIRECTORY_HARDLINKS)" && \ ln "$$bindir/$$p" "$$execdir/$$p" 2>/dev/null || \ cp "$$bindir/$$p" "$$execdir/$$p" || exit; \ done; \ } && \ for p in $(filter $(install_bindir_programs),$(BUILT_INS)); do \ $(RM) "$$bindir/$$p" && \ + test -z "$(NO_INSTALL_HARDLINKS)" && \ ln "$$bindir/git$X" "$$bindir/$$p" 2>/dev/null || \ ln -s "git$X" "$$bindir/$$p" 2>/dev/null || \ cp "$$bindir/git$X" "$$bindir/$$p" || exit; \ done && \ for p in $(BUILT_INS); do \ $(RM) "$$execdir/$$p" && \ + test -z "$(NO_INSTALL_HARDLINKS)" && \ ln "$$execdir/git$X" "$$execdir/$$p" 2>/dev/null || \ ln -s "git$X" "$$execdir/$$p" 2>/dev/null || \ cp "$$execdir/git$X" "$$execdir/$$p" || exit; \ @@@ -2603,7 -2548,6 +2603,7 @@@ remote_curl_aliases="$(REMOTE_CURL_ALIASES)" && \ for p in $$remote_curl_aliases; do \ $(RM) "$$execdir/$$p" && \ + test -z "$(NO_INSTALL_HARDLINKS)" && \ ln "$$execdir/git-remote-http$X" "$$execdir/$$p" 2>/dev/null || \ ln -s "git-remote-http$X" "$$execdir/$$p" 2>/dev/null || \ cp "$$execdir/git-remote-http$X" "$$execdir/$$p" || exit; \ @@@ -2691,6 -2635,7 +2691,6 @@@ dist-doc distclean: clean $(RM) configure - $(RM) po/git.pot profile-clean: $(RM) $(addsuffix *.gcda,$(addprefix $(PROFILE_DIR)/, $(object_dirs))) diff --combined builtin/index-pack.c index 83555e5635,807ee56f79..dc2cfe6e6f --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@@ -9,6 -9,7 +9,7 @@@ #include "progress.h" #include "fsck.h" #include "exec_cmd.h" + #include "thread-utils.h" static const char index_pack_usage[] = "git index-pack [-v] [-o ] [--keep | --keep=] [--verify] [--strict] ( | --stdin [--fix-thin] [])"; @@@ -38,6 -39,19 +39,19 @@@ struct base_data int ofs_first, ofs_last; }; + #if !defined(NO_PTHREADS) && defined(NO_PREAD) + /* NO_PREAD uses compat/pread.c, which is not thread-safe. Disable threading. */ + #define NO_PTHREADS + #endif + + struct thread_local { + #ifndef NO_PTHREADS + pthread_t thread; + #endif + struct base_data *base_cache; + size_t base_cache_used; + }; + /* * Even if sizeof(union delta_base) == 24 on 64-bit archs, we really want * to memcmp() only the first 20 bytes. @@@ -54,11 -68,11 +68,11 @@@ struct delta_entry static struct object_entry *objects; static struct delta_entry *deltas; - static struct base_data *base_cache; - static size_t base_cache_used; + static struct thread_local nothread_data; static int nr_objects; static int nr_deltas; static int nr_resolved_deltas; + static int nr_threads; static int from_stdin; static int strict; @@@ -75,13 -89,84 +89,84 @@@ static git_SHA_CTX input_ctx static uint32_t input_crc32; static int input_fd, output_fd, pack_fd; + #ifndef NO_PTHREADS + + static struct thread_local *thread_data; + static int nr_dispatched; + static int threads_active; + + static pthread_mutex_t read_mutex; + #define read_lock() lock_mutex(&read_mutex) + #define read_unlock() unlock_mutex(&read_mutex) + + static pthread_mutex_t counter_mutex; + #define counter_lock() lock_mutex(&counter_mutex) + #define counter_unlock() unlock_mutex(&counter_mutex) + + static pthread_mutex_t work_mutex; + #define work_lock() lock_mutex(&work_mutex) + #define work_unlock() unlock_mutex(&work_mutex) + + static pthread_key_t key; + + static inline void lock_mutex(pthread_mutex_t *mutex) + { + if (threads_active) + pthread_mutex_lock(mutex); + } + + static inline void unlock_mutex(pthread_mutex_t *mutex) + { + if (threads_active) + pthread_mutex_unlock(mutex); + } + + /* + * Mutex and conditional variable can't be statically-initialized on Windows. + */ + static void init_thread(void) + { + init_recursive_mutex(&read_mutex); + pthread_mutex_init(&counter_mutex, NULL); + pthread_mutex_init(&work_mutex, NULL); + pthread_key_create(&key, NULL); + thread_data = xcalloc(nr_threads, sizeof(*thread_data)); + threads_active = 1; + } + + static void cleanup_thread(void) + { + if (!threads_active) + return; + threads_active = 0; + pthread_mutex_destroy(&read_mutex); + pthread_mutex_destroy(&counter_mutex); + pthread_mutex_destroy(&work_mutex); + pthread_key_delete(key); + free(thread_data); + } + + #else + + #define read_lock() + #define read_unlock() + + #define counter_lock() + #define counter_unlock() + + #define work_lock() + #define work_unlock() + + #endif + + static int mark_link(struct object *obj, int type, void *data) { if (!obj) return -1; if (type != OBJ_ANY && obj->type != type) - die("object type mismatch at %s", sha1_to_hex(obj->sha1)); + die(_("object type mismatch at %s"), sha1_to_hex(obj->sha1)); obj->flags |= FLAG_LINK; return 0; @@@ -101,7 -186,7 +186,7 @@@ static void check_object(struct object unsigned long size; int type = sha1_object_info(obj->sha1, &size); if (type != obj->type || type <= 0) - die("object of unexpected type"); + die(_("object of unexpected type")); obj->flags |= FLAG_CHECKED; return; } @@@ -138,18 -223,15 +223,18 @@@ static void *fill(int min if (min <= input_len) return input_buffer + input_offset; if (min > sizeof(input_buffer)) - die("cannot fill %d bytes", min); + die(Q_("cannot fill %d byte", + "cannot fill %d bytes", + min), + min); flush(); do { ssize_t ret = xread(input_fd, input_buffer + input_len, sizeof(input_buffer) - input_len); if (ret <= 0) { if (!ret) - die("early EOF"); - die_errno("read error on input"); + die(_("early EOF")); + die_errno(_("read error on input")); } input_len += ret; if (from_stdin) @@@ -161,14 -243,14 +246,14 @@@ static void use(int bytes) { if (bytes > input_len) - die("used more bytes than were available"); + die(_("used more bytes than were available")); input_crc32 = crc32(input_crc32, input_buffer + input_offset, bytes); input_len -= bytes; input_offset += bytes; /* make sure off_t is sufficiently large not to wrap */ if (signed_add_overflows(consumed_bytes, bytes)) - die("pack too large for current definition of off_t"); + die(_("pack too large for current definition of off_t")); consumed_bytes += bytes; } @@@ -184,12 -266,12 +269,12 @@@ static const char *open_pack_file(cons } else output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600); if (output_fd < 0) - die_errno("unable to create '%s'", pack_name); + die_errno(_("unable to create '%s'"), pack_name); pack_fd = output_fd; } else { input_fd = open(pack_name, O_RDONLY); if (input_fd < 0) - die_errno("cannot open packfile '%s'", pack_name); + die_errno(_("cannot open packfile '%s'"), pack_name); output_fd = -1; pack_fd = input_fd; } @@@ -203,7 -285,7 +288,7 @@@ static void parse_pack_header(void /* Header consistency check */ if (hdr->hdr_signature != htonl(PACK_SIGNATURE)) - die("pack signature mismatch"); + die(_("pack signature mismatch")); if (!pack_version_ok(hdr->hdr_version)) die("pack version %"PRIu32" unsupported", ntohl(hdr->hdr_version)); @@@ -223,9 -305,28 +308,28 @@@ static NORETURN void bad_object(unsigne va_start(params, format); vsnprintf(buf, sizeof(buf), format, params); va_end(params); - die("pack has bad object at offset %lu: %s", offset, buf); + die(_("pack has bad object at offset %lu: %s"), offset, buf); } + static inline struct thread_local *get_thread_data(void) + { + #ifndef NO_PTHREADS + if (threads_active) + return pthread_getspecific(key); + assert(!threads_active && + "This should only be reached when all threads are gone"); + #endif + return ¬hread_data; + } + + #ifndef NO_PTHREADS + static void set_thread_data(struct thread_local *data) + { + if (threads_active) + pthread_setspecific(key, data); + } + #endif + static struct base_data *alloc_base_data(void) { struct base_data *base = xmalloc(sizeof(struct base_data)); @@@ -240,15 -341,16 +344,16 @@@ static void free_base_data(struct base_ if (c->data) { free(c->data); c->data = NULL; - base_cache_used -= c->size; + get_thread_data()->base_cache_used -= c->size; } } static void prune_base_data(struct base_data *retain) { struct base_data *b; - for (b = base_cache; - base_cache_used > delta_base_cache_limit && b; + struct thread_local *data = get_thread_data(); + for (b = data->base_cache; + data->base_cache_used > delta_base_cache_limit && b; b = b->child) { if (b->data && b != retain) free_base_data(b); @@@ -260,12 -362,12 +365,12 @@@ static void link_base_data(struct base_ if (base) base->child = c; else - base_cache = c; + get_thread_data()->base_cache = c; c->base = base; c->child = NULL; if (c->data) - base_cache_used += c->size; + get_thread_data()->base_cache_used += c->size; prune_base_data(c); } @@@ -275,7 -377,7 +380,7 @@@ static void unlink_base_data(struct bas if (base) base->child = NULL; else - base_cache = NULL; + get_thread_data()->base_cache = NULL; free_base_data(c); } @@@ -297,7 -399,7 +402,7 @@@ static void *unpack_entry_data(unsigne use(input_len - stream.avail_in); } while (status == Z_OK); if (stream.total_out != size || status != Z_STREAM_END) - bad_object(offset, "inflate returned %d", status); + bad_object(offset, _("inflate returned %d"), status); git_inflate_end(&stream); return buf; } @@@ -342,7 -444,7 +447,7 @@@ static void *unpack_raw_entry(struct ob while (c & 128) { base_offset += 1; if (!base_offset || MSB(base_offset, 7)) - bad_object(obj->idx.offset, "offset value overflow for delta base object"); + bad_object(obj->idx.offset, _("offset value overflow for delta base object")); p = fill(1); c = *p; use(1); @@@ -350,7 -452,7 +455,7 @@@ } delta_base->offset = obj->idx.offset - base_offset; if (delta_base->offset <= 0 || delta_base->offset >= obj->idx.offset) - bad_object(obj->idx.offset, "delta base offset is out of bound"); + bad_object(obj->idx.offset, _("delta base offset is out of bound")); break; case OBJ_COMMIT: case OBJ_TREE: @@@ -358,7 -460,7 +463,7 @@@ case OBJ_TAG: break; default: - bad_object(obj->idx.offset, "unknown object type %d", obj->type); + bad_object(obj->idx.offset, _("unknown object type %d"), obj->type); } obj->hdr_size = consumed_bytes - obj->idx.offset; @@@ -387,12 -489,9 +492,12 @@@ static void *get_data_from_pack(struct ssize_t n = (len < 64*1024) ? len : 64*1024; n = pread(pack_fd, inbuf, n, from); if (n < 0) - die_errno("cannot pread pack file"); + die_errno(_("cannot pread pack file")); if (!n) - die("premature end of pack file, %lu bytes missing", len); + die(Q_("premature end of pack file, %lu byte missing", + "premature end of pack file, %lu bytes missing", + len), + len); from += n; len -= n; stream.next_in = inbuf; @@@ -402,7 -501,7 +507,7 @@@ /* This has been inflated OK when first encountered, so... */ if (status != Z_STREAM_END || stream.total_out != obj->size) - die("serious inflate inconsistency"); + die(_("serious inflate inconsistency")); git_inflate_end(&stream); free(inbuf); @@@ -467,25 -566,30 +572,30 @@@ static void sha1_object(const void *dat enum object_type type, unsigned char *sha1) { hash_sha1_file(data, size, typename(type), sha1); + read_lock(); if (has_sha1_file(sha1)) { void *has_data; enum object_type has_type; unsigned long has_size; has_data = read_sha1_file(sha1, &has_type, &has_size); + read_unlock(); if (!has_data) - die("cannot read existing object %s", sha1_to_hex(sha1)); + die(_("cannot read existing object %s"), sha1_to_hex(sha1)); if (size != has_size || type != has_type || memcmp(data, has_data, size) != 0) - die("SHA1 COLLISION FOUND WITH %s !", sha1_to_hex(sha1)); + die(_("SHA1 COLLISION FOUND WITH %s !"), sha1_to_hex(sha1)); free(has_data); - } + } else + read_unlock(); + if (strict) { + read_lock(); if (type == OBJ_BLOB) { struct blob *blob = lookup_blob(sha1); if (blob) blob->object.flags |= FLAG_CHECKED; else - die("invalid blob object %s", sha1_to_hex(sha1)); + die(_("invalid blob object %s"), sha1_to_hex(sha1)); } else { struct object *obj; int eaten; @@@ -497,11 -601,11 +607,11 @@@ */ obj = parse_object_buffer(sha1, type, size, buf, &eaten); if (!obj) - die("invalid %s", typename(type)); + die(_("invalid %s"), typename(type)); if (fsck_object(obj, 1, fsck_error_function)) - die("Error in object"); + die(_("Error in object")); if (fsck_walk(obj, mark_link, NULL)) - die("Not all child objects of %s are reachable", sha1_to_hex(obj->sha1)); + die(_("Not all child objects of %s are reachable"), sha1_to_hex(obj->sha1)); if (obj->type == OBJ_TREE) { struct tree *item = (struct tree *) obj; @@@ -513,6 -617,7 +623,7 @@@ } obj->flags |= FLAG_CHECKED; } + read_unlock(); } } @@@ -558,7 -663,7 +669,7 @@@ static void *get_base_data(struct base_ if (!delta_nr) { c->data = get_data_from_pack(obj); c->size = obj->size; - base_cache_used += c->size; + get_thread_data()->base_cache_used += c->size; prune_base_data(c); } for (; delta_nr > 0; delta_nr--) { @@@ -573,8 -678,8 +684,8 @@@ &c->size); free(raw); if (!c->data) - bad_object(obj->idx.offset, "failed to apply delta"); + bad_object(obj->idx.offset, _("failed to apply delta")); - base_cache_used += c->size; + get_thread_data()->base_cache_used += c->size; prune_base_data(c); } free(delta); @@@ -599,10 -704,12 +710,12 @@@ static void resolve_delta(struct object delta_data, delta_obj->size, &result->size); free(delta_data); if (!result->data) - bad_object(delta_obj->idx.offset, "failed to apply delta"); + bad_object(delta_obj->idx.offset, _("failed to apply delta")); sha1_object(result->data, result->size, delta_obj->real_type, delta_obj->idx.sha1); + counter_lock(); nr_resolved_deltas++; + counter_unlock(); } static struct base_data *find_unresolved_deltas_1(struct base_data *base, @@@ -688,22 -795,53 +801,53 @@@ static int compare_delta_entry(const vo objects[delta_b->obj_no].type); } - /* Parse all objects and return the pack content SHA1 hash */ + static void resolve_base(struct object_entry *obj) + { + struct base_data *base_obj = alloc_base_data(); + base_obj->obj = obj; + base_obj->data = NULL; + find_unresolved_deltas(base_obj); + } + + #ifndef NO_PTHREADS + static void *threaded_second_pass(void *data) + { + set_thread_data(data); + for (;;) { + int i; + work_lock(); + display_progress(progress, nr_resolved_deltas); + while (nr_dispatched < nr_objects && + is_delta_type(objects[nr_dispatched].type)) + nr_dispatched++; + if (nr_dispatched >= nr_objects) { + work_unlock(); + break; + } + i = nr_dispatched++; + work_unlock(); + + resolve_base(&objects[i]); + } + return NULL; + } + #endif + + /* + * First pass: + * - find locations of all objects; + * - calculate SHA1 of all non-delta objects; + * - remember base (SHA1 or offset) for all deltas. + */ static void parse_pack_objects(unsigned char *sha1) { int i; struct delta_entry *delta = deltas; struct stat st; - /* - * First pass: - * - find locations of all objects; - * - calculate SHA1 of all non-delta objects; - * - remember base (SHA1 or offset) for all deltas. - */ if (verbose) progress = start_progress( - from_stdin ? "Receiving objects" : "Indexing objects", + from_stdin ? _("Receiving objects") : _("Indexing objects"), nr_objects); for (i = 0; i < nr_objects; i++) { struct object_entry *obj = &objects[i]; @@@ -725,15 -863,28 +869,28 @@@ flush(); git_SHA1_Final(sha1, &input_ctx); if (hashcmp(fill(20), sha1)) - die("pack is corrupted (SHA1 mismatch)"); + die(_("pack is corrupted (SHA1 mismatch)")); use(20); /* If input_fd is a file, we should have reached its end now. */ if (fstat(input_fd, &st)) - die_errno("cannot fstat packfile"); + die_errno(_("cannot fstat packfile")); if (S_ISREG(st.st_mode) && lseek(input_fd, 0, SEEK_CUR) - input_len != st.st_size) - die("pack has junk at the end"); + die(_("pack has junk at the end")); + } + + /* + * Second pass: + * - for all non-delta objects, look if it is used as a base for + * deltas; + * - if used as a base, uncompress the object and apply all deltas, + * recursively checking if the resulting object is used as a base + * for some more deltas. + */ + static void resolve_deltas(void) + { + int i; if (!nr_deltas) return; @@@ -742,29 -893,81 +899,83 @@@ qsort(deltas, nr_deltas, sizeof(struct delta_entry), compare_delta_entry); - /* - * Second pass: - * - for all non-delta objects, look if it is used as a base for - * deltas; - * - if used as a base, uncompress the object and apply all deltas, - * recursively checking if the resulting object is used as a base - * for some more deltas. - */ if (verbose) - progress = start_progress("Resolving deltas", nr_deltas); + progress = start_progress(_("Resolving deltas"), nr_deltas); + + #ifndef NO_PTHREADS + nr_dispatched = 0; + if (nr_threads > 1 || getenv("GIT_FORCE_THREADS")) { + init_thread(); + for (i = 0; i < nr_threads; i++) { + int ret = pthread_create(&thread_data[i].thread, NULL, + threaded_second_pass, thread_data + i); + if (ret) + die("unable to create thread: %s", strerror(ret)); + } + for (i = 0; i < nr_threads; i++) + pthread_join(thread_data[i].thread, NULL); + cleanup_thread(); + return; + } + #endif + for (i = 0; i < nr_objects; i++) { struct object_entry *obj = &objects[i]; - struct base_data *base_obj = alloc_base_data(); if (is_delta_type(obj->type)) continue; - base_obj->obj = obj; - base_obj->data = NULL; - find_unresolved_deltas(base_obj); + resolve_base(obj); display_progress(progress, nr_resolved_deltas); } } + /* + * Third pass: + * - append objects to convert thin pack to full pack if required + * - write the final 20-byte SHA-1 + */ + static void fix_unresolved_deltas(struct sha1file *f, int nr_unresolved); + static void conclude_pack(int fix_thin_pack, const char *curr_pack, unsigned char *pack_sha1) + { + if (nr_deltas == nr_resolved_deltas) { + stop_progress(&progress); + /* Flush remaining pack final 20-byte SHA1. */ + flush(); + return; + } + + if (fix_thin_pack) { + struct sha1file *f; + unsigned char read_sha1[20], tail_sha1[20]; + char msg[48]; + int nr_unresolved = nr_deltas - nr_resolved_deltas; + int nr_objects_initial = nr_objects; + if (nr_unresolved <= 0) - die("confusion beyond insanity"); ++ die(_("confusion beyond insanity")); + objects = xrealloc(objects, + (nr_objects + nr_unresolved + 1) + * sizeof(*objects)); + f = sha1fd(output_fd, curr_pack); + fix_unresolved_deltas(f, nr_unresolved); + sprintf(msg, "completed with %d local objects", + nr_objects - nr_objects_initial); + stop_progress_msg(&progress, msg); + sha1close(f, tail_sha1, 0); + hashcpy(read_sha1, pack_sha1); + fixup_pack_header_footer(output_fd, pack_sha1, + curr_pack, nr_objects, + read_sha1, consumed_bytes-20); + if (hashcmp(read_sha1, tail_sha1) != 0) + die("Unexpected tail checksum for %s " + "(disk corruption?)", curr_pack); + } + if (nr_deltas != nr_resolved_deltas) - die("pack has %d unresolved deltas", ++ die(Q_("pack has %d unresolved delta", ++ "pack has %d unresolved deltas", ++ nr_deltas - nr_resolved_deltas), + nr_deltas - nr_resolved_deltas); + } + static int write_compressed(struct sha1file *f, void *in, unsigned int size) { git_zstream stream; @@@ -784,7 -987,7 +995,7 @@@ } while (status == Z_OK); if (status != Z_STREAM_END) - die("unable to deflate appended object (%d)", status); + die(_("unable to deflate appended object (%d)"), status); size = stream.total_out; git_deflate_end(&stream); return size; @@@ -863,7 -1066,7 +1074,7 @@@ static void fix_unresolved_deltas(struc if (check_sha1_signature(d->base.sha1, base_obj->data, base_obj->size, typename(type))) - die("local object %s is corrupt", sha1_to_hex(d->base.sha1)); + die(_("local object %s is corrupt"), sha1_to_hex(d->base.sha1)); base_obj->obj = append_obj_to_pack(f, d->base.sha1, base_obj->data, base_obj->size, type); find_unresolved_deltas(base_obj); @@@ -887,7 -1090,7 +1098,7 @@@ static void final(const char *final_pac fsync_or_die(output_fd, curr_pack_name); err = close(output_fd); if (err) - die_errno("error while closing pack file"); + die_errno(_("error while closing pack file")); } if (keep_msg) { @@@ -900,7 -1103,7 +1111,7 @@@ if (keep_fd < 0) { if (errno != EEXIST) - die_errno("cannot write keep file '%s'", + die_errno(_("cannot write keep file '%s'"), keep_name); } else { if (keep_msg_len > 0) { @@@ -908,7 -1111,7 +1119,7 @@@ write_or_die(keep_fd, "\n", 1); } if (close(keep_fd) != 0) - die_errno("cannot close written keep file '%s'", + die_errno(_("cannot close written keep file '%s'"), keep_name); report = "keep"; } @@@ -921,7 -1124,7 +1132,7 @@@ final_pack_name = name; } if (move_temp_to_file(curr_pack_name, final_pack_name)) - die("cannot store pack file"); + die(_("cannot store pack file")); } else if (from_stdin) chmod(final_pack_name, 0444); @@@ -932,7 -1135,7 +1143,7 @@@ final_index_name = name; } if (move_temp_to_file(curr_index_name, final_index_name)) - die("cannot store index file"); + die(_("cannot store index file")); } else chmod(final_index_name, 0444); @@@ -968,6 -1171,18 +1179,18 @@@ static int git_index_pack_config(const die("bad pack.indexversion=%"PRIu32, opts->version); return 0; } + if (!strcmp(k, "pack.threads")) { + nr_threads = git_config_int(k, v); + if (nr_threads < 0) + die("invalid number of threads specified (%d)", + nr_threads); + #ifdef NO_PTHREADS + if (nr_threads != 1) + warning("no threads support, ignoring %s", k); + nr_threads = 1; + #endif + return 0; + } return git_default_config(k, v, cb); } @@@ -1021,9 -1236,9 +1244,9 @@@ static void read_idx_option(struct pack struct packed_git *p = add_packed_git(pack_name, strlen(pack_name), 1); if (!p) - die("Cannot open existing pack file '%s'", pack_name); + die(_("Cannot open existing pack file '%s'"), pack_name); if (open_pack_index(p)) - die("Cannot open existing pack idx file for '%s'", pack_name); + die(_("Cannot open existing pack idx file for '%s'"), pack_name); /* Read the attributes from the existing idx file */ opts->version = p->index_version; @@@ -1070,18 -1285,15 +1293,18 @@@ static void show_pack_info(int stat_onl } if (baseobjects) - printf("non delta: %d object%s\n", - baseobjects, baseobjects > 1 ? "s" : ""); + printf_ln(Q_("non delta: %d object", + "non delta: %d objects", + baseobjects), + baseobjects); for (i = 0; i < deepest_delta; i++) { if (!chain_histogram[i]) continue; - printf("chain length = %d: %lu object%s\n", - i + 1, - chain_histogram[i], - chain_histogram[i] > 1 ? "s" : ""); + printf_ln(Q_("chain length = %d: %lu object", + "chain length = %d: %lu objects", + chain_histogram[i]), + i + 1, + chain_histogram[i]); } } @@@ -1104,7 -1316,7 +1327,7 @@@ int cmd_index_pack(int argc, const cha reset_pack_idx_option(&opts); git_config(git_index_pack_config, &opts); if (prefix && chdir(prefix)) - die("Cannot come back to cwd"); + die(_("Cannot come back to cwd")); for (i = 1; i < argc; i++) { const char *arg = argv[i]; @@@ -1129,6 -1341,17 +1352,17 @@@ keep_msg = ""; } else if (!prefixcmp(arg, "--keep=")) { keep_msg = arg + 7; + } else if (!prefixcmp(arg, "--threads=")) { + char *end; + nr_threads = strtoul(arg+10, &end, 0); + if (!arg[10] || *end || nr_threads < 0) + usage(index_pack_usage); + #ifdef NO_PTHREADS + if (nr_threads != 1) + warning("no threads support, " + "ignoring %s", arg); + nr_threads = 1; + #endif } else if (!prefixcmp(arg, "--pack_header=")) { struct pack_header *hdr; char *c; @@@ -1137,10 -1360,10 +1371,10 @@@ hdr->hdr_signature = htonl(PACK_SIGNATURE); hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10)); if (*c != ',') - die("bad %s", arg); + die(_("bad %s"), arg); hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10)); if (*c) - die("bad %s", arg); + die(_("bad %s"), arg); input_len = sizeof(*hdr); } else if (!strcmp(arg, "-v")) { verbose = 1; @@@ -1152,11 -1375,11 +1386,11 @@@ char *c; opts.version = strtoul(arg + 16, &c, 10); if (opts.version > 2) - die("bad %s", arg); + die(_("bad %s"), arg); if (*c == ',') opts.off32_limit = strtoul(c+1, &c, 0); if (*c || opts.off32_limit & 0x80000000) - die("bad %s", arg); + die(_("bad %s"), arg); } else usage(index_pack_usage); continue; @@@ -1170,11 -1393,11 +1404,11 @@@ if (!pack_name && !from_stdin) usage(index_pack_usage); if (fix_thin_pack && !from_stdin) - die("--fix-thin cannot be used without --stdin"); + die(_("--fix-thin cannot be used without --stdin")); if (!index_name && pack_name) { int len = strlen(pack_name); if (!has_extension(pack_name, ".pack")) - die("packfile name '%s' does not end with '.pack'", + die(_("packfile name '%s' does not end with '.pack'"), pack_name); index_name_buf = xmalloc(len); memcpy(index_name_buf, pack_name, len - 5); @@@ -1184,7 -1407,7 +1418,7 @@@ if (keep_msg && !keep_name && pack_name) { int len = strlen(pack_name); if (!has_extension(pack_name, ".pack")) - die("packfile name '%s' does not end with '.pack'", + die(_("packfile name '%s' does not end with '.pack'"), pack_name); keep_name_buf = xmalloc(len); memcpy(keep_name_buf, pack_name, len - 5); @@@ -1193,54 -1416,29 +1427,29 @@@ } if (verify) { if (!index_name) - die("--verify with no packfile name given"); + die(_("--verify with no packfile name given")); read_idx_option(&opts, index_name); opts.flags |= WRITE_IDX_VERIFY | WRITE_IDX_STRICT; } if (strict) opts.flags |= WRITE_IDX_STRICT; + #ifndef NO_PTHREADS + if (!nr_threads) { + nr_threads = online_cpus(); + /* An experiment showed that more threads does not mean faster */ + if (nr_threads > 3) + nr_threads = 3; + } + #endif + curr_pack = open_pack_file(pack_name); parse_pack_header(); objects = xcalloc(nr_objects + 1, sizeof(struct object_entry)); deltas = xcalloc(nr_objects, sizeof(struct delta_entry)); parse_pack_objects(pack_sha1); - if (nr_deltas == nr_resolved_deltas) { - stop_progress(&progress); - /* Flush remaining pack final 20-byte SHA1. */ - flush(); - } else { - if (fix_thin_pack) { - struct sha1file *f; - unsigned char read_sha1[20], tail_sha1[20]; - char msg[48]; - int nr_unresolved = nr_deltas - nr_resolved_deltas; - int nr_objects_initial = nr_objects; - if (nr_unresolved <= 0) - die(_("confusion beyond insanity")); - objects = xrealloc(objects, - (nr_objects + nr_unresolved + 1) - * sizeof(*objects)); - f = sha1fd(output_fd, curr_pack); - fix_unresolved_deltas(f, nr_unresolved); - sprintf(msg, "completed with %d local objects", - nr_objects - nr_objects_initial); - stop_progress_msg(&progress, msg); - sha1close(f, tail_sha1, 0); - hashcpy(read_sha1, pack_sha1); - fixup_pack_header_footer(output_fd, pack_sha1, - curr_pack, nr_objects, - read_sha1, consumed_bytes-20); - if (hashcmp(read_sha1, tail_sha1) != 0) - die("Unexpected tail checksum for %s " - "(disk corruption?)", curr_pack); - } - if (nr_deltas != nr_resolved_deltas) - die(Q_("pack has %d unresolved delta", - "pack has %d unresolved deltas", - nr_deltas - nr_resolved_deltas), - nr_deltas - nr_resolved_deltas); - } + resolve_deltas(); + conclude_pack(fix_thin_pack, curr_pack, pack_sha1); free(deltas); if (strict) check_objects();