From: Junio C Hamano <gitster@pobox.com>
Date: Mon, 26 Sep 2016 23:09:19 +0000 (-0700)
Subject: Merge branch 'js/regexec-buf'
X-Git-Tag: v2.11.0-rc0~110
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/6a67695268562f67babdb7d5195c8a43cc4015fa?hp=-c

Merge branch 'js/regexec-buf'

Some codepaths in "git diff" used regexec(3) on a buffer that was
mmap(2)ed, which may not have a terminating NUL, leading to a read
beyond the end of the mapped region.  This was fixed by introducing
a regexec_buf() helper that takes a <ptr,len> pair with REG_STARTEND
extension.

* js/regexec-buf:
regex: use regexec_buf()
regex: add regexec_buf() that can work on a non NUL-terminated string
regex: -G<pattern> feeds a non NUL-terminated string to regexec() and fails
---

6a67695268562f67babdb7d5195c8a43cc4015fa
diff --combined Makefile
index e8b060a235,c6dc42d067..1aad150b34
--- a/Makefile
+++ b/Makefile
@@@ -296,12 -296,8 +296,13 @@@ all:
  # Define USE_NED_ALLOCATOR if you want to replace the platforms default
  # memory allocators with the nedmalloc allocator written by Niall Douglas.
  #
 +# Define OVERRIDE_STRDUP to override the libc version of strdup(3).
 +# This is necessary when using a custom allocator in order to avoid
 +# crashes due to allocation and free working on different 'heaps'.
 +# It's defined automatically if USE_NED_ALLOCATOR is set.
 +#
- # Define NO_REGEX if you have no or inferior regex support in your C library.
+ # Define NO_REGEX if your C library lacks regex support with REG_STARTEND
+ # feature.
  #
  # Define HAVE_DEV_TTY if your system can open /dev/tty to interact with the
  # user.
@@@ -356,12 -352,9 +357,12 @@@
  # Define GMTIME_UNRELIABLE_ERRORS if your gmtime() function does not
  # return NULL when it receives a bogus time_t.
  #
 -# Define HAVE_CLOCK_GETTIME if your platform has clock_gettime in librt.
 +# Define HAVE_CLOCK_GETTIME if your platform has clock_gettime.
 +#
 +# Define HAVE_CLOCK_MONOTONIC if your platform has CLOCK_MONOTONIC.
  #
 -# Define HAVE_CLOCK_MONOTONIC if your platform has CLOCK_MONOTONIC in librt.
 +# Define NEEDS_LIBRT if your platform requires linking with librt (glibc version
 +# before 2.17) for clock_gettime and CLOCK_MONOTONIC.
  #
  # Define USE_PARENS_AROUND_GETTEXT_N to "yes" if your compiler happily
  # compiles the following initialization:
@@@ -375,14 -368,6 +376,14 @@@
  # Define HAVE_BSD_SYSCTL if your platform has a BSD-compatible sysctl function.
  #
  # Define HAVE_GETDELIM if your system has the getdelim() function.
 +#
 +# Define PAGER_ENV to a SP separated VAR=VAL pairs to define
 +# default environment variables to be passed when a pager is spawned, e.g.
 +#
 +#    PAGER_ENV = LESS=FRX LV=-c
 +#
 +# to say "export LESS=FRX (and LV=-c) if the environment variable
 +# LESS (and LV) is not set, respectively".
  
  GIT-VERSION-FILE: FORCE
  	@$(SHELL_PATH) ./GIT-VERSION-GEN
@@@ -391,7 -376,13 +392,7 @@@
  # CFLAGS and LDFLAGS are for the users to override from the command line.
  
  CFLAGS = -g -O2 -Wall
 -LDFLAGS =
 -ALL_CFLAGS = $(CPPFLAGS) $(CFLAGS)
 -ALL_LDFLAGS = $(LDFLAGS)
 -STRIP ?= strip
 -
 -ifdef DEVELOPER
 -CFLAGS += -Werror \
 +DEVELOPER_CFLAGS = -Werror \
  	-Wdeclaration-after-statement \
  	-Wno-format-zero-length \
  	-Wold-style-definition \
@@@ -400,10 -391,7 +401,10 @@@
  	-Wstrict-prototypes \
  	-Wunused \
  	-Wvla
 -endif
 +LDFLAGS =
 +ALL_CFLAGS = $(CPPFLAGS) $(CFLAGS)
 +ALL_LDFLAGS = $(LDFLAGS)
 +STRIP ?= strip
  
  # Create as necessary, replace existing, make ranlib unneeded.
  ARFLAGS = rcs
@@@ -453,6 -441,7 +454,6 @@@ DIFF = dif
  TAR = tar
  FIND = find
  INSTALL = install
 -RPMBUILD = rpmbuild
  TCL_PATH = tclsh
  TCLTK_PATH = wish
  XGETTEXT = xgettext
@@@ -461,7 -450,6 +462,7 @@@ CURL_CONFIG = curl-confi
  PTHREAD_LIBS = -lpthread
  PTHREAD_CFLAGS =
  GCOV = gcov
 +SPATCH = spatch
  
  export TCL_PATH TCLTK_PATH
  
@@@ -634,7 -622,7 +635,7 @@@ TEST_PROGRAMS_NEED_X += test-svn-f
  TEST_PROGRAMS_NEED_X += test-urlmatch-normalization
  TEST_PROGRAMS_NEED_X += test-wildmatch
  
 -TEST_PROGRAMS = $(patsubst %,%$X,$(TEST_PROGRAMS_NEED_X))
 +TEST_PROGRAMS = $(patsubst %,t/helper/%$X,$(TEST_PROGRAMS_NEED_X))
  
  # List built-in command $C whose implementation cmd_$C() is not in
  # builtin/$C.o but is linked in as part of some other command.
@@@ -697,7 -685,6 +698,7 @@@ LIB_OBJS += abspath.
  LIB_OBJS += advice.o
  LIB_OBJS += alias.o
  LIB_OBJS += alloc.o
 +LIB_OBJS += apply.o
  LIB_OBJS += archive.o
  LIB_OBJS += archive-tar.o
  LIB_OBJS += archive-zip.o
@@@ -736,7 -723,6 +737,7 @@@ LIB_OBJS += diff-lib.
  LIB_OBJS += diff-no-index.o
  LIB_OBJS += diff.o
  LIB_OBJS += dir.o
 +LIB_OBJS += dir-iterator.o
  LIB_OBJS += editor.o
  LIB_OBJS += entry.o
  LIB_OBJS += environment.o
@@@ -770,7 -756,6 +771,7 @@@ LIB_OBJS += merge.
  LIB_OBJS += merge-blobs.o
  LIB_OBJS += merge-recursive.o
  LIB_OBJS += mergesort.o
 +LIB_OBJS += mru.o
  LIB_OBJS += name-hash.o
  LIB_OBJS += notes.o
  LIB_OBJS += notes-cache.o
@@@ -802,7 -787,6 +803,7 @@@ LIB_OBJS += read-cache.
  LIB_OBJS += reflog-walk.o
  LIB_OBJS += refs.o
  LIB_OBJS += refs/files-backend.o
 +LIB_OBJS += refs/iterator.o
  LIB_OBJS += ref-filter.o
  LIB_OBJS += remote.o
  LIB_OBJS += replace_object.o
@@@ -960,7 -944,7 +961,7 @@@ BUILTIN_OBJS += builtin/verify-tag.
  BUILTIN_OBJS += builtin/worktree.o
  BUILTIN_OBJS += builtin/write-tree.o
  
 -GITLIBS = $(LIB_FILE) $(XDIFF_LIB)
 +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB)
  EXTLIBS =
  
  GIT_USER_AGENT = git/$(GIT_VERSION)
@@@ -969,10 -953,6 +970,10 @@@ include config.mak.unam
  -include config.mak.autogen
  -include config.mak
  
 +ifdef DEVELOPER
 +CFLAGS += $(DEVELOPER_CFLAGS)
 +endif
 +
  ifndef sysconfdir
  ifeq ($(prefix),/usr)
  sysconfdir = /etc
@@@ -1463,14 -1443,8 +1464,14 @@@ ifdef NATIVE_CRL
  endif
  
  ifdef USE_NED_ALLOCATOR
 -       COMPAT_CFLAGS += -Icompat/nedmalloc
 -       COMPAT_OBJS += compat/nedmalloc/nedmalloc.o
 +	COMPAT_CFLAGS += -Icompat/nedmalloc
 +	COMPAT_OBJS += compat/nedmalloc/nedmalloc.o
 +	OVERRIDE_STRDUP = YesPlease
 +endif
 +
 +ifdef OVERRIDE_STRDUP
 +	COMPAT_CFLAGS += -DOVERRIDE_STRDUP
 +	COMPAT_OBJS += compat/strdup.o
  endif
  
  ifdef GIT_TEST_CMP_USE_COPIED_CONTEXT
@@@ -1492,16 -1466,13 +1493,16 @@@ endi
  
  ifdef HAVE_CLOCK_GETTIME
  	BASIC_CFLAGS += -DHAVE_CLOCK_GETTIME
 -	EXTLIBS += -lrt
  endif
  
  ifdef HAVE_CLOCK_MONOTONIC
  	BASIC_CFLAGS += -DHAVE_CLOCK_MONOTONIC
  endif
  
 +ifdef NEEDS_LIBRT
 +	EXTLIBS += -lrt
 +endif
 +
  ifdef HAVE_BSD_SYSCTL
  	BASIC_CFLAGS += -DHAVE_BSD_SYSCTL
  endif
@@@ -1522,10 -1493,6 +1523,10 @@@ ifeq ($(PYTHON_PATH),
  NO_PYTHON = NoThanks
  endif
  
 +ifndef PAGER_ENV
 +PAGER_ENV = LESS=FRX LV=-c
 +endif
 +
  QUIET_SUBDIR0  = +$(MAKE) -C # space to separate -C and subdir
  QUIET_SUBDIR1  =
  
@@@ -1606,15 -1573,7 +1607,15 @@@ TCLTK_PATH_SQ = $(subst ','\'',$(TCLTK_
  DIFF_SQ = $(subst ','\'',$(DIFF))
  PERLLIB_EXTRA_SQ = $(subst ','\'',$(PERLLIB_EXTRA))
  
 -LIBS = $(GITLIBS) $(EXTLIBS)
 +# We must filter out any object files from $(GITLIBS),
 +# as it is typically used like:
 +#
 +#   foo: foo.o $(GITLIBS)
 +#	$(CC) $(filter %.o,$^) $(LIBS)
 +#
 +# where we use it as a dependency. Since we also pull object files
 +# from the dependency list, that would make each entry appear twice.
 +LIBS = $(filter-out %.o, $(GITLIBS)) $(EXTLIBS)
  
  BASIC_CFLAGS += -DSHA1_HEADER='$(SHA1_HEADER_SQ)' \
  	$(COMPAT_CFLAGS)
@@@ -1655,11 -1614,6 +1656,11 @@@ ifdef DEFAULT_HELP_FORMA
  BASIC_CFLAGS += -DDEFAULT_HELP_FORMAT='"$(DEFAULT_HELP_FORMAT)"'
  endif
  
 +PAGER_ENV_SQ = $(subst ','\'',$(PAGER_ENV))
 +PAGER_ENV_CQ = "$(subst ",\",$(subst \,\\,$(PAGER_ENV)))"
 +PAGER_ENV_CQ_SQ = $(subst ','\'',$(PAGER_ENV_CQ))
 +BASIC_CFLAGS += -DPAGER_ENV='$(PAGER_ENV_CQ_SQ)'
 +
  ALL_CFLAGS += $(BASIC_CFLAGS)
  ALL_LDFLAGS += $(BASIC_LDFLAGS)
  
@@@ -1755,8 -1709,8 +1756,8 @@@ git.sp git.s git.o: EXTRA_CPPFLAGS = 
  	'-DGIT_INFO_PATH="$(infodir_relative_SQ)"'
  
  git$X: git.o GIT-LDFLAGS $(BUILTIN_OBJS) $(GITLIBS)
 -	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) git.o \
 -		$(BUILTIN_OBJS) $(LIBS)
 +	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \
 +		$(filter %.o,$^) $(LIBS)
  
  help.sp help.s help.o: common-cmds.h
  
@@@ -1784,7 -1738,7 +1785,7 @@@ common-cmds.h: $(wildcard Documentation
  
  SCRIPT_DEFINES = $(SHELL_PATH_SQ):$(DIFF_SQ):$(GIT_VERSION):\
  	$(localedir_SQ):$(NO_CURL):$(USE_GETTEXT_SCHEME):$(SANE_TOOL_PATH_SQ):\
 -	$(gitwebdir_SQ):$(PERL_PATH_SQ):$(SANE_TEXT_GREP)
 +	$(gitwebdir_SQ):$(PERL_PATH_SQ):$(SANE_TEXT_GREP):$(PAGER_ENV)
  define cmd_munge_script
  $(RM) $@ $@+ && \
  sed -e '1s|#!.*/sh|#!$(SHELL_PATH_SQ)|' \
@@@ -1797,7 -1751,6 +1798,7 @@@
      -e 's|@@GITWEBDIR@@|$(gitwebdir_SQ)|g' \
      -e 's|@@PERL@@|$(PERL_PATH_SQ)|g' \
      -e 's|@@SANE_TEXT_GREP@@|$(SANE_TEXT_GREP)|g' \
 +    -e 's|@@PAGER_ENV@@|$(PAGER_ENV_SQ)|g' \
      $@.sh >$@+
  endef
  
@@@ -1946,11 -1899,10 +1947,11 @@@ VCSSVN_OBJS += vcs-svn/fast_export.
  VCSSVN_OBJS += vcs-svn/svndiff.o
  VCSSVN_OBJS += vcs-svn/svndump.o
  
 -TEST_OBJS := $(patsubst test-%$X,test-%.o,$(TEST_PROGRAMS))
 +TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS))
  OBJECTS := $(LIB_OBJS) $(BUILTIN_OBJS) $(PROGRAM_OBJS) $(TEST_OBJS) \
  	$(XDIFF_OBJS) \
  	$(VCSSVN_OBJS) \
 +	common-main.o \
  	git.o
  ifndef NO_CURL
  	OBJECTS += http.o http-walker.o remote-curl.o
@@@ -2042,7 -1994,7 +2043,7 @@@ endi
  
  ifdef USE_NED_ALLOCATOR
  compat/nedmalloc/nedmalloc.sp compat/nedmalloc/nedmalloc.o: EXTRA_CPPFLAGS = \
 -	-DNDEBUG -DOVERRIDE_STRDUP -DREPLACE_SYSTEM_ALLOCATOR
 +	-DNDEBUG -DREPLACE_SYSTEM_ALLOCATOR
  compat/nedmalloc/nedmalloc.sp: SPARSE_FLAGS += -Wno-non-pointer-null
  endif
  
@@@ -2112,10 -2064,7 +2113,10 @@@ XGETTEXT_FLAGS_SH = $(XGETTEXT_FLAGS) -
  	--keyword=gettextln --keyword=eval_gettextln
  XGETTEXT_FLAGS_PERL = $(XGETTEXT_FLAGS) --keyword=__ --language=Perl
  LOCALIZED_C = $(C_OBJ:o=c) $(LIB_H) $(GENERATED_H)
 -LOCALIZED_SH = $(SCRIPT_SH) git-parse-remote.sh
 +LOCALIZED_SH = $(SCRIPT_SH)
 +LOCALIZED_SH += git-parse-remote.sh
 +LOCALIZED_SH += git-rebase--interactive.sh
 +LOCALIZED_SH += git-sh-setup.sh
  LOCALIZED_PERL = $(SCRIPT_PERL)
  
  ifdef XGETTEXT_INCLUDE_TESTS
@@@ -2205,7 -2154,6 +2206,7 @@@ GIT-BUILD-OPTIONS: FORC
  	@echo NO_PERL=\''$(subst ','\'',$(subst ','\'',$(NO_PERL)))'\' >>$@+
  	@echo NO_PYTHON=\''$(subst ','\'',$(subst ','\'',$(NO_PYTHON)))'\' >>$@+
  	@echo NO_UNIX_SOCKETS=\''$(subst ','\'',$(subst ','\'',$(NO_UNIX_SOCKETS)))'\' >>$@+
 +	@echo PAGER_ENV=\''$(subst ','\'',$(subst ','\'',$(PAGER_ENV)))'\' >>$@+
  ifdef TEST_OUTPUT_DIRECTORY
  	@echo TEST_OUTPUT_DIRECTORY=\''$(subst ','\'',$(subst ','\'',$(TEST_OUTPUT_DIRECTORY)))'\' >>$@+
  endif
@@@ -2258,7 -2206,7 +2259,7 @@@ bin-wrappers/%: wrap-for-bin.s
  	@mkdir -p bin-wrappers
  	$(QUIET_GEN)sed -e '1s|#!.*/sh|#!$(SHELL_PATH_SQ)|' \
  	     -e 's|@@BUILD_DIR@@|$(shell pwd)|' \
 -	     -e 's|@@PROG@@|$(@F)|' < $< > $@ && \
 +	     -e 's|@@PROG@@|$(patsubst test-%,t/helper/test-%,$(@F))|' < $< > $@ && \
  	chmod +x $@
  
  # GNU make supports exporting all variables by "export" without parameters.
@@@ -2278,17 -2226,25 +2279,17 @@@ perf: al
  
  .PHONY: test perf
  
 -test-ctype$X: ctype.o
 +t/helper/test-line-buffer$X: $(VCSSVN_LIB)
  
 -test-date$X: date.o ctype.o
 -
 -test-delta$X: diff-delta.o patch-delta.o
 -
 -test-line-buffer$X: vcs-svn/lib.a
 -
 -test-parse-options$X: parse-options.o parse-options-cb.o
 -
 -test-svn-fe$X: vcs-svn/lib.a
 +t/helper/test-svn-fe$X: $(VCSSVN_LIB)
  
  .PRECIOUS: $(TEST_OBJS)
  
 -test-%$X: test-%.o GIT-LDFLAGS $(GITLIBS)
 +t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS)
  	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(filter %.a,$^) $(LIBS)
  
 -check-sha1:: test-sha1$X
 -	./test-sha1.sh
 +check-sha1:: t/helper/test-sha1$X
 +	t/helper/test-sha1.sh
  
  SP_OBJ = $(patsubst %.o,%.sp,$(C_OBJ))
  
@@@ -2309,18 -2265,6 +2310,18 @@@ check: common-cmds.
  		exit 1; \
  	fi
  
 +C_SOURCES = $(patsubst %.o,%.c,$(C_OBJ))
 +%.cocci.patch: %.cocci $(C_SOURCES)
 +	@echo '    ' SPATCH $<; \
 +	for f in $(C_SOURCES); do \
 +		$(SPATCH) --sp-file $< $$f; \
 +	done >$@ 2>$@.log; \
 +	if test -s $@; \
 +	then \
 +		echo '    ' SPATCH result: $@; \
 +	fi
 +coccicheck: $(patsubst %.cocci,%.cocci.patch,$(wildcard contrib/coccinelle/*.cocci))
 +
  ### Installation rules
  
  ifneq ($(filter /%,$(firstword $(template_dir))),)
@@@ -2447,25 -2391,31 +2448,25 @@@ quick-install-html
  
  ### Maintainer's dist rules
  
 -git.spec: git.spec.in GIT-VERSION-FILE
 -	sed -e 's/@@VERSION@@/$(GIT_VERSION)/g' < $< > $@+
 -	mv $@+ $@
 -
  GIT_TARNAME = git-$(GIT_VERSION)
 -dist: git.spec git-archive$(X) configure
 +dist: git-archive$(X) configure
  	./git-archive --format=tar \
  		--prefix=$(GIT_TARNAME)/ HEAD^{tree} > $(GIT_TARNAME).tar
  	@mkdir -p $(GIT_TARNAME)
 -	@cp git.spec configure $(GIT_TARNAME)
 +	@cp configure $(GIT_TARNAME)
  	@echo $(GIT_VERSION) > $(GIT_TARNAME)/version
  	@$(MAKE) -C git-gui TARDIR=../$(GIT_TARNAME)/git-gui dist-version
  	$(TAR) rf $(GIT_TARNAME).tar \
 -		$(GIT_TARNAME)/git.spec \
  		$(GIT_TARNAME)/configure \
  		$(GIT_TARNAME)/version \
  		$(GIT_TARNAME)/git-gui/version
  	@$(RM) -r $(GIT_TARNAME)
  	gzip -f -9 $(GIT_TARNAME).tar
  
 -rpm: dist
 -	$(RPMBUILD) \
 -		--define "_source_filedigest_algorithm md5" \
 -		--define "_binary_filedigest_algorithm md5" \
 -		-ta $(GIT_TARNAME).tar.gz
 +rpm::
 +	@echo >&2 "Use distro packaged sources to run rpmbuild"
 +	@false
 +.PHONY: rpm
  
  htmldocs = git-htmldocs-$(GIT_VERSION)
  manpages = git-manpages-$(GIT_VERSION)
@@@ -2501,8 -2451,8 +2502,8 @@@ profile-clean
  	$(RM) $(addsuffix *.gcno,$(addprefix $(PROFILE_DIR)/, $(object_dirs)))
  
  clean: profile-clean coverage-clean
 -	$(RM) *.o *.res refs/*.o block-sha1/*.o ppc/*.o compat/*.o compat/*/*.o
 -	$(RM) xdiff/*.o vcs-svn/*.o ewah/*.o builtin/*.o
 +	$(RM) *.res
 +	$(RM) $(OBJECTS)
  	$(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB)
  	$(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X
  	$(RM) $(TEST_PROGRAMS) $(NO_INSTALL)
@@@ -2512,7 -2462,6 +2513,7 @@@
  	$(RM) -r $(GIT_TARNAME) .doc-tmp-dir
  	$(RM) $(GIT_TARNAME).tar.gz git-core_$(GIT_VERSION)-*.tar.gz
  	$(RM) $(htmldocs).tar.gz $(manpages).tar.gz
 +	$(RM) contrib/coccinelle/*.cocci.patch*
  	$(MAKE) -C Documentation/ clean
  ifndef NO_PERL
  	$(MAKE) -C gitweb clean
@@@ -2542,7 -2491,6 +2543,7 @@@ ALL_COMMANDS += git-gui git-citoo
  
  .PHONY: check-docs
  check-docs::
 +	$(MAKE) -C Documentation lint-docs
  	@(for v in $(ALL_COMMANDS); \
  	do \
  		case "$$v" in \
diff --combined diff.c
index 2cc3a2e29b,f77324e9e0..a178ed39bc
--- a/diff.c
+++ b/diff.c
@@@ -18,7 -18,6 +18,7 @@@
  #include "ll-merge.h"
  #include "string-list.h"
  #include "argv-array.h"
 +#include "graph.h"
  
  #ifdef NO_FAST_WORKING_DIRECTORY
  #define FAST_WORKING_DIRECTORY 0
@@@ -27,8 -26,6 +27,8 @@@
  #endif
  
  static int diff_detect_rename_default;
 +static int diff_indent_heuristic; /* experimental */
 +static int diff_compaction_heuristic; /* experimental */
  static int diff_rename_limit_default = 400;
  static int diff_suppress_blank_empty;
  static int diff_use_color_default = -1;
@@@ -56,11 -53,6 +56,11 @@@ static char diff_colors[][COLOR_MAXLEN
  	GIT_COLOR_NORMAL,	/* FUNCINFO */
  };
  
 +static NORETURN void die_want_option(const char *option_name)
 +{
 +	die(_("option '%s' requires a value"), option_name);
 +}
 +
  static int parse_diff_color_slot(const char *var)
  {
  	if (!strcasecmp(var, "context") || !strcasecmp(var, "plain"))
@@@ -138,11 -130,9 +138,11 @@@ static int parse_dirstat_params(struct 
  static int parse_submodule_params(struct diff_options *options, const char *value)
  {
  	if (!strcmp(value, "log"))
 -		DIFF_OPT_SET(options, SUBMODULE_LOG);
 +		options->submodule_format = DIFF_SUBMODULE_LOG;
  	else if (!strcmp(value, "short"))
 -		DIFF_OPT_CLR(options, SUBMODULE_LOG);
 +		options->submodule_format = DIFF_SUBMODULE_SHORT;
 +	else if (!strcmp(value, "diff"))
 +		options->submodule_format = DIFF_SUBMODULE_INLINE_DIFF;
  	else
  		return -1;
  	return 0;
@@@ -178,26 -168,6 +178,26 @@@ long parse_algorithm_value(const char *
   * never be affected by the setting of diff.renames
   * the user happens to have in the configuration file.
   */
 +void init_diff_ui_defaults(void)
 +{
 +	diff_detect_rename_default = 1;
 +}
 +
 +int git_diff_heuristic_config(const char *var, const char *value, void *cb)
 +{
 +	if (!strcmp(var, "diff.indentheuristic")) {
 +		diff_indent_heuristic = git_config_bool(var, value);
 +		if (diff_indent_heuristic)
 +			diff_compaction_heuristic = 0;
 +	}
 +	if (!strcmp(var, "diff.compactionheuristic")) {
 +		diff_compaction_heuristic = git_config_bool(var, value);
 +		if (diff_compaction_heuristic)
 +			diff_indent_heuristic = 0;
 +	}
 +	return 0;
 +}
 +
  int git_diff_ui_config(const char *var, const char *value, void *cb)
  {
  	if (!strcmp(var, "diff.color") || !strcmp(var, "color.diff")) {
@@@ -254,8 -224,6 +254,8 @@@
  		return 0;
  	}
  
 +	if (git_diff_heuristic_config(var, value, cb) < 0)
 +		return -1;
  	if (git_color_config(var, value, cb) < 0)
  		return -1;
  
@@@ -376,6 -344,7 +376,6 @@@ struct emit_callback 
  	const char **label_path;
  	struct diff_words_data *diff_words;
  	struct diff_options *opt;
 -	int *found_changesp;
  	struct strbuf *header;
  };
  
@@@ -743,6 -712,7 +743,6 @@@ static void emit_rewrite_diff(const cha
  
  	memset(&ecbdata, 0, sizeof(ecbdata));
  	ecbdata.color_diff = want_color(o->use_color);
 -	ecbdata.found_changesp = &o->found_changes;
  	ecbdata.ws_rule = whitespace_rule(name_b);
  	ecbdata.opt = o;
  	if (ecbdata.ws_rule & WS_BLANK_AT_EOF) {
@@@ -971,7 -941,8 +971,8 @@@ static int find_word_boundaries(mmfile_
  {
  	if (word_regex && *begin < buffer->size) {
  		regmatch_t match[1];
- 		if (!regexec(word_regex, buffer->ptr + *begin, 1, match, 0)) {
+ 		if (!regexec_buf(word_regex, buffer->ptr + *begin,
+ 				 buffer->size - *begin, 1, match, 0)) {
  			char *p = memchr(buffer->ptr + *begin + match[0].rm_so,
  					'\n', match[0].rm_eo - match[0].rm_so);
  			*end = p ? p - buffer->ptr : match[0].rm_eo + *begin;
@@@ -1236,13 -1207,12 +1237,13 @@@ static void fn_out_consume(void *priv, 
  	struct diff_options *o = ecbdata->opt;
  	const char *line_prefix = diff_line_prefix(o);
  
 +	o->found_changes = 1;
 +
  	if (ecbdata->header) {
 -		fprintf(ecbdata->opt->file, "%s", ecbdata->header->buf);
 +		fprintf(o->file, "%s", ecbdata->header->buf);
  		strbuf_reset(ecbdata->header);
  		ecbdata->header = NULL;
  	}
 -	*(ecbdata->found_changesp) = 1;
  
  	if (ecbdata->label_path[0]) {
  		const char *name_a_tab, *name_b_tab;
@@@ -1250,9 -1220,9 +1251,9 @@@
  		name_a_tab = strchr(ecbdata->label_path[0], ' ') ? "\t" : "";
  		name_b_tab = strchr(ecbdata->label_path[1], ' ') ? "\t" : "";
  
 -		fprintf(ecbdata->opt->file, "%s%s--- %s%s%s\n",
 +		fprintf(o->file, "%s%s--- %s%s%s\n",
  			line_prefix, meta, ecbdata->label_path[0], reset, name_a_tab);
 -		fprintf(ecbdata->opt->file, "%s%s+++ %s%s%s\n",
 +		fprintf(o->file, "%s%s+++ %s%s%s\n",
  			line_prefix, meta, ecbdata->label_path[1], reset, name_b_tab);
  		ecbdata->label_path[0] = ecbdata->label_path[1] = NULL;
  	}
@@@ -1270,7 -1240,15 +1271,7 @@@
  		find_lno(line, ecbdata);
  		emit_hunk_header(ecbdata, line, len);
  		if (line[len-1] != '\n')
 -			putc('\n', ecbdata->opt->file);
 -		return;
 -	}
 -
 -	if (len < 1) {
 -		emit_line(ecbdata->opt, reset, reset, line, len);
 -		if (ecbdata->diff_words
 -		    && ecbdata->diff_words->type == DIFF_WORDS_PORCELAIN)
 -			fputs("~\n", ecbdata->opt->file);
 +			putc('\n', o->file);
  		return;
  	}
  
@@@ -1295,8 -1273,8 +1296,8 @@@
  		}
  		diff_words_flush(ecbdata);
  		if (ecbdata->diff_words->type == DIFF_WORDS_PORCELAIN) {
 -			emit_line(ecbdata->opt, context, reset, line, len);
 -			fputs("~\n", ecbdata->opt->file);
 +			emit_line(o, context, reset, line, len);
 +			fputs("~\n", o->file);
  		} else {
  			/*
  			 * Skip the prefix character, if any.  With
@@@ -1307,7 -1285,7 +1308,7 @@@
  			      line++;
  			      len--;
  			}
 -			emit_line(ecbdata->opt, context, reset, line, len);
 +			emit_line(o, context, reset, line, len);
  		}
  		return;
  	}
@@@ -1329,7 -1307,8 +1330,7 @@@
  	default:
  		/* incomplete line at the end */
  		ecbdata->lno_in_preimage++;
 -		emit_line(ecbdata->opt,
 -			  diff_get_color(ecbdata->color_diff, DIFF_CONTEXT),
 +		emit_line(o, diff_get_color(ecbdata->color_diff, DIFF_CONTEXT),
  			  reset, line, len);
  		break;
  	}
@@@ -1637,7 -1616,7 +1638,7 @@@ static void show_stats(struct diffstat_
  	 */
  
  	if (options->stat_width == -1)
 -		width = term_columns() - options->output_prefix_length;
 +		width = term_columns() - strlen(line_prefix);
  	else
  		width = options->stat_width ? options->stat_width : 80;
  	number_width = decimal_width(max_change) > number_width ?
@@@ -1945,8 -1924,8 +1946,8 @@@ static void show_dirstat(struct diff_op
  
  		name = p->two->path ? p->two->path : p->one->path;
  
 -		if (p->one->sha1_valid && p->two->sha1_valid)
 -			content_changed = hashcmp(p->one->sha1, p->two->sha1);
 +		if (p->one->oid_valid && p->two->oid_valid)
 +			content_changed = oidcmp(&p->one->oid, &p->two->oid);
  		else
  			content_changed = 1;
  
@@@ -2311,37 -2290,16 +2312,37 @@@ static void builtin_diff(const char *na
  	struct strbuf header = STRBUF_INIT;
  	const char *line_prefix = diff_line_prefix(o);
  
 -	if (DIFF_OPT_TST(o, SUBMODULE_LOG) &&
 -			(!one->mode || S_ISGITLINK(one->mode)) &&
 -			(!two->mode || S_ISGITLINK(two->mode))) {
 +	diff_set_mnemonic_prefix(o, "a/", "b/");
 +	if (DIFF_OPT_TST(o, REVERSE_DIFF)) {
 +		a_prefix = o->b_prefix;
 +		b_prefix = o->a_prefix;
 +	} else {
 +		a_prefix = o->a_prefix;
 +		b_prefix = o->b_prefix;
 +	}
 +
 +	if (o->submodule_format == DIFF_SUBMODULE_LOG &&
 +	    (!one->mode || S_ISGITLINK(one->mode)) &&
 +	    (!two->mode || S_ISGITLINK(two->mode))) {
  		const char *del = diff_get_color_opt(o, DIFF_FILE_OLD);
  		const char *add = diff_get_color_opt(o, DIFF_FILE_NEW);
  		show_submodule_summary(o->file, one->path ? one->path : two->path,
  				line_prefix,
 -				one->sha1, two->sha1, two->dirty_submodule,
 +				&one->oid, &two->oid,
 +				two->dirty_submodule,
  				meta, del, add, reset);
  		return;
 +	} else if (o->submodule_format == DIFF_SUBMODULE_INLINE_DIFF &&
 +		   (!one->mode || S_ISGITLINK(one->mode)) &&
 +		   (!two->mode || S_ISGITLINK(two->mode))) {
 +		const char *del = diff_get_color_opt(o, DIFF_FILE_OLD);
 +		const char *add = diff_get_color_opt(o, DIFF_FILE_NEW);
 +		show_submodule_inline_diff(o->file, one->path ? one->path : two->path,
 +				line_prefix,
 +				&one->oid, &two->oid,
 +				two->dirty_submodule,
 +				meta, del, add, reset, o);
 +		return;
  	}
  
  	if (DIFF_OPT_TST(o, ALLOW_TEXTCONV)) {
@@@ -2349,6 -2307,15 +2350,6 @@@
  		textconv_two = get_textconv(two);
  	}
  
 -	diff_set_mnemonic_prefix(o, "a/", "b/");
 -	if (DIFF_OPT_TST(o, REVERSE_DIFF)) {
 -		a_prefix = o->b_prefix;
 -		b_prefix = o->a_prefix;
 -	} else {
 -		a_prefix = o->a_prefix;
 -		b_prefix = o->b_prefix;
 -	}
 -
  	/* Never use a non-valid filename anywhere if at all possible */
  	name_a = DIFF_FILE_VALID(one) ? name_a : name_b;
  	name_b = DIFF_FILE_VALID(two) ? name_b : name_a;
@@@ -2408,7 -2375,7 +2409,7 @@@
  		if (!one->data && !two->data &&
  		    S_ISREG(one->mode) && S_ISREG(two->mode) &&
  		    !DIFF_OPT_TST(o, BINARY)) {
 -			if (!hashcmp(one->sha1, two->sha1)) {
 +			if (!oidcmp(&one->oid, &two->oid)) {
  				if (must_show_header)
  					fprintf(o->file, "%s", header.buf);
  				goto free_ab_and_return;
@@@ -2461,6 -2428,7 +2462,6 @@@
  		memset(&ecbdata, 0, sizeof(ecbdata));
  		ecbdata.label_path = lbl;
  		ecbdata.color_diff = want_color(o->use_color);
 -		ecbdata.found_changesp = &o->found_changes;
  		ecbdata.ws_rule = whitespace_rule(name_b);
  		if (ecbdata.ws_rule & WS_BLANK_AT_EOF)
  			check_blank_at_eof(&mf1, &mf2, &ecbdata);
@@@ -2528,7 -2496,7 +2529,7 @@@ static void builtin_diffstat(const cha
  		return;
  	}
  
 -	same_contents = !hashcmp(one->sha1, two->sha1);
 +	same_contents = !oidcmp(&one->oid, &two->oid);
  
  	if (diff_filespec_is_binary(one) || diff_filespec_is_binary(two)) {
  		data->is_binary = 1;
@@@ -2661,8 -2629,8 +2662,8 @@@ void fill_filespec(struct diff_filespe
  {
  	if (mode) {
  		spec->mode = canon_mode(mode);
 -		hashcpy(spec->sha1, sha1);
 -		spec->sha1_valid = sha1_valid;
 +		hashcpy(spec->oid.hash, sha1);
 +		spec->oid_valid = sha1_valid;
  	}
  }
  
@@@ -2705,13 -2673,6 +2706,13 @@@ static int reuse_worktree_file(const ch
  	if (!FAST_WORKING_DIRECTORY && !want_file && has_sha1_pack(sha1))
  		return 0;
  
 +	/*
 +	 * Similarly, if we'd have to convert the file contents anyway, that
 +	 * makes the optimization not worthwhile.
 +	 */
 +	if (!want_file && would_convert_to_git(name))
 +		return 0;
 +
  	len = strlen(name);
  	pos = cache_name_pos(name, len);
  	if (pos < 0)
@@@ -2722,7 -2683,7 +2723,7 @@@
  	 * This is not the sha1 we are looking for, or
  	 * unreusable because it is not a regular file.
  	 */
 -	if (hashcmp(sha1, ce->sha1) || !S_ISREG(ce->ce_mode))
 +	if (hashcmp(sha1, ce->oid.hash) || !S_ISREG(ce->ce_mode))
  		return 0;
  
  	/*
@@@ -2751,8 -2712,7 +2752,8 @@@ static int diff_populate_gitlink(struc
  	if (s->dirty_submodule)
  		dirty = "-dirty";
  
 -	strbuf_addf(&buf, "Subproject commit %s%s\n", sha1_to_hex(s->sha1), dirty);
 +	strbuf_addf(&buf, "Subproject commit %s%s\n",
 +		    oid_to_hex(&s->oid), dirty);
  	s->size = buf.len;
  	if (size_only) {
  		s->data = NULL;
@@@ -2795,8 -2755,8 +2796,8 @@@ int diff_populate_filespec(struct diff_
  	if (S_ISGITLINK(s->mode))
  		return diff_populate_gitlink(s, size_only);
  
 -	if (!s->sha1_valid ||
 -	    reuse_worktree_file(s->path, s->sha1, 0)) {
 +	if (!s->oid_valid ||
 +	    reuse_worktree_file(s->path, s->oid.hash, 0)) {
  		struct strbuf buf = STRBUF_INIT;
  		struct stat st;
  		int fd;
@@@ -2853,10 -2813,9 +2854,10 @@@
  	else {
  		enum object_type type;
  		if (size_only || (flags & CHECK_BINARY)) {
 -			type = sha1_object_info(s->sha1, &s->size);
 +			type = sha1_object_info(s->oid.hash, &s->size);
  			if (type < 0)
 -				die("unable to read %s", sha1_to_hex(s->sha1));
 +				die("unable to read %s",
 +				    oid_to_hex(&s->oid));
  			if (size_only)
  				return 0;
  			if (s->size > big_file_threshold && s->is_binary == -1) {
@@@ -2864,9 -2823,9 +2865,9 @@@
  				return 0;
  			}
  		}
 -		s->data = read_sha1_file(s->sha1, &type, &s->size);
 +		s->data = read_sha1_file(s->oid.hash, &type, &s->size);
  		if (!s->data)
 -			die("unable to read %s", sha1_to_hex(s->sha1));
 +			die("unable to read %s", oid_to_hex(&s->oid));
  		s->should_free = 1;
  	}
  	return 0;
@@@ -2895,7 -2854,7 +2896,7 @@@ void diff_free_filespec_data(struct dif
  static void prep_temp_blob(const char *path, struct diff_tempfile *temp,
  			   void *blob,
  			   unsigned long size,
 -			   const unsigned char *sha1,
 +			   const struct object_id *oid,
  			   int mode)
  {
  	int fd;
@@@ -2920,7 -2879,7 +2921,7 @@@
  		die_errno("unable to write temp-file");
  	close_tempfile(&temp->tempfile);
  	temp->name = get_tempfile_path(&temp->tempfile);
 -	sha1_to_hex_r(temp->hex, sha1);
 +	oid_to_hex_r(temp->hex, oid);
  	xsnprintf(temp->mode, sizeof(temp->mode), "%06o", mode);
  	strbuf_release(&buf);
  	strbuf_release(&template);
@@@ -2944,8 -2903,8 +2945,8 @@@ static struct diff_tempfile *prepare_te
  	}
  
  	if (!S_ISGITLINK(one->mode) &&
 -	    (!one->sha1_valid ||
 -	     reuse_worktree_file(name, one->sha1, 1))) {
 +	    (!one->oid_valid ||
 +	     reuse_worktree_file(name, one->oid.hash, 1))) {
  		struct stat st;
  		if (lstat(name, &st) < 0) {
  			if (errno == ENOENT)
@@@ -2957,19 -2916,19 +2958,19 @@@
  			if (strbuf_readlink(&sb, name, st.st_size) < 0)
  				die_errno("readlink(%s)", name);
  			prep_temp_blob(name, temp, sb.buf, sb.len,
 -				       (one->sha1_valid ?
 -					one->sha1 : null_sha1),
 -				       (one->sha1_valid ?
 +				       (one->oid_valid ?
 +					&one->oid : &null_oid),
 +				       (one->oid_valid ?
  					one->mode : S_IFLNK));
  			strbuf_release(&sb);
  		}
  		else {
  			/* we can borrow from the file in the work tree */
  			temp->name = name;
 -			if (!one->sha1_valid)
 +			if (!one->oid_valid)
  				sha1_to_hex_r(temp->hex, null_sha1);
  			else
 -				sha1_to_hex_r(temp->hex, one->sha1);
 +				sha1_to_hex_r(temp->hex, one->oid.hash);
  			/* Even though we may sometimes borrow the
  			 * contents from the work tree, we always want
  			 * one->mode.  mode is trustworthy even when
@@@ -2984,7 -2943,7 +2985,7 @@@
  		if (diff_populate_filespec(one, 0))
  			die("cannot read data blob for %s", one->path);
  		prep_temp_blob(name, temp, one->data, one->size,
 -			       one->sha1, one->mode);
 +			       &one->oid, one->mode);
  	}
  	return temp;
  }
@@@ -3097,7 -3056,7 +3098,7 @@@ static void fill_metainfo(struct strbu
  	default:
  		*must_show_header = 0;
  	}
 -	if (one && two && hashcmp(one->sha1, two->sha1)) {
 +	if (one && two && oidcmp(&one->oid, &two->oid)) {
  		int abbrev = DIFF_OPT_TST(o, FULL_INDEX) ? 40 : DEFAULT_ABBREV;
  
  		if (DIFF_OPT_TST(o, BINARY)) {
@@@ -3107,8 -3066,8 +3108,8 @@@
  				abbrev = 40;
  		}
  		strbuf_addf(msg, "%s%sindex %s..", line_prefix, set,
 -			    find_unique_abbrev(one->sha1, abbrev));
 -		strbuf_addstr(msg, find_unique_abbrev(two->sha1, abbrev));
 +			    find_unique_abbrev(one->oid.hash, abbrev));
 +		strbuf_addstr(msg, find_unique_abbrev(two->oid.hash, abbrev));
  		if (one->mode == two->mode)
  			strbuf_addf(msg, " %06o", one->mode);
  		strbuf_addf(msg, "%s\n", reset);
@@@ -3163,20 -3122,20 +3164,20 @@@ static void run_diff_cmd(const char *pg
  static void diff_fill_sha1_info(struct diff_filespec *one)
  {
  	if (DIFF_FILE_VALID(one)) {
 -		if (!one->sha1_valid) {
 +		if (!one->oid_valid) {
  			struct stat st;
  			if (one->is_stdin) {
 -				hashcpy(one->sha1, null_sha1);
 +				oidclr(&one->oid);
  				return;
  			}
  			if (lstat(one->path, &st) < 0)
  				die_errno("stat '%s'", one->path);
 -			if (index_path(one->sha1, one->path, &st, 0))
 +			if (index_path(one->oid.hash, one->path, &st, 0))
  				die("cannot hash %s", one->path);
  		}
  	}
  	else
 -		hashclr(one->sha1);
 +		oidclr(&one->oid);
  }
  
  static void strip_prefix(int prefix_length, const char **namep, const char **otherp)
@@@ -3315,10 -3274,6 +3316,10 @@@ void diff_setup(struct diff_options *op
  	options->use_color = diff_use_color_default;
  	options->detect_rename = diff_detect_rename_default;
  	options->xdl_opts |= diff_algorithm;
 +	if (diff_indent_heuristic)
 +		DIFF_XDL_SET(options, INDENT_HEURISTIC);
 +	else if (diff_compaction_heuristic)
 +		DIFF_XDL_SET(options, COMPACTION_HEURISTIC);
  
  	options->orderfile = diff_order_file_cfg;
  
@@@ -3346,7 -3301,7 +3347,7 @@@ void diff_setup_done(struct diff_option
  	if (options->output_format & DIFF_FORMAT_NO_OUTPUT)
  		count++;
  	if (count > 1)
 -		die("--name-only, --name-status, --check and -s are mutually exclusive");
 +		die(_("--name-only, --name-status, --check and -s are mutually exclusive"));
  
  	/*
  	 * Most of the time we can say "there are changes"
@@@ -3542,7 -3497,7 +3543,7 @@@ static int stat_opt(struct diff_option
  			if (*arg == '=')
  				width = strtoul(arg + 1, &end, 10);
  			else if (!*arg && !av[1])
 -				die("Option '--stat-width' requires a value");
 +				die_want_option("--stat-width");
  			else if (!*arg) {
  				width = strtoul(av[1], &end, 10);
  				argcount = 2;
@@@ -3551,7 -3506,7 +3552,7 @@@
  			if (*arg == '=')
  				name_width = strtoul(arg + 1, &end, 10);
  			else if (!*arg && !av[1])
 -				die("Option '--stat-name-width' requires a value");
 +				die_want_option("--stat-name-width");
  			else if (!*arg) {
  				name_width = strtoul(av[1], &end, 10);
  				argcount = 2;
@@@ -3560,7 -3515,7 +3561,7 @@@
  			if (*arg == '=')
  				graph_width = strtoul(arg + 1, &end, 10);
  			else if (!*arg && !av[1])
 -				die("Option '--stat-graph-width' requires a value");
 +				die_want_option("--stat-graph-width");
  			else if (!*arg) {
  				graph_width = strtoul(av[1], &end, 10);
  				argcount = 2;
@@@ -3569,7 -3524,7 +3570,7 @@@
  			if (*arg == '=')
  				count = strtoul(arg + 1, &end, 10);
  			else if (!*arg && !av[1])
 -				die("Option '--stat-count' requires a value");
 +				die_want_option("--stat-count");
  			else if (!*arg) {
  				count = strtoul(av[1], &end, 10);
  				argcount = 2;
@@@ -3839,16 -3794,6 +3840,16 @@@ int diff_opt_parse(struct diff_options 
  		DIFF_XDL_SET(options, IGNORE_WHITESPACE_AT_EOL);
  	else if (!strcmp(arg, "--ignore-blank-lines"))
  		DIFF_XDL_SET(options, IGNORE_BLANK_LINES);
 +	else if (!strcmp(arg, "--indent-heuristic")) {
 +		DIFF_XDL_SET(options, INDENT_HEURISTIC);
 +		DIFF_XDL_CLR(options, COMPACTION_HEURISTIC);
 +	} else if (!strcmp(arg, "--no-indent-heuristic"))
 +		DIFF_XDL_CLR(options, INDENT_HEURISTIC);
 +	else if (!strcmp(arg, "--compaction-heuristic")) {
 +		DIFF_XDL_SET(options, COMPACTION_HEURISTIC);
 +		DIFF_XDL_CLR(options, INDENT_HEURISTIC);
 +	} else if (!strcmp(arg, "--no-compaction-heuristic"))
 +		DIFF_XDL_CLR(options, COMPACTION_HEURISTIC);
  	else if (!strcmp(arg, "--patience"))
  		options->xdl_opts = DIFF_WITH_ALG(options, PATIENCE_DIFF);
  	else if (!strcmp(arg, "--histogram"))
@@@ -3945,7 -3890,7 +3946,7 @@@
  		DIFF_OPT_SET(options, OVERRIDE_SUBMODULE_CONFIG);
  		handle_ignore_submodules_arg(options, arg);
  	} else if (!strcmp(arg, "--submodule"))
 -		DIFF_OPT_SET(options, SUBMODULE_LOG);
 +		options->submodule_format = DIFF_SUBMODULE_LOG;
  	else if (skip_prefix(arg, "--submodule=", &arg))
  		return parse_submodule_opt(options, arg);
  	else if (skip_prefix(arg, "--ws-error-highlight=", &arg))
@@@ -3996,12 -3941,6 +3997,12 @@@
  		options->a_prefix = optarg;
  		return argcount;
  	}
 +	else if ((argcount = parse_long_opt("line-prefix", av, &optarg))) {
 +		options->line_prefix = optarg;
 +		options->line_prefix_length = strlen(options->line_prefix);
 +		graph_setup_line_prefix(options);
 +		return argcount;
 +	}
  	else if ((argcount = parse_long_opt("dst-prefix", av, &optarg))) {
  		options->b_prefix = optarg;
  		return argcount;
@@@ -4023,8 -3962,6 +4024,8 @@@
  		if (!options->file)
  			die_errno("Could not open '%s'", path);
  		options->close_file = 1;
 +		if (options->use_color != GIT_COLOR_ALWAYS)
 +			options->use_color = GIT_COLOR_NEVER;
  		return argcount;
  	} else
  		return 0;
@@@ -4166,9 -4103,8 +4167,9 @@@ static void diff_flush_raw(struct diff_
  	fprintf(opt->file, "%s", diff_line_prefix(opt));
  	if (!(opt->output_format & DIFF_FORMAT_NAME_STATUS)) {
  		fprintf(opt->file, ":%06o %06o %s ", p->one->mode, p->two->mode,
 -			diff_unique_abbrev(p->one->sha1, opt->abbrev));
 -		fprintf(opt->file, "%s ", diff_unique_abbrev(p->two->sha1, opt->abbrev));
 +			diff_unique_abbrev(p->one->oid.hash, opt->abbrev));
 +		fprintf(opt->file, "%s ",
 +			diff_unique_abbrev(p->two->oid.hash, opt->abbrev));
  	}
  	if (p->score) {
  		fprintf(opt->file, "%c%03d%c", p->status, similarity_index(p),
@@@ -4217,11 -4153,11 +4218,11 @@@ int diff_unmodified_pair(struct diff_fi
  	/* both are valid and point at the same path.  that is, we are
  	 * dealing with a change.
  	 */
 -	if (one->sha1_valid && two->sha1_valid &&
 -	    !hashcmp(one->sha1, two->sha1) &&
 +	if (one->oid_valid && two->oid_valid &&
 +	    !oidcmp(&one->oid, &two->oid) &&
  	    !one->dirty_submodule && !two->dirty_submodule)
  		return 1; /* no change */
 -	if (!one->sha1_valid && !two->sha1_valid)
 +	if (!one->oid_valid && !two->oid_valid)
  		return 1; /* both look at the same file on the filesystem. */
  	return 0;
  }
@@@ -4282,7 -4218,7 +4283,7 @@@ void diff_debug_filespec(struct diff_fi
  		s->path,
  		DIFF_FILE_VALID(s) ? "valid" : "invalid",
  		s->mode,
 -		s->sha1_valid ? sha1_to_hex(s->sha1) : "");
 +		s->oid_valid ? oid_to_hex(&s->oid) : "");
  	fprintf(stderr, "queue[%d] %s size %lu\n",
  		x, one ? one : "",
  		s->size);
@@@ -4352,11 -4288,11 +4353,11 @@@ static void diff_resolve_rename_copy(vo
  			else
  				p->status = DIFF_STATUS_RENAMED;
  		}
 -		else if (hashcmp(p->one->sha1, p->two->sha1) ||
 +		else if (oidcmp(&p->one->oid, &p->two->oid) ||
  			 p->one->mode != p->two->mode ||
  			 p->one->dirty_submodule ||
  			 p->two->dirty_submodule ||
 -			 is_null_sha1(p->one->sha1))
 +			 is_null_oid(&p->one->oid))
  			p->status = DIFF_STATUS_MODIFIED;
  		else {
  			/* This is a "no-change" entry and should not
@@@ -4498,7 -4434,7 +4499,7 @@@ static void patch_id_consume(void *priv
  }
  
  /* returns 0 upon success, and writes result into sha1 */
 -static int diff_get_patch_id(struct diff_options *options, unsigned char *sha1)
 +static int diff_get_patch_id(struct diff_options *options, unsigned char *sha1, int diff_header_only)
  {
  	struct diff_queue_struct *q = &diff_queued_diff;
  	int i;
@@@ -4533,6 -4469,9 +4534,6 @@@
  
  		diff_fill_sha1_info(p->one);
  		diff_fill_sha1_info(p->two);
 -		if (fill_mmfile(&mf1, p->one) < 0 ||
 -				fill_mmfile(&mf2, p->two) < 0)
 -			return error("unable to read files to diff");
  
  		len1 = remove_space(p->one->path, strlen(p->one->path));
  		len2 = remove_space(p->two->path, strlen(p->two->path));
@@@ -4567,19 -4506,10 +4568,19 @@@
  					len2, p->two->path);
  		git_SHA1_Update(&ctx, buffer, len1);
  
 +		if (diff_header_only)
 +			continue;
 +
 +		if (fill_mmfile(&mf1, p->one) < 0 ||
 +		    fill_mmfile(&mf2, p->two) < 0)
 +			return error("unable to read files to diff");
 +
  		if (diff_filespec_is_binary(p->one) ||
  		    diff_filespec_is_binary(p->two)) {
 -			git_SHA1_Update(&ctx, sha1_to_hex(p->one->sha1), 40);
 -			git_SHA1_Update(&ctx, sha1_to_hex(p->two->sha1), 40);
 +			git_SHA1_Update(&ctx, oid_to_hex(&p->one->oid),
 +					40);
 +			git_SHA1_Update(&ctx, oid_to_hex(&p->two->oid),
 +					40);
  			continue;
  		}
  
@@@ -4596,11 -4526,11 +4597,11 @@@
  	return 0;
  }
  
 -int diff_flush_patch_id(struct diff_options *options, unsigned char *sha1)
 +int diff_flush_patch_id(struct diff_options *options, unsigned char *sha1, int diff_header_only)
  {
  	struct diff_queue_struct *q = &diff_queued_diff;
  	int i;
 -	int result = diff_get_patch_id(options, sha1);
 +	int result = diff_get_patch_id(options, sha1, diff_header_only);
  
  	for (i = 0; i < q->nr; i++)
  		diff_free_filepair(q->queue[i]);
@@@ -4871,7 -4801,7 +4872,7 @@@ static int diff_filespec_check_stat_unm
  	 */
  	if (!DIFF_FILE_VALID(p->one) || /* (1) */
  	    !DIFF_FILE_VALID(p->two) ||
 -	    (p->one->sha1_valid && p->two->sha1_valid) ||
 +	    (p->one->oid_valid && p->two->oid_valid) ||
  	    (p->one->mode != p->two->mode) ||
  	    diff_populate_filespec(p->one, CHECK_SIZE_ONLY) ||
  	    diff_populate_filespec(p->two, CHECK_SIZE_ONLY) ||
@@@ -5167,9 -5097,8 +5168,9 @@@ size_t fill_textconv(struct userdiff_dr
  	if (!driver->textconv)
  		die("BUG: fill_textconv called with non-textconv driver");
  
 -	if (driver->textconv_cache && df->sha1_valid) {
 -		*outbuf = notes_cache_get(driver->textconv_cache, df->sha1,
 +	if (driver->textconv_cache && df->oid_valid) {
 +		*outbuf = notes_cache_get(driver->textconv_cache,
 +					  df->oid.hash,
  					  &size);
  		if (*outbuf)
  			return size;
@@@ -5179,9 -5108,9 +5180,9 @@@
  	if (!*outbuf)
  		die("unable to read files to diff");
  
 -	if (driver->textconv_cache && df->sha1_valid) {
 +	if (driver->textconv_cache && df->oid_valid) {
  		/* ignore errors, as we might be in a readonly repository */
 -		notes_cache_put(driver->textconv_cache, df->sha1, *outbuf,
 +		notes_cache_put(driver->textconv_cache, df->oid.hash, *outbuf,
  				size);
  		/*
  		 * we could save up changes and flush them all at the end,
diff --combined diffcore-pickaxe.c
index 55067cab6c,8413d76582..9795ca1c15
--- a/diffcore-pickaxe.c
+++ b/diffcore-pickaxe.c
@@@ -7,8 -7,6 +7,8 @@@
  #include "diffcore.h"
  #include "xdiff-interface.h"
  #include "kwset.h"
 +#include "commit.h"
 +#include "quote.h"
  
  typedef int (*pickaxe_fn)(mmfile_t *one, mmfile_t *two,
  			  struct diff_options *o,
@@@ -23,7 -21,6 +23,6 @@@ static void diffgrep_consume(void *priv
  {
  	struct diffgrep_cb *data = priv;
  	regmatch_t regmatch;
- 	int hold;
  
  	if (line[0] != '+' && line[0] != '-')
  		return;
@@@ -33,11 -30,8 +32,8 @@@
  		 * caller early.
  		 */
  		return;
- 	/* Yuck -- line ought to be "const char *"! */
- 	hold = line[len];
- 	line[len] = '\0';
- 	data->hit = !regexec(data->regexp, line + 1, 1, &regmatch, 0);
- 	line[len] = hold;
+ 	data->hit = !regexec_buf(data->regexp, line + 1, len - 1, 1,
+ 				 &regmatch, 0);
  }
  
  static int diff_grep(mmfile_t *one, mmfile_t *two,
@@@ -50,9 -44,11 +46,11 @@@
  	xdemitconf_t xecfg;
  
  	if (!one)
- 		return !regexec(regexp, two->ptr, 1, &regmatch, 0);
+ 		return !regexec_buf(regexp, two->ptr, two->size,
+ 				    1, &regmatch, 0);
  	if (!two)
- 		return !regexec(regexp, one->ptr, 1, &regmatch, 0);
+ 		return !regexec_buf(regexp, one->ptr, one->size,
+ 				    1, &regmatch, 0);
  
  	/*
  	 * We have both sides; need to run textual diff and see if
@@@ -83,8 -79,8 +81,8 @@@ static unsigned int contains(mmfile_t *
  		regmatch_t regmatch;
  		int flags = 0;
  
- 		assert(data[sz] == '\0');
- 		while (*data && !regexec(regexp, data, 1, &regmatch, flags)) {
+ 		while (*data &&
+ 		       !regexec_buf(regexp, data, sz, 1, &regmatch, flags)) {
  			flags |= REG_NOTBOL;
  			data += regmatch.rm_eo;
  			if (*data && regmatch.rm_so == regmatch.rm_eo)
@@@ -200,18 -196,6 +198,18 @@@ static void pickaxe(struct diff_queue_s
  	*q = outq;
  }
  
 +static void regcomp_or_die(regex_t *regex, const char *needle, int cflags)
 +{
 +	int err = regcomp(regex, needle, cflags);
 +	if (err) {
 +		/* The POSIX.2 people are surely sick */
 +		char errbuf[1024];
 +		regerror(err, regex, errbuf, 1024);
 +		regfree(regex);
 +		die("invalid regex: %s", errbuf);
 +	}
 +}
 +
  void diffcore_pickaxe(struct diff_options *o)
  {
  	const char *needle = o->pickaxe;
@@@ -220,19 -204,18 +218,19 @@@
  	kwset_t kws = NULL;
  
  	if (opts & (DIFF_PICKAXE_REGEX | DIFF_PICKAXE_KIND_G)) {
 -		int err;
  		int cflags = REG_EXTENDED | REG_NEWLINE;
  		if (DIFF_OPT_TST(o, PICKAXE_IGNORE_CASE))
  			cflags |= REG_ICASE;
 -		err = regcomp(&regex, needle, cflags);
 -		if (err) {
 -			/* The POSIX.2 people are surely sick */
 -			char errbuf[1024];
 -			regerror(err, &regex, errbuf, 1024);
 -			regfree(&regex);
 -			die("invalid regex: %s", errbuf);
 -		}
 +		regcomp_or_die(&regex, needle, cflags);
 +		regexp = &regex;
 +	} else if (DIFF_OPT_TST(o, PICKAXE_IGNORE_CASE) &&
 +		   has_non_ascii(needle)) {
 +		struct strbuf sb = STRBUF_INIT;
 +		int cflags = REG_NEWLINE | REG_ICASE;
 +
 +		basic_regex_quote_buf(&sb, needle);
 +		regcomp_or_die(&regex, sb.buf, cflags);
 +		strbuf_release(&sb);
  		regexp = &regex;
  	} else {
  		kws = kwsalloc(DIFF_OPT_TST(o, PICKAXE_IGNORE_CASE)
diff --combined git-compat-util.h
index 37cce0746f,7047d281e5..8aab0c3047
--- a/git-compat-util.h
+++ b/git-compat-util.h
@@@ -409,9 -409,7 +409,9 @@@ extern NORETURN void usagef(const char 
  extern NORETURN void die(const char *err, ...) __attribute__((format (printf, 1, 2)));
  extern NORETURN void die_errno(const char *err, ...) __attribute__((format (printf, 1, 2)));
  extern int error(const char *err, ...) __attribute__((format (printf, 1, 2)));
 +extern int error_errno(const char *err, ...) __attribute__((format (printf, 1, 2)));
  extern void warning(const char *err, ...) __attribute__((format (printf, 1, 2)));
 +extern void warning_errno(const char *err, ...) __attribute__((format (printf, 1, 2)));
  
  #ifndef NO_OPENSSL
  #ifdef APPLE_COMMON_CRYPTO
@@@ -436,14 -434,10 +436,14 @@@ static inline int const_error(void
  	return -1;
  }
  #define error(...) (error(__VA_ARGS__), const_error())
 +#define error_errno(...) (error_errno(__VA_ARGS__), const_error())
  #endif
  
  extern void set_die_routine(NORETURN_PTR void (*routine)(const char *err, va_list params));
  extern void set_error_routine(void (*routine)(const char *err, va_list params));
 +extern void (*get_error_routine(void))(const char *err, va_list params);
 +extern void set_warn_routine(void (*routine)(const char *warn, va_list params));
 +extern void (*get_warn_routine(void))(const char *warn, va_list params);
  extern void set_die_is_recursing_routine(int (*routine)(void));
  extern void set_error_handle(FILE *);
  
@@@ -477,23 -471,6 +477,23 @@@ static inline int skip_prefix(const cha
  	return 0;
  }
  
 +/*
 + * Like skip_prefix, but promises never to read past "len" bytes of the input
 + * buffer, and returns the remaining number of bytes in "out" via "outlen".
 + */
 +static inline int skip_prefix_mem(const char *buf, size_t len,
 +				  const char *prefix,
 +				  const char **out, size_t *outlen)
 +{
 +	size_t prefix_len = strlen(prefix);
 +	if (prefix_len <= len && !memcmp(buf, prefix, prefix_len)) {
 +		*out = buf + prefix_len;
 +		*outlen = len - prefix_len;
 +		return 1;
 +	}
 +	return 0;
 +}
 +
  /*
   * If buf ends with suffix, return 1 and subtract the length of the suffix
   * from *len. Otherwise, return 0 and leave *len untouched.
@@@ -667,22 -644,10 +667,22 @@@ void *gitmemmem(const void *haystack, s
                  const void *needle, size_t needlelen);
  #endif
  
 +#ifdef OVERRIDE_STRDUP
 +#ifdef strdup
 +#undef strdup
 +#endif
 +#define strdup gitstrdup
 +char *gitstrdup(const char *s);
 +#endif
 +
  #ifdef NO_GETPAGESIZE
  #define getpagesize() sysconf(_SC_PAGESIZE)
  #endif
  
 +#ifndef O_CLOEXEC
 +#define O_CLOEXEC 0
 +#endif
 +
  #ifdef FREAD_READS_DIRECTORIES
  #ifdef fopen
  #undef fopen
@@@ -831,7 -796,7 +831,7 @@@ extern FILE *fopen_for_writing(const ch
   * you can do:
   *
   *   struct foo *f;
 - *   FLEX_ALLOC_STR(f, name, src);
 + *   FLEXPTR_ALLOC_STR(f, name, src);
   *
   * and "name" will point to a block of memory after the struct, which will be
   * freed along with the struct (but the pointer can be repointed anywhere).
@@@ -977,6 -942,19 +977,19 @@@ void git_qsort(void *base, size_t nmemb
  #define qsort git_qsort
  #endif
  
+ #ifndef REG_STARTEND
+ #error "Git requires REG_STARTEND support. Compile with NO_REGEX=NeedsStartEnd"
+ #endif
+ 
+ static inline int regexec_buf(const regex_t *preg, const char *buf, size_t size,
+ 			      size_t nmatch, regmatch_t pmatch[], int eflags)
+ {
+ 	assert(nmatch > 0 && pmatch);
+ 	pmatch[0].rm_so = 0;
+ 	pmatch[0].rm_eo = size;
+ 	return regexec(preg, buf, nmatch, pmatch, eflags | REG_STARTEND);
+ }
+ 
  #ifndef DIR_HAS_BSD_GROUP_SEMANTICS
  # define FORCE_DIR_SET_GID S_ISGID
  #else
@@@ -1078,5 -1056,3 +1091,5 @@@ struct tm *git_gmtime_r(const time_t *
  #endif
  
  #endif
 +
 +extern int cmd_main(int, const char **);
diff --combined grep.c
index d7d00b87cb,8ed56236f0..1194d35b5d
--- a/grep.c
+++ b/grep.c
@@@ -4,8 -4,6 +4,8 @@@
  #include "xdiff-interface.h"
  #include "diff.h"
  #include "diffcore.h"
 +#include "commit.h"
 +#include "quote.h"
  
  static int grep_source_load(struct grep_source *gs);
  static int grep_source_is_binary(struct grep_source *gs);
@@@ -163,7 -161,17 +163,7 @@@ void grep_init(struct grep_opt *opt, co
  	color_set(opt->color_sep, def->color_sep);
  }
  
 -void grep_commit_pattern_type(enum grep_pattern_type pattern_type, struct grep_opt *opt)
 -{
 -	if (pattern_type != GREP_PATTERN_TYPE_UNSPECIFIED)
 -		grep_set_pattern_type_option(pattern_type, opt);
 -	else if (opt->pattern_type_option != GREP_PATTERN_TYPE_UNSPECIFIED)
 -		grep_set_pattern_type_option(opt->pattern_type_option, opt);
 -	else if (opt->extended_regexp_option)
 -		grep_set_pattern_type_option(GREP_PATTERN_TYPE_ERE, opt);
 -}
 -
 -void grep_set_pattern_type_option(enum grep_pattern_type pattern_type, struct grep_opt *opt)
 +static void grep_set_pattern_type_option(enum grep_pattern_type pattern_type, struct grep_opt *opt)
  {
  	switch (pattern_type) {
  	case GREP_PATTERN_TYPE_UNSPECIFIED:
@@@ -195,16 -203,6 +195,16 @@@
  	}
  }
  
 +void grep_commit_pattern_type(enum grep_pattern_type pattern_type, struct grep_opt *opt)
 +{
 +	if (pattern_type != GREP_PATTERN_TYPE_UNSPECIFIED)
 +		grep_set_pattern_type_option(pattern_type, opt);
 +	else if (opt->pattern_type_option != GREP_PATTERN_TYPE_UNSPECIFIED)
 +		grep_set_pattern_type_option(opt->pattern_type_option, opt);
 +	else if (opt->extended_regexp_option)
 +		grep_set_pattern_type_option(GREP_PATTERN_TYPE_ERE, opt);
 +}
 +
  static struct grep_pat *create_grep_pat(const char *pat, size_t patlen,
  					const char *origin, int no,
  					enum grep_pat_token t,
@@@ -324,16 -322,11 +324,16 @@@ static void compile_pcre_regexp(struct 
  	int erroffset;
  	int options = PCRE_MULTILINE;
  
 -	if (opt->ignore_case)
 +	if (opt->ignore_case) {
 +		if (has_non_ascii(p->pattern))
 +			p->pcre_tables = pcre_maketables();
  		options |= PCRE_CASELESS;
 +	}
 +	if (is_utf8_locale() && has_non_ascii(p->pattern))
 +		options |= PCRE_UTF8;
  
  	p->pcre_regexp = pcre_compile(p->pattern, options, &error, &erroffset,
 -			NULL);
 +				      p->pcre_tables);
  	if (!p->pcre_regexp)
  		compile_regexp_failed(p, error);
  
@@@ -367,7 -360,6 +367,7 @@@ static void free_pcre_regexp(struct gre
  {
  	pcre_free(p->pcre_regexp);
  	pcre_free(p->pcre_extra_info);
 +	pcre_free((void *)p->pcre_tables);
  }
  #else /* !USE_LIBPCRE */
  static void compile_pcre_regexp(struct grep_pat *p, const struct grep_opt *opt)
@@@ -404,68 -396,26 +404,68 @@@ static int is_fixed(const char *s, size
  	return 1;
  }
  
 +static void compile_fixed_regexp(struct grep_pat *p, struct grep_opt *opt)
 +{
 +	struct strbuf sb = STRBUF_INIT;
 +	int err;
 +	int regflags;
 +
 +	basic_regex_quote_buf(&sb, p->pattern);
 +	regflags = opt->regflags & ~REG_EXTENDED;
 +	if (opt->ignore_case)
 +		regflags |= REG_ICASE;
 +	err = regcomp(&p->regexp, sb.buf, regflags);
 +	if (opt->debug)
 +		fprintf(stderr, "fixed %s\n", sb.buf);
 +	strbuf_release(&sb);
 +	if (err) {
 +		char errbuf[1024];
 +		regerror(err, &p->regexp, errbuf, sizeof(errbuf));
 +		regfree(&p->regexp);
 +		compile_regexp_failed(p, errbuf);
 +	}
 +}
 +
  static void compile_regexp(struct grep_pat *p, struct grep_opt *opt)
  {
 +	int icase, ascii_only;
  	int err;
  
  	p->word_regexp = opt->word_regexp;
  	p->ignore_case = opt->ignore_case;
 +	icase	       = opt->regflags & REG_ICASE || p->ignore_case;
 +	ascii_only     = !has_non_ascii(p->pattern);
  
 +	/*
 +	 * Even when -F (fixed) asks us to do a non-regexp search, we
 +	 * may not be able to correctly case-fold when -i
 +	 * (ignore-case) is asked (in which case, we'll synthesize a
 +	 * regexp to match the pattern that matches regexp special
 +	 * characters literally, while ignoring case differences).  On
 +	 * the other hand, even without -F, if the pattern does not
 +	 * have any regexp special characters and there is no need for
 +	 * case-folding search, we can internally turn it into a
 +	 * simple string match using kws.  p->fixed tells us if we
 +	 * want to use kws.
 +	 */
  	if (opt->fixed || is_fixed(p->pattern, p->patternlen))
 -		p->fixed = 1;
 +		p->fixed = !icase || ascii_only;
  	else
  		p->fixed = 0;
  
  	if (p->fixed) {
 -		if (opt->regflags & REG_ICASE || p->ignore_case)
 -			p->kws = kwsalloc(tolower_trans_tbl);
 -		else
 -			p->kws = kwsalloc(NULL);
 +		p->kws = kwsalloc(icase ? tolower_trans_tbl : NULL);
  		kwsincr(p->kws, p->pattern, p->patternlen);
  		kwsprep(p->kws);
  		return;
 +	} else if (opt->fixed) {
 +		/*
 +		 * We come here when the pattern has the non-ascii
 +		 * characters we cannot case-fold, and asked to
 +		 * ignore-case.
 +		 */
 +		compile_fixed_regexp(p, opt);
 +		return;
  	}
  
  	if (opt->pcre) {
@@@ -693,10 -643,10 +693,10 @@@ static struct grep_expr *prep_header_pa
  
  	for (p = opt->header_list; p; p = p->next) {
  		if (p->token != GREP_PATTERN_HEAD)
 -			die("bug: a non-header pattern in grep header list.");
 +			die("BUG: a non-header pattern in grep header list.");
  		if (p->field < GREP_HEADER_FIELD_MIN ||
  		    GREP_HEADER_FIELD_MAX <= p->field)
 -			die("bug: unknown header field %d", p->field);
 +			die("BUG: unknown header field %d", p->field);
  		compile_regexp(p, opt);
  	}
  
@@@ -709,7 -659,7 +709,7 @@@
  
  		h = compile_pattern_atom(&pp);
  		if (!h || pp != p->next)
 -			die("bug: malformed header expr");
 +			die("BUG: malformed header expr");
  		if (!header_group[p->field]) {
  			header_group[p->field] = h;
  			continue;
@@@ -898,17 -848,6 +898,6 @@@ static int fixmatch(struct grep_pat *p
  	}
  }
  
- static int regmatch(const regex_t *preg, char *line, char *eol,
- 		    regmatch_t *match, int eflags)
- {
- #ifdef REG_STARTEND
- 	match->rm_so = 0;
- 	match->rm_eo = eol - line;
- 	eflags |= REG_STARTEND;
- #endif
- 	return regexec(preg, line, 1, match, eflags);
- }
- 
  static int patmatch(struct grep_pat *p, char *line, char *eol,
  		    regmatch_t *match, int eflags)
  {
@@@ -919,7 -858,8 +908,8 @@@
  	else if (p->pcre_regexp)
  		hit = !pcrematch(p, line, eol, match, eflags);
  	else
- 		hit = !regmatch(&p->regexp, line, eol, match, eflags);
+ 		hit = !regexec_buf(&p->regexp, line, eol - line, 1, match,
+ 				   eflags);
  
  	return hit;
  }
@@@ -1446,17 -1386,9 +1436,17 @@@ static int fill_textconv_grep(struct us
  	return 0;
  }
  
 +static int is_empty_line(const char *bol, const char *eol)
 +{
 +	while (bol < eol && isspace(*bol))
 +		bol++;
 +	return bol == eol;
 +}
 +
  static int grep_source_1(struct grep_opt *opt, struct grep_source *gs, int collect_hits)
  {
  	char *bol;
 +	char *peek_bol = NULL;
  	unsigned long left;
  	unsigned lno = 1;
  	unsigned last_hit = 0;
@@@ -1514,7 -1446,7 +1504,7 @@@
  		case GREP_BINARY_TEXT:
  			break;
  		default:
 -			die("bug: unknown binary handling mode");
 +			die("BUG: unknown binary handling mode");
  		}
  	}
  
@@@ -1601,24 -1533,8 +1591,24 @@@
  				show_function = 1;
  			goto next_line;
  		}
 -		if (show_function && match_funcname(opt, gs, bol, eol))
 -			show_function = 0;
 +		if (show_function && (!peek_bol || peek_bol < bol)) {
 +			unsigned long peek_left = left;
 +			char *peek_eol = eol;
 +
 +			/*
 +			 * Trailing empty lines are not interesting.
 +			 * Peek past them to see if they belong to the
 +			 * body of the current function.
 +			 */
 +			peek_bol = bol;
 +			while (is_empty_line(peek_bol, peek_eol)) {
 +				peek_bol = peek_eol + 1;
 +				peek_eol = end_of_line(peek_bol, &peek_left);
 +			}
 +
 +			if (match_funcname(opt, gs, peek_bol, peek_eol))
 +				show_function = 0;
 +		}
  		if (show_function ||
  		    (last_hit && lno <= last_hit + opt->post_context)) {
  			/* If the last hit is within the post context,
@@@ -1806,7 -1722,7 +1796,7 @@@ static int grep_source_load_file(struc
  	if (lstat(filename, &st) < 0) {
  	err_ret:
  		if (errno != ENOENT)
 -			error(_("'%s': %s"), filename, strerror(errno));
 +			error_errno(_("failed to stat '%s'"), filename);
  		return -1;
  	}
  	if (!S_ISREG(st.st_mode))
@@@ -1817,7 -1733,7 +1807,7 @@@
  		goto err_ret;
  	data = xmallocz(size);
  	if (st.st_size != read_in_full(i, data, size)) {
 -		error(_("'%s': short read %s"), filename, strerror(errno));
 +		error_errno(_("'%s': short read"), filename);
  		close(i);
  		free(data);
  		return -1;
diff --combined xdiff-interface.c
index 3bfc69cade,08a7313e6a..060038c2d6
--- a/xdiff-interface.c
+++ b/xdiff-interface.c
@@@ -100,9 -100,9 +100,9 @@@ static int xdiff_outf(void *priv_, mmbu
  
  /*
   * Trim down common substring at the end of the buffers,
 - * but leave at least ctx lines at the end.
 + * but end on a complete line.
   */
 -static void trim_common_tail(mmfile_t *a, mmfile_t *b, long ctx)
 +static void trim_common_tail(mmfile_t *a, mmfile_t *b)
  {
  	const int blk = 1024;
  	long trimmed = 0, recovered = 0;
@@@ -110,6 -110,9 +110,6 @@@
  	char *bp = b->ptr + b->size;
  	long smaller = (a->size < b->size) ? a->size : b->size;
  
 -	if (ctx)
 -		return;
 -
  	while (blk + trimmed <= smaller && !memcmp(ap - blk, bp - blk, blk)) {
  		trimmed += blk;
  		ap -= blk;
@@@ -131,8 -134,7 +131,8 @@@ int xdi_diff(mmfile_t *mf1, mmfile_t *m
  	if (mf1->size > MAX_XDIFF_SIZE || mf2->size > MAX_XDIFF_SIZE)
  		return -1;
  
 -	trim_common_tail(&a, &b, xecfg->ctxlen);
 +	if (!xecfg->ctxlen && !(xecfg->flags & XDL_EMIT_FUNCCONTEXT))
 +		trim_common_tail(&a, &b);
  
  	return xdl_diff(&a, &b, xpp, xecfg, xecb);
  }
@@@ -178,20 -180,20 +178,20 @@@ int read_mmfile(mmfile_t *ptr, const ch
  	return 0;
  }
  
 -void read_mmblob(mmfile_t *ptr, const unsigned char *sha1)
 +void read_mmblob(mmfile_t *ptr, const struct object_id *oid)
  {
  	unsigned long size;
  	enum object_type type;
  
 -	if (!hashcmp(sha1, null_sha1)) {
 +	if (!oidcmp(oid, &null_oid)) {
  		ptr->ptr = xstrdup("");
  		ptr->size = 0;
  		return;
  	}
  
 -	ptr->ptr = read_sha1_file(sha1, &type, &size);
 +	ptr->ptr = read_sha1_file(oid->hash, &type, &size);
  	if (!ptr->ptr || type != OBJ_BLOB)
 -		die("unable to read blob object %s", sha1_to_hex(sha1));
 +		die("unable to read blob object %s", oid_to_hex(oid));
  	ptr->size = size;
  }
  
@@@ -214,11 -216,10 +214,10 @@@ struct ff_regs 
  static long ff_regexp(const char *line, long len,
  		char *buffer, long buffer_size, void *priv)
  {
- 	char *line_buffer;
  	struct ff_regs *regs = priv;
  	regmatch_t pmatch[2];
  	int i;
- 	int result = -1;
+ 	int result;
  
  	/* Exclude terminating newline (and cr) from matching */
  	if (len > 0 && line[len-1] == '\n') {
@@@ -228,18 -229,16 +227,16 @@@
  			len--;
  	}
  
- 	line_buffer = xstrndup(line, len); /* make NUL terminated */
- 
  	for (i = 0; i < regs->nr; i++) {
  		struct ff_reg *reg = regs->array + i;
- 		if (!regexec(&reg->re, line_buffer, 2, pmatch, 0)) {
+ 		if (!regexec_buf(&reg->re, line, len, 2, pmatch, 0)) {
  			if (reg->negate)
- 				goto fail;
+ 				return -1;
  			break;
  		}
  	}
  	if (regs->nr <= i)
- 		goto fail;
+ 		return -1;
  	i = pmatch[1].rm_so >= 0 ? 1 : 0;
  	line += pmatch[i].rm_so;
  	result = pmatch[i].rm_eo - pmatch[i].rm_so;
@@@ -248,8 -247,6 +245,6 @@@
  	while (result > 0 && (isspace(line[result - 1])))
  		result--;
  	memcpy(buffer, line, result);
-  fail:
- 	free(line_buffer);
  	return result;
  }