From: Junio C Hamano <gitster@pobox.com>
Date: Fri, 1 Jun 2012 20:01:41 +0000 (-0700)
Subject: Merge branch 'rs/maint-grep-F' into maint
X-Git-Tag: v1.7.10.4~4
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/2147cb2762c67d0ec2101ab13a4c1b0ef8cab599?hp=-c

Merge branch 'rs/maint-grep-F' into maint

"git grep -e '$pattern'", unlike the case where the patterns are read from
a file, did not treat individual lines in the given pattern argument as
separate regular expressions as it should.

By René Scharfe
* rs/maint-grep-F:
grep: stop leaking line strings with -f
grep: support newline separated pattern list
grep: factor out do_append_grep_pat()
grep: factor out create_grep_pat()
---

2147cb2762c67d0ec2101ab13a4c1b0ef8cab599
diff --combined Documentation/git-grep.txt
index 4785f1c5c6,c3306f3a3b..3bec036883
--- a/Documentation/git-grep.txt
+++ b/Documentation/git-grep.txt
@@@ -20,18 -20,18 +20,20 @@@ SYNOPSI
  	   [-c | --count] [--all-match] [-q | --quiet]
  	   [--max-depth <depth>]
  	   [--color[=<when>] | --no-color]
 +	   [--break] [--heading] [-p | --show-function]
  	   [-A <post-context>] [-B <pre-context>] [-C <context>]
 +	   [-W | --function-context]
  	   [-f <file>] [-e] <pattern>
  	   [--and|--or|--not|(|)|-e <pattern>...]
 -	   [--cached | --no-index | <tree>...]
 +	   [ [--exclude-standard] [--cached | --no-index | --untracked] | <tree>...]
  	   [--] [<pathspec>...]
  
  DESCRIPTION
  -----------
  Look for specified patterns in the tracked files in the work tree, blobs
- registered in the index file, or blobs in given tree objects.
+ registered in the index file, or blobs in given tree objects.  Patterns
+ are lists of one or more search expressions separated by newline
+ characters.  An empty string as search expression matches all lines.
  
  
  CONFIGURATION
@@@ -51,20 -51,7 +53,20 @@@ OPTION
  	blobs registered in the index file.
  
  --no-index::
 -	Search files in the current directory, not just those tracked by git.
 +	Search files in the current directory that is not managed by git.
 +
 +--untracked::
 +	In addition to searching in the tracked files in the working
 +	tree, search also in untracked files.
 +
 +--no-exclude-standard::
 +	Also search in ignored files by not honoring the `.gitignore`
 +	mechanism. Only useful with `--untracked`.
 +
 +--exclude-standard::
 +	Do not pay attention to ignored files specified via the	`.gitignore`
 +	mechanism.  Only useful when searching files in the current directory
 +	with `--no-index`.
  
  -a::
  --text::
@@@ -81,9 -68,6 +83,9 @@@
  --max-depth <depth>::
  	For each <pathspec> given on command line, descend at most <depth>
  	levels of directories. A negative value means no limit.
 +	This option is ignored if <pathspec> contains active wildcards.
 +	In other words if "a*" matches a directory named "a*",
 +	"*" is matched literally so --max-depth is still effective.
  
  -w::
  --word-regexp::
@@@ -166,12 -150,14 +168,12 @@@
  	gives the default to color output.
  	Same as `--color=never`.
  
 --[ABC] <context>::
 -	Show `context` trailing (`A` -- after), or leading (`B`
 -	-- before), or both (`C` -- context) lines, and place a
 -	line containing `--` between contiguous groups of
 -	matches.
 +--break::
 +	Print an empty line between matches from different files.
  
 --<num>::
 -	A shortcut for specifying `-C<num>`.
 +--heading::
 +	Show the filename above the matches in that file instead of
 +	at the start of each shown line.
  
  -p::
  --show-function::
@@@ -181,29 -167,6 +183,29 @@@
  	patch hunk headers (see 'Defining a custom hunk-header' in
  	linkgit:gitattributes[5]).
  
 +-<num>::
 +-C <num>::
 +--context <num>::
 +	Show <num> leading and trailing lines, and place a line
 +	containing `--` between contiguous groups of matches.
 +
 +-A <num>::
 +--after-context <num>::
 +	Show <num> trailing lines, and place a line containing
 +	`--` between contiguous groups of matches.
 +
 +-B <num>::
 +--before-context <num>::
 +	Show <num> leading lines, and place a line containing
 +	`--` between contiguous groups of matches.
 +
 +-W::
 +--function-context::
 +	Show the surrounding text from the previous line containing a
 +	function name up to the one before the next function name,
 +	effectively showing the whole function in which the match was
 +	found.
 +
  -f <file>::
  	Read patterns from <file>, one per line.
  
@@@ -247,15 -210,15 +249,15 @@@
  Examples
  --------
  
 -git grep {apostrophe}time_t{apostrophe} \-- {apostrophe}*.[ch]{apostrophe}::
 +`git grep 'time_t' -- '*.[ch]'`::
  	Looks for `time_t` in all tracked .c and .h files in the working
  	directory and its subdirectories.
  
 -git grep -e {apostrophe}#define{apostrophe} --and \( -e MAX_PATH -e PATH_MAX \)::
 +`git grep -e '#define' --and \( -e MAX_PATH -e PATH_MAX \)`::
  	Looks for a line that has `#define` and either `MAX_PATH` or
  	`PATH_MAX`.
  
 -git grep --all-match -e NODE -e Unexpected::
 +`git grep --all-match -e NODE -e Unexpected`::
  	Looks for a line that has `NODE` or `Unexpected` in
  	files that have lines that match both.
  
diff --combined builtin/grep.c
index 643938d905,f1392014f1..fe1726f5ef
--- a/builtin/grep.c
+++ b/builtin/grep.c
@@@ -17,6 -17,7 +17,6 @@@
  #include "grep.h"
  #include "quote.h"
  #include "dir.h"
 -#include "thread-utils.h"
  
  static char const * const grep_usage[] = {
  	"git grep [options] [-e] <pattern> [<rev>...] [[--] <path>...]",
@@@ -29,12 -30,25 +29,12 @@@ static int use_threads = 1
  #define THREADS 8
  static pthread_t threads[THREADS];
  
 -static void *load_sha1(const unsigned char *sha1, unsigned long *size,
 -		       const char *name);
 -static void *load_file(const char *filename, size_t *sz);
 -
 -enum work_type {WORK_SHA1, WORK_FILE};
 -
  /* We use one producer thread and THREADS consumer
   * threads. The producer adds struct work_items to 'todo' and the
   * consumers pick work items from the same array.
   */
  struct work_item {
 -	enum work_type type;
 -	char *name;
 -
 -	/* if type == WORK_SHA1, then 'identifier' is a SHA1,
 -	 * otherwise type == WORK_FILE, and 'identifier' is a NUL
 -	 * terminated filename.
 -	 */
 -	void *identifier;
 +	struct grep_source source;
  	char done;
  	struct strbuf out;
  };
@@@ -60,17 -74,13 +60,17 @@@ static int all_work_added
  /* This lock protects all the variables above. */
  static pthread_mutex_t grep_mutex;
  
 -/* Used to serialize calls to read_sha1_file. */
 -static pthread_mutex_t read_sha1_mutex;
 +static inline void grep_lock(void)
 +{
 +	if (use_threads)
 +		pthread_mutex_lock(&grep_mutex);
 +}
  
 -#define grep_lock() pthread_mutex_lock(&grep_mutex)
 -#define grep_unlock() pthread_mutex_unlock(&grep_mutex)
 -#define read_sha1_lock() pthread_mutex_lock(&read_sha1_mutex)
 -#define read_sha1_unlock() pthread_mutex_unlock(&read_sha1_mutex)
 +static inline void grep_unlock(void)
 +{
 +	if (use_threads)
 +		pthread_mutex_unlock(&grep_mutex);
 +}
  
  /* Signalled when a new work_item is added to todo. */
  static pthread_cond_t cond_add;
@@@ -83,10 -93,10 +83,10 @@@ static pthread_cond_t cond_write
  /* Signalled when we are finished with everything. */
  static pthread_cond_t cond_result;
  
 -static int print_hunk_marks_between_files;
 -static int printed_something;
 +static int skip_first_line;
  
 -static void add_work(enum work_type type, char *name, void *id)
 +static void add_work(struct grep_opt *opt, enum grep_source_type type,
 +		     const char *name, const void *id)
  {
  	grep_lock();
  
@@@ -94,9 -104,9 +94,9 @@@
  		pthread_cond_wait(&cond_write, &grep_mutex);
  	}
  
 -	todo[todo_end].type = type;
 -	todo[todo_end].name = name;
 -	todo[todo_end].identifier = id;
 +	grep_source_init(&todo[todo_end].source, type, name, id);
 +	if (opt->binary != GREP_BINARY_TEXT)
 +		grep_source_load_driver(&todo[todo_end].source);
  	todo[todo_end].done = 0;
  	strbuf_reset(&todo[todo_end].out);
  	todo_end = (todo_end + 1) % ARRAY_SIZE(todo);
@@@ -124,6 -134,21 +124,6 @@@ static struct work_item *get_work(void
  	return ret;
  }
  
 -static void grep_sha1_async(struct grep_opt *opt, char *name,
 -			    const unsigned char *sha1)
 -{
 -	unsigned char *s;
 -	s = xmalloc(20);
 -	memcpy(s, sha1, 20);
 -	add_work(WORK_SHA1, name, s);
 -}
 -
 -static void grep_file_async(struct grep_opt *opt, char *name,
 -			    const char *filename)
 -{
 -	add_work(WORK_FILE, name, xstrdup(filename));
 -}
 -
  static void work_done(struct work_item *w)
  {
  	int old_done;
@@@ -135,22 -160,13 +135,22 @@@
  	    todo_done = (todo_done+1) % ARRAY_SIZE(todo)) {
  		w = &todo[todo_done];
  		if (w->out.len) {
 -			if (print_hunk_marks_between_files && printed_something)
 -				write_or_die(1, "--\n", 3);
 -			write_or_die(1, w->out.buf, w->out.len);
 -			printed_something = 1;
 +			const char *p = w->out.buf;
 +			size_t len = w->out.len;
 +
 +			/* Skip the leading hunk mark of the first file. */
 +			if (skip_first_line) {
 +				while (len) {
 +					len--;
 +					if (*p++ == '\n')
 +						break;
 +				}
 +				skip_first_line = 0;
 +			}
 +
 +			write_or_die(1, p, len);
  		}
 -		free(w->name);
 -		free(w->identifier);
 +		grep_source_clear(&w->source);
  	}
  
  	if (old_done != todo_done)
@@@ -173,8 -189,25 +173,8 @@@ static void *run(void *arg
  			break;
  
  		opt->output_priv = w;
 -		if (w->type == WORK_SHA1) {
 -			unsigned long sz;
 -			void* data = load_sha1(w->identifier, &sz, w->name);
 -
 -			if (data) {
 -				hit |= grep_buffer(opt, w->name, data, sz);
 -				free(data);
 -			}
 -		} else if (w->type == WORK_FILE) {
 -			size_t sz;
 -			void* data = load_file(w->identifier, &sz);
 -			if (data) {
 -				hit |= grep_buffer(opt, w->name, data, sz);
 -				free(data);
 -			}
 -		} else {
 -			assert(0);
 -		}
 -
 +		hit |= grep_source(opt, &w->source);
 +		grep_source_clear_data(&w->source);
  		work_done(w);
  	}
  	free_grep_patterns(arg);
@@@ -194,12 -227,10 +194,12 @@@ static void start_threads(struct grep_o
  	int i;
  
  	pthread_mutex_init(&grep_mutex, NULL);
 -	pthread_mutex_init(&read_sha1_mutex, NULL);
 +	pthread_mutex_init(&grep_read_mutex, NULL);
 +	pthread_mutex_init(&grep_attr_mutex, NULL);
  	pthread_cond_init(&cond_add, NULL);
  	pthread_cond_init(&cond_write, NULL);
  	pthread_cond_init(&cond_result, NULL);
 +	grep_use_locks = 1;
  
  	for (i = 0; i < ARRAY_SIZE(todo); i++) {
  		strbuf_init(&todo[i].out, 0);
@@@ -243,16 -274,16 +243,16 @@@ static int wait_all(void
  	}
  
  	pthread_mutex_destroy(&grep_mutex);
 -	pthread_mutex_destroy(&read_sha1_mutex);
 +	pthread_mutex_destroy(&grep_read_mutex);
 +	pthread_mutex_destroy(&grep_attr_mutex);
  	pthread_cond_destroy(&cond_add);
  	pthread_cond_destroy(&cond_write);
  	pthread_cond_destroy(&cond_result);
 +	grep_use_locks = 0;
  
  	return hit;
  }
  #else /* !NO_PTHREADS */
 -#define read_sha1_lock()
 -#define read_sha1_unlock()
  
  static int wait_all(void)
  {
@@@ -265,8 -296,11 +265,8 @@@ static int grep_config(const char *var
  	struct grep_opt *opt = cb;
  	char *color = NULL;
  
 -	switch (userdiff_config(var, value)) {
 -	case 0: break;
 -	case -1: return -1;
 -	default: return 0;
 -	}
 +	if (userdiff_config(var, value) < 0)
 +		return -1;
  
  	if (!strcmp(var, "grep.extendedregexp")) {
  		if (git_config_bool(var, value))
@@@ -282,7 -316,7 +282,7 @@@
  	}
  
  	if (!strcmp(var, "color.grep"))
 -		opt->color = git_config_colorbool(var, value, -1);
 +		opt->color = git_config_colorbool(var, value);
  	else if (!strcmp(var, "color.grep.context"))
  		color = opt->color_context;
  	else if (!strcmp(var, "color.grep.filename"))
@@@ -311,9 -345,25 +311,9 @@@ static void *lock_and_read_sha1_file(co
  {
  	void *data;
  
 -	if (use_threads) {
 -		read_sha1_lock();
 -		data = read_sha1_file(sha1, type, size);
 -		read_sha1_unlock();
 -	} else {
 -		data = read_sha1_file(sha1, type, size);
 -	}
 -	return data;
 -}
 -
 -static void *load_sha1(const unsigned char *sha1, unsigned long *size,
 -		       const char *name)
 -{
 -	enum object_type type;
 -	void *data = lock_and_read_sha1_file(sha1, &type, size);
 -
 -	if (!data)
 -		error(_("'%s': unable to read %s"), name, sha1_to_hex(sha1));
 -
 +	grep_read_lock();
 +	data = read_sha1_file(sha1, type, size);
 +	grep_read_unlock();
  	return data;
  }
  
@@@ -321,6 -371,7 +321,6 @@@ static int grep_sha1(struct grep_opt *o
  		     const char *filename, int tree_name_len)
  {
  	struct strbuf pathbuf = STRBUF_INIT;
 -	char *name;
  
  	if (opt->relative && opt->prefix_length) {
  		quote_path_relative(filename + tree_name_len, -1, &pathbuf,
@@@ -330,51 -381,87 +330,51 @@@
  		strbuf_addstr(&pathbuf, filename);
  	}
  
 -	name = strbuf_detach(&pathbuf, NULL);
 -
  #ifndef NO_PTHREADS
  	if (use_threads) {
 -		grep_sha1_async(opt, name, sha1);
 +		add_work(opt, GREP_SOURCE_SHA1, pathbuf.buf, sha1);
 +		strbuf_release(&pathbuf);
  		return 0;
  	} else
  #endif
  	{
 +		struct grep_source gs;
  		int hit;
 -		unsigned long sz;
 -		void *data = load_sha1(sha1, &sz, name);
 -		if (!data)
 -			hit = 0;
 -		else
 -			hit = grep_buffer(opt, name, data, sz);
 -
 -		free(data);
 -		free(name);
 -		return hit;
 -	}
 -}
  
 -static void *load_file(const char *filename, size_t *sz)
 -{
 -	struct stat st;
 -	char *data;
 -	int i;
 +		grep_source_init(&gs, GREP_SOURCE_SHA1, pathbuf.buf, sha1);
 +		strbuf_release(&pathbuf);
 +		hit = grep_source(opt, &gs);
  
 -	if (lstat(filename, &st) < 0) {
 -	err_ret:
 -		if (errno != ENOENT)
 -			error(_("'%s': %s"), filename, strerror(errno));
 -		return NULL;
 -	}
 -	if (!S_ISREG(st.st_mode))
 -		return NULL;
 -	*sz = xsize_t(st.st_size);
 -	i = open(filename, O_RDONLY);
 -	if (i < 0)
 -		goto err_ret;
 -	data = xmalloc(*sz + 1);
 -	if (st.st_size != read_in_full(i, data, *sz)) {
 -		error(_("'%s': short read %s"), filename, strerror(errno));
 -		close(i);
 -		free(data);
 -		return NULL;
 +		grep_source_clear(&gs);
 +		return hit;
  	}
 -	close(i);
 -	data[*sz] = 0;
 -	return data;
  }
  
  static int grep_file(struct grep_opt *opt, const char *filename)
  {
  	struct strbuf buf = STRBUF_INIT;
 -	char *name;
  
  	if (opt->relative && opt->prefix_length)
  		quote_path_relative(filename, -1, &buf, opt->prefix);
  	else
  		strbuf_addstr(&buf, filename);
 -	name = strbuf_detach(&buf, NULL);
  
  #ifndef NO_PTHREADS
  	if (use_threads) {
 -		grep_file_async(opt, name, filename);
 +		add_work(opt, GREP_SOURCE_FILE, buf.buf, filename);
 +		strbuf_release(&buf);
  		return 0;
  	} else
  #endif
  	{
 +		struct grep_source gs;
  		int hit;
 -		size_t sz;
 -		void *data = load_file(filename, &sz);
 -		if (!data)
 -			hit = 0;
 -		else
 -			hit = grep_buffer(opt, name, data, sz);
  
 -		free(data);
 -		free(name);
 +		grep_source_init(&gs, GREP_SOURCE_FILE, buf.buf, filename);
 +		strbuf_release(&buf);
 +		hit = grep_source(opt, &gs);
 +
 +		grep_source_clear(&gs);
  		return hit;
  	}
  }
@@@ -446,19 -533,18 +446,19 @@@ static int grep_cache(struct grep_opt *
  static int grep_tree(struct grep_opt *opt, const struct pathspec *pathspec,
  		     struct tree_desc *tree, struct strbuf *base, int tn_len)
  {
 -	int hit = 0, match = 0;
 +	int hit = 0;
 +	enum interesting match = entry_not_interesting;
  	struct name_entry entry;
  	int old_baselen = base->len;
  
  	while (tree_entry(tree, &entry)) {
 -		int te_len = tree_entry_len(entry.path, entry.sha1);
 +		int te_len = tree_entry_len(&entry);
  
 -		if (match != 2) {
 +		if (match != all_entries_interesting) {
  			match = tree_entry_interesting(&entry, base, tn_len, pathspec);
 -			if (match < 0)
 +			if (match == all_entries_not_interesting)
  				break;
 -			if (match == 0)
 +			if (match == entry_not_interesting)
  				continue;
  		}
  
@@@ -503,11 -589,8 +503,11 @@@ static int grep_object(struct grep_opt 
  		struct strbuf base;
  		int hit, len;
  
 +		grep_read_lock();
  		data = read_object_with_reference(obj->sha1, tree_type,
  						  &size, NULL);
 +		grep_read_unlock();
 +
  		if (!data)
  			die(_("unable to read tree (%s)"), sha1_to_hex(obj->sha1));
  
@@@ -545,15 -628,13 +545,15 @@@ static int grep_objects(struct grep_op
  	return hit;
  }
  
 -static int grep_directory(struct grep_opt *opt, const struct pathspec *pathspec)
 +static int grep_directory(struct grep_opt *opt, const struct pathspec *pathspec,
 +			  int exc_std)
  {
  	struct dir_struct dir;
  	int i, hit = 0;
  
  	memset(&dir, 0, sizeof(dir));
 -	setup_standard_excludes(&dir);
 +	if (exc_std)
 +		setup_standard_excludes(&dir);
  
  	fill_directory(&dir, pathspec->raw);
  	for (i = 0; i < dir.nr; i++) {
@@@ -600,15 -681,12 +600,12 @@@ static int file_callback(const struct o
  	if (!patterns)
  		die_errno(_("cannot open '%s'"), arg);
  	while (strbuf_getline(&sb, patterns, '\n') == 0) {
- 		char *s;
- 		size_t len;
- 
  		/* ignore empty line like grep does */
  		if (sb.len == 0)
  			continue;
  
- 		s = strbuf_detach(&sb, &len);
- 		append_grep_pat(grep_opt, s, len, arg, ++lno, GREP_PATTERN);
+ 		append_grep_pat(grep_opt, sb.buf, sb.len, arg, ++lno,
+ 				GREP_PATTERN);
  	}
  	if (!from_stdin)
  		fclose(patterns);
@@@ -660,7 -738,7 +657,7 @@@ static int help_callback(const struct o
  int cmd_grep(int argc, const char **argv, const char *prefix)
  {
  	int hit = 0;
 -	int cached = 0;
 +	int cached = 0, untracked = 0, opt_exclude = -1;
  	int seen_dashdash = 0;
  	int external_grep_allowed__ignored;
  	const char *show_in_pager = NULL, *default_pager = "dummy";
@@@ -684,12 -762,8 +681,12 @@@
  	struct option options[] = {
  		OPT_BOOLEAN(0, "cached", &cached,
  			"search in index instead of in the work tree"),
 -		OPT_BOOLEAN(0, "index", &use_index,
 -			"--no-index finds in contents not managed by git"),
 +		OPT_NEGBIT(0, "no-index", &use_index,
 +			 "finds in contents not managed by git", 1),
 +		OPT_BOOLEAN(0, "untracked", &untracked,
 +			"search in both tracked and untracked files"),
 +		OPT_SET_INT(0, "exclude-standard", &opt_exclude,
 +			    "search also in ignored files", 1),
  		OPT_GROUP(""),
  		OPT_BOOLEAN('v', "invert-match", &opt.invert,
  			"show non-matching lines"),
@@@ -736,24 -810,18 +733,24 @@@
  		OPT_BOOLEAN('c', "count", &opt.count,
  			"show the number of matches instead of matching lines"),
  		OPT__COLOR(&opt.color, "highlight matches"),
 +		OPT_BOOLEAN(0, "break", &opt.file_break,
 +			"print empty line between matches from different files"),
 +		OPT_BOOLEAN(0, "heading", &opt.heading,
 +			"show filename only once above matches from same file"),
  		OPT_GROUP(""),
 -		OPT_CALLBACK('C', NULL, &opt, "n",
 +		OPT_CALLBACK('C', "context", &opt, "n",
  			"show <n> context lines before and after matches",
  			context_callback),
 -		OPT_INTEGER('B', NULL, &opt.pre_context,
 +		OPT_INTEGER('B', "before-context", &opt.pre_context,
  			"show <n> context lines before matches"),
 -		OPT_INTEGER('A', NULL, &opt.post_context,
 +		OPT_INTEGER('A', "after-context", &opt.post_context,
  			"show <n> context lines after matches"),
  		OPT_NUMBER_CALLBACK(&opt, "shortcut for -C NUM",
  			context_callback),
  		OPT_BOOLEAN('p', "show-function", &opt.funcname,
  			"show a line with the function name before matches"),
 +		OPT_BOOLEAN('W', "function-context", &opt.funcbody,
 +			"show the surrounding function"),
  		OPT_GROUP(""),
  		OPT_CALLBACK('f', NULL, &opt, "file",
  			"read patterns from file", file_callback),
@@@ -812,6 -880,8 +809,6 @@@
  	strcpy(opt.color_sep, GIT_COLOR_CYAN);
  	opt.color = -1;
  	git_config(grep_config, &opt);
 -	if (opt.color == -1)
 -		opt.color = git_use_color_default;
  
  	/*
  	 * If there is no -- then the paths must exist in the working
@@@ -889,6 -959,19 +886,6 @@@
  	if (!opt.fixed && opt.ignore_case)
  		opt.regflags |= REG_ICASE;
  
 -#ifndef NO_PTHREADS
 -	if (online_cpus() == 1 || !grep_threads_ok(&opt))
 -		use_threads = 0;
 -
 -	if (use_threads) {
 -		if (opt.pre_context || opt.post_context)
 -			print_hunk_marks_between_files = 1;
 -		start_threads(&opt);
 -	}
 -#else
 -	use_threads = 0;
 -#endif
 -
  	compile_grep_patterns(&opt);
  
  	/* Check revs and then paths */
@@@ -910,23 -993,6 +907,23 @@@
  		break;
  	}
  
 +#ifndef NO_PTHREADS
 +	if (list.nr || cached || online_cpus() == 1)
 +		use_threads = 0;
 +#else
 +	use_threads = 0;
 +#endif
 +
 +#ifndef NO_PTHREADS
 +	if (use_threads) {
 +		if (!(opt.name_only || opt.unmatch_name_only || opt.count)
 +		    && (opt.pre_context || opt.post_context ||
 +			opt.file_break || opt.funcbody))
 +			skip_first_line = 1;
 +		start_threads(&opt);
 +	}
 +#endif
 +
  	/* The rest are paths */
  	if (!seen_dashdash) {
  		int j;
@@@ -962,16 -1028,13 +959,16 @@@
  	if (!show_in_pager)
  		setup_pager();
  
 +	if (!use_index && (untracked || cached))
 +		die(_("--cached or --untracked cannot be used with --no-index."));
  
 -	if (!use_index) {
 -		if (cached)
 -			die(_("--cached cannot be used with --no-index."));
 +	if (!use_index || untracked) {
 +		int use_exclude = (opt_exclude < 0) ? use_index : !!opt_exclude;
  		if (list.nr)
 -			die(_("--no-index cannot be used with revs."));
 -		hit = grep_directory(&opt, &pathspec);
 +			die(_("--no-index or --untracked cannot be used with revs."));
 +		hit = grep_directory(&opt, &pathspec, use_exclude);
 +	} else if (0 <= opt_exclude) {
 +		die(_("--[no-]exclude-standard cannot be used for tracked contents."));
  	} else if (!list.nr) {
  		if (!cached)
  			setup_work_tree();
diff --combined grep.c
index f8ffa46209,02258039d9..04e3ec6c6e
--- a/grep.c
+++ b/grep.c
@@@ -3,18 -3,64 +3,64 @@@
  #include "userdiff.h"
  #include "xdiff-interface.h"
  
- void append_header_grep_pattern(struct grep_opt *opt, enum grep_header_field field, const char *pat)
+ static struct grep_pat *create_grep_pat(const char *pat, size_t patlen,
+ 					const char *origin, int no,
+ 					enum grep_pat_token t,
+ 					enum grep_header_field field)
  {
  	struct grep_pat *p = xcalloc(1, sizeof(*p));
- 	p->pattern = pat;
- 	p->patternlen = strlen(pat);
- 	p->origin = "header";
- 	p->no = 0;
- 	p->token = GREP_PATTERN_HEAD;
+ 	p->pattern = xmemdupz(pat, patlen);
+ 	p->patternlen = patlen;
+ 	p->origin = origin;
+ 	p->no = no;
+ 	p->token = t;
  	p->field = field;
- 	*opt->header_tail = p;
- 	opt->header_tail = &p->next;
+ 	return p;
+ }
+ 
+ static void do_append_grep_pat(struct grep_pat ***tail, struct grep_pat *p)
+ {
+ 	**tail = p;
+ 	*tail = &p->next;
  	p->next = NULL;
+ 
+ 	switch (p->token) {
+ 	case GREP_PATTERN: /* atom */
+ 	case GREP_PATTERN_HEAD:
+ 	case GREP_PATTERN_BODY:
+ 		for (;;) {
+ 			struct grep_pat *new_pat;
+ 			size_t len = 0;
+ 			char *cp = p->pattern + p->patternlen, *nl = NULL;
+ 			while (++len <= p->patternlen) {
+ 				if (*(--cp) == '\n') {
+ 					nl = cp;
+ 					break;
+ 				}
+ 			}
+ 			if (!nl)
+ 				break;
+ 			new_pat = create_grep_pat(nl + 1, len - 1, p->origin,
+ 						  p->no, p->token, p->field);
+ 			new_pat->next = p->next;
+ 			if (!p->next)
+ 				*tail = &new_pat->next;
+ 			p->next = new_pat;
+ 			*nl = '\0';
+ 			p->patternlen -= len;
+ 		}
+ 		break;
+ 	default:
+ 		break;
+ 	}
+ }
+ 
+ void append_header_grep_pattern(struct grep_opt *opt,
+ 				enum grep_header_field field, const char *pat)
+ {
+ 	struct grep_pat *p = create_grep_pat(pat, strlen(pat), "header", 0,
+ 					     GREP_PATTERN_HEAD, field);
+ 	do_append_grep_pat(&opt->header_tail, p);
  }
  
  void append_grep_pattern(struct grep_opt *opt, const char *pat,
@@@ -26,15 -72,8 +72,8 @@@
  void append_grep_pat(struct grep_opt *opt, const char *pat, size_t patlen,
  		     const char *origin, int no, enum grep_pat_token t)
  {
- 	struct grep_pat *p = xcalloc(1, sizeof(*p));
- 	p->pattern = pat;
- 	p->patternlen = patlen;
- 	p->origin = origin;
- 	p->no = no;
- 	p->token = t;
- 	*opt->pattern_tail = p;
- 	opt->pattern_tail = &p->next;
- 	p->next = NULL;
+ 	struct grep_pat *p = create_grep_pat(pat, patlen, origin, no, t, 0);
+ 	do_append_grep_pat(&opt->pattern_tail, p);
  }
  
  struct grep_opt *grep_opt_dup(const struct grep_opt *opt)
@@@ -79,7 -118,7 +118,7 @@@ static void compile_pcre_regexp(struct 
  {
  	const char *error;
  	int erroffset;
 -	int options = 0;
 +	int options = PCRE_MULTILINE;
  
  	if (opt->ignore_case)
  		options |= PCRE_CASELESS;
@@@ -137,45 -176,16 +176,45 @@@ static void free_pcre_regexp(struct gre
  }
  #endif /* !USE_LIBPCRE */
  
 +static int is_fixed(const char *s, size_t len)
 +{
 +	size_t i;
 +
 +	/* regcomp cannot accept patterns with NULs so we
 +	 * consider any pattern containing a NUL fixed.
 +	 */
 +	if (memchr(s, 0, len))
 +		return 1;
 +
 +	for (i = 0; i < len; i++) {
 +		if (is_regex_special(s[i]))
 +			return 0;
 +	}
 +
 +	return 1;
 +}
 +
  static void compile_regexp(struct grep_pat *p, struct grep_opt *opt)
  {
  	int err;
  
  	p->word_regexp = opt->word_regexp;
  	p->ignore_case = opt->ignore_case;
 -	p->fixed = opt->fixed;
  
 -	if (p->fixed)
 +	if (opt->fixed || is_fixed(p->pattern, p->patternlen))
 +		p->fixed = 1;
 +	else
 +		p->fixed = 0;
 +
 +	if (p->fixed) {
 +		if (opt->regflags & REG_ICASE || p->ignore_case)
 +			p->kws = kwsalloc(tolower_trans_tbl);
 +		else
 +			p->kws = kwsalloc(NULL);
 +		kwsincr(p->kws, p->pattern, p->patternlen);
 +		kwsprep(p->kws);
  		return;
 +	}
  
  	if (opt->pcre) {
  		compile_pcre_regexp(p, opt);
@@@ -318,7 -328,7 +357,7 @@@ static struct grep_expr *prep_header_pa
  
  	if (!opt->header_list)
  		return NULL;
 -	p = opt->header_list;
 +
  	for (p = opt->header_list; p; p = p->next) {
  		if (p->token != GREP_PATTERN_HEAD)
  			die("bug: a non-header pattern in grep header list.");
@@@ -424,12 -434,11 +463,13 @@@ void free_grep_patterns(struct grep_op
  		case GREP_PATTERN: /* atom */
  		case GREP_PATTERN_HEAD:
  		case GREP_PATTERN_BODY:
 -			if (p->pcre_regexp)
 +			if (p->kws)
 +				kwsfree(p->kws);
 +			else if (p->pcre_regexp)
  				free_pcre_regexp(p);
  			else
  				regfree(&p->regexp);
+ 			free(p->pattern);
  			break;
  		default:
  			break;
@@@ -461,7 -470,7 +501,7 @@@ static int word_char(char ch
  static void output_color(struct grep_opt *opt, const void *data, size_t size,
  			 const char *color)
  {
 -	if (opt->color && color && color[0]) {
 +	if (want_color(opt->color) && color && color[0]) {
  		opt->output(opt, color, strlen(color));
  		opt->output(opt, data, size);
  		opt->output(opt, GIT_COLOR_RESET, strlen(GIT_COLOR_RESET));
@@@ -486,14 -495,26 +526,14 @@@ static void show_name(struct grep_opt *
  static int fixmatch(struct grep_pat *p, char *line, char *eol,
  		    regmatch_t *match)
  {
 -	char *hit;
 -
 -	if (p->ignore_case) {
 -		char *s = line;
 -		do {
 -			hit = strcasestr(s, p->pattern);
 -			if (hit)
 -				break;
 -			s += strlen(s) + 1;
 -		} while (s < eol);
 -	} else
 -		hit = memmem(line, eol - line, p->pattern, p->patternlen);
 -
 -	if (!hit) {
 +	struct kwsmatch kwsm;
 +	size_t offset = kwsexec(p->kws, line, eol - line, &kwsm);
 +	if (offset == -1) {
  		match->rm_so = match->rm_eo = -1;
  		return REG_NOMATCH;
 -	}
 -	else {
 -		match->rm_so = hit - line;
 -		match->rm_eo = match->rm_so + p->patternlen;
 +	} else {
 +		match->rm_so = offset;
 +		match->rm_eo = match->rm_so + kwsm.size[0];
  		return 0;
  	}
  }
@@@ -740,10 -761,7 +780,10 @@@ static void show_line(struct grep_opt *
  	int rest = eol - bol;
  	char *line_color = NULL;
  
 -	if (opt->pre_context || opt->post_context) {
 +	if (opt->file_break && opt->last_shown == 0) {
 +		if (opt->show_hunk_mark)
 +			opt->output(opt, "\n", 1);
 +	} else if (opt->pre_context || opt->post_context || opt->funcbody) {
  		if (opt->last_shown == 0) {
  			if (opt->show_hunk_mark) {
  				output_color(opt, "--", 2, opt->color_sep);
@@@ -754,13 -772,9 +794,13 @@@
  			opt->output(opt, "\n", 1);
  		}
  	}
 +	if (opt->heading && opt->last_shown == 0) {
 +		output_color(opt, name, strlen(name), opt->color_filename);
 +		opt->output(opt, "\n", 1);
 +	}
  	opt->last_shown = lno;
  
 -	if (opt->pathname) {
 +	if (!opt->heading && opt->pathname) {
  		output_color(opt, name, strlen(name), opt->color_filename);
  		output_sep(opt, sign);
  	}
@@@ -801,51 -815,10 +841,51 @@@
  	opt->output(opt, "\n", 1);
  }
  
 -static int match_funcname(struct grep_opt *opt, char *bol, char *eol)
 +#ifndef NO_PTHREADS
 +int grep_use_locks;
 +
 +/*
 + * This lock protects access to the gitattributes machinery, which is
 + * not thread-safe.
 + */
 +pthread_mutex_t grep_attr_mutex;
 +
 +static inline void grep_attr_lock(void)
 +{
 +	if (grep_use_locks)
 +		pthread_mutex_lock(&grep_attr_mutex);
 +}
 +
 +static inline void grep_attr_unlock(void)
 +{
 +	if (grep_use_locks)
 +		pthread_mutex_unlock(&grep_attr_mutex);
 +}
 +
 +/*
 + * Same as git_attr_mutex, but protecting the thread-unsafe object db access.
 + */
 +pthread_mutex_t grep_read_mutex;
 +
 +#else
 +#define grep_attr_lock()
 +#define grep_attr_unlock()
 +#endif
 +
 +static int match_funcname(struct grep_opt *opt, struct grep_source *gs, char *bol, char *eol)
  {
  	xdemitconf_t *xecfg = opt->priv;
 -	if (xecfg && xecfg->find_func) {
 +	if (xecfg && !xecfg->find_func) {
 +		grep_source_load_driver(gs);
 +		if (gs->driver->funcname.pattern) {
 +			const struct userdiff_funcname *pe = &gs->driver->funcname;
 +			xdiff_set_find_func(xecfg, pe->pattern, pe->cflags);
 +		} else {
 +			xecfg = opt->priv = NULL;
 +		}
 +	}
 +
 +	if (xecfg) {
  		char buf[1];
  		return xecfg->find_func(bol, eol - bol, buf, 1,
  					xecfg->find_func_priv) >= 0;
@@@ -858,34 -831,31 +898,34 @@@
  	return 0;
  }
  
 -static void show_funcname_line(struct grep_opt *opt, const char *name,
 -			       char *buf, char *bol, unsigned lno)
 +static void show_funcname_line(struct grep_opt *opt, struct grep_source *gs,
 +			       char *bol, unsigned lno)
  {
 -	while (bol > buf) {
 +	while (bol > gs->buf) {
  		char *eol = --bol;
  
 -		while (bol > buf && bol[-1] != '\n')
 +		while (bol > gs->buf && bol[-1] != '\n')
  			bol--;
  		lno--;
  
  		if (lno <= opt->last_shown)
  			break;
  
 -		if (match_funcname(opt, bol, eol)) {
 -			show_line(opt, bol, eol, name, lno, '=');
 +		if (match_funcname(opt, gs, bol, eol)) {
 +			show_line(opt, bol, eol, gs->name, lno, '=');
  			break;
  		}
  	}
  }
  
 -static void show_pre_context(struct grep_opt *opt, const char *name, char *buf,
 -			     char *bol, unsigned lno)
 +static void show_pre_context(struct grep_opt *opt, struct grep_source *gs,
 +			     char *bol, char *end, unsigned lno)
  {
  	unsigned cur = lno, from = 1, funcname_lno = 0;
 -	int funcname_needed = opt->funcname;
 +	int funcname_needed = !!opt->funcname;
 +
 +	if (opt->funcbody && !match_funcname(opt, gs, bol, end))
 +		funcname_needed = 2;
  
  	if (opt->pre_context < lno)
  		from = lno - opt->pre_context;
@@@ -893,14 -863,13 +933,14 @@@
  		from = opt->last_shown + 1;
  
  	/* Rewind. */
 -	while (bol > buf && cur > from) {
 +	while (bol > gs->buf &&
 +	       cur > (funcname_needed == 2 ? opt->last_shown + 1 : from)) {
  		char *eol = --bol;
  
 -		while (bol > buf && bol[-1] != '\n')
 +		while (bol > gs->buf && bol[-1] != '\n')
  			bol--;
  		cur--;
 -		if (funcname_needed && match_funcname(opt, bol, eol)) {
 +		if (funcname_needed && match_funcname(opt, gs, bol, eol)) {
  			funcname_lno = cur;
  			funcname_needed = 0;
  		}
@@@ -908,7 -877,7 +948,7 @@@
  
  	/* We need to look even further back to find a function signature. */
  	if (opt->funcname && funcname_needed)
 -		show_funcname_line(opt, name, buf, bol, cur);
 +		show_funcname_line(opt, gs, bol, cur);
  
  	/* Back forward. */
  	while (cur < lno) {
@@@ -916,7 -885,7 +956,7 @@@
  
  		while (*eol != '\n')
  			eol++;
 -		show_line(opt, bol, eol, name, cur, sign);
 +		show_line(opt, bol, eol, gs->name, cur, sign);
  		bol = eol + 1;
  		cur++;
  	}
@@@ -978,49 -947,52 +1018,49 @@@ static int look_ahead(struct grep_opt *
  	return 0;
  }
  
 -int grep_threads_ok(const struct grep_opt *opt)
 -{
 -	/* If this condition is true, then we may use the attribute
 -	 * machinery in grep_buffer_1. The attribute code is not
 -	 * thread safe, so we disable the use of threads.
 -	 */
 -	if (opt->funcname && !opt->unmatch_name_only && !opt->status_only &&
 -	    !opt->name_only)
 -		return 0;
 -
 -	return 1;
 -}
 -
  static void std_output(struct grep_opt *opt, const void *buf, size_t size)
  {
  	fwrite(buf, size, 1, stdout);
  }
  
 -static int grep_buffer_1(struct grep_opt *opt, const char *name,
 -			 char *buf, unsigned long size, int collect_hits)
 +static int grep_source_1(struct grep_opt *opt, struct grep_source *gs, int collect_hits)
  {
 -	char *bol = buf;
 -	unsigned long left = size;
 +	char *bol;
 +	unsigned long left;
  	unsigned lno = 1;
  	unsigned last_hit = 0;
  	int binary_match_only = 0;
  	unsigned count = 0;
  	int try_lookahead = 0;
 +	int show_function = 0;
  	enum grep_context ctx = GREP_CONTEXT_HEAD;
  	xdemitconf_t xecfg;
  
  	if (!opt->output)
  		opt->output = std_output;
  
 -	if (opt->last_shown && (opt->pre_context || opt->post_context) &&
 -	    opt->output == std_output)
 -		opt->show_hunk_mark = 1;
 +	if (opt->pre_context || opt->post_context || opt->file_break ||
 +	    opt->funcbody) {
 +		/* Show hunk marks, except for the first file. */
 +		if (opt->last_shown)
 +			opt->show_hunk_mark = 1;
 +		/*
 +		 * If we're using threads then we can't easily identify
 +		 * the first file.  Always put hunk marks in that case
 +		 * and skip the very first one later in work_done().
 +		 */
 +		if (opt->output != std_output)
 +			opt->show_hunk_mark = 1;
 +	}
  	opt->last_shown = 0;
  
  	switch (opt->binary) {
  	case GREP_BINARY_DEFAULT:
 -		if (buffer_is_binary(buf, size))
 +		if (grep_source_is_binary(gs))
  			binary_match_only = 1;
  		break;
  	case GREP_BINARY_NOMATCH:
 -		if (buffer_is_binary(buf, size))
 +		if (grep_source_is_binary(gs))
  			return 0; /* Assume unmatch */
  		break;
  	case GREP_BINARY_TEXT:
@@@ -1030,15 -1002,17 +1070,15 @@@
  	}
  
  	memset(&xecfg, 0, sizeof(xecfg));
 -	if (opt->funcname && !opt->unmatch_name_only && !opt->status_only &&
 -	    !opt->name_only && !binary_match_only && !collect_hits) {
 -		struct userdiff_driver *drv = userdiff_find_by_path(name);
 -		if (drv && drv->funcname.pattern) {
 -			const struct userdiff_funcname *pe = &drv->funcname;
 -			xdiff_set_find_func(&xecfg, pe->pattern, pe->cflags);
 -			opt->priv = &xecfg;
 -		}
 -	}
 +	opt->priv = &xecfg;
 +
  	try_lookahead = should_lookahead(opt);
  
 +	if (grep_source_load(gs) < 0)
 +		return 0;
 +
 +	bol = gs->buf;
 +	left = gs->size;
  	while (left) {
  		char *eol, ch;
  		int hit;
@@@ -1054,8 -1028,7 +1094,8 @@@
  		 */
  		if (try_lookahead
  		    && !(last_hit
 -			 && lno <= last_hit + opt->post_context)
 +			 && (show_function ||
 +			     lno <= last_hit + opt->post_context))
  		    && look_ahead(opt, &left, &lno, &bol))
  			break;
  		eol = end_of_line(bol, &left);
@@@ -1087,14 -1060,14 +1127,14 @@@
  			if (opt->status_only)
  				return 1;
  			if (opt->name_only) {
 -				show_name(opt, name);
 +				show_name(opt, gs->name);
  				return 1;
  			}
  			if (opt->count)
  				goto next_line;
  			if (binary_match_only) {
  				opt->output(opt, "Binary file ", 12);
 -				output_color(opt, name, strlen(name),
 +				output_color(opt, gs->name, strlen(gs->name),
  					     opt->color_filename);
  				opt->output(opt, " matches\n", 9);
  				return 1;
@@@ -1102,24 -1075,19 +1142,24 @@@
  			/* Hit at this line.  If we haven't shown the
  			 * pre-context lines, we would need to show them.
  			 */
 -			if (opt->pre_context)
 -				show_pre_context(opt, name, buf, bol, lno);
 +			if (opt->pre_context || opt->funcbody)
 +				show_pre_context(opt, gs, bol, eol, lno);
  			else if (opt->funcname)
 -				show_funcname_line(opt, name, buf, bol, lno);
 -			show_line(opt, bol, eol, name, lno, ':');
 +				show_funcname_line(opt, gs, bol, lno);
 +			show_line(opt, bol, eol, gs->name, lno, ':');
  			last_hit = lno;
 +			if (opt->funcbody)
 +				show_function = 1;
 +			goto next_line;
  		}
 -		else if (last_hit &&
 -			 lno <= last_hit + opt->post_context) {
 +		if (show_function && match_funcname(opt, gs, bol, eol))
 +			show_function = 0;
 +		if (show_function ||
 +		    (last_hit && lno <= last_hit + opt->post_context)) {
  			/* If the last hit is within the post context,
  			 * we need to show this line.
  			 */
 -			show_line(opt, bol, eol, name, lno, '-');
 +			show_line(opt, bol, eol, gs->name, lno, '-');
  		}
  
  	next_line:
@@@ -1137,7 -1105,7 +1177,7 @@@
  		return 0;
  	if (opt->unmatch_name_only) {
  		/* We did not see any hit, so we want to show this */
 -		show_name(opt, name);
 +		show_name(opt, gs->name);
  		return 1;
  	}
  
@@@ -1151,7 -1119,7 +1191,7 @@@
  	 */
  	if (opt->count && count) {
  		char buf[32];
 -		output_color(opt, name, strlen(name), opt->color_filename);
 +		output_color(opt, gs->name, strlen(gs->name), opt->color_filename);
  		output_sep(opt, ':');
  		snprintf(buf, sizeof(buf), "%u\n", count);
  		opt->output(opt, buf, strlen(buf));
@@@ -1186,174 -1154,23 +1226,174 @@@ static int chk_hit_marker(struct grep_e
  	}
  }
  
 -int grep_buffer(struct grep_opt *opt, const char *name, char *buf, unsigned long size)
 +int grep_source(struct grep_opt *opt, struct grep_source *gs)
  {
  	/*
  	 * we do not have to do the two-pass grep when we do not check
  	 * buffer-wide "all-match".
  	 */
  	if (!opt->all_match)
 -		return grep_buffer_1(opt, name, buf, size, 0);
 +		return grep_source_1(opt, gs, 0);
  
  	/* Otherwise the toplevel "or" terms hit a bit differently.
  	 * We first clear hit markers from them.
  	 */
  	clr_hit_marker(opt->pattern_expression);
 -	grep_buffer_1(opt, name, buf, size, 1);
 +	grep_source_1(opt, gs, 1);
  
  	if (!chk_hit_marker(opt->pattern_expression))
  		return 0;
  
 -	return grep_buffer_1(opt, name, buf, size, 0);
 +	return grep_source_1(opt, gs, 0);
 +}
 +
 +int grep_buffer(struct grep_opt *opt, char *buf, unsigned long size)
 +{
 +	struct grep_source gs;
 +	int r;
 +
 +	grep_source_init(&gs, GREP_SOURCE_BUF, NULL, NULL);
 +	gs.buf = buf;
 +	gs.size = size;
 +
 +	r = grep_source(opt, &gs);
 +
 +	grep_source_clear(&gs);
 +	return r;
 +}
 +
 +void grep_source_init(struct grep_source *gs, enum grep_source_type type,
 +		      const char *name, const void *identifier)
 +{
 +	gs->type = type;
 +	gs->name = name ? xstrdup(name) : NULL;
 +	gs->buf = NULL;
 +	gs->size = 0;
 +	gs->driver = NULL;
 +
 +	switch (type) {
 +	case GREP_SOURCE_FILE:
 +		gs->identifier = xstrdup(identifier);
 +		break;
 +	case GREP_SOURCE_SHA1:
 +		gs->identifier = xmalloc(20);
 +		memcpy(gs->identifier, identifier, 20);
 +		break;
 +	case GREP_SOURCE_BUF:
 +		gs->identifier = NULL;
 +	}
 +}
 +
 +void grep_source_clear(struct grep_source *gs)
 +{
 +	free(gs->name);
 +	gs->name = NULL;
 +	free(gs->identifier);
 +	gs->identifier = NULL;
 +	grep_source_clear_data(gs);
 +}
 +
 +void grep_source_clear_data(struct grep_source *gs)
 +{
 +	switch (gs->type) {
 +	case GREP_SOURCE_FILE:
 +	case GREP_SOURCE_SHA1:
 +		free(gs->buf);
 +		gs->buf = NULL;
 +		gs->size = 0;
 +		break;
 +	case GREP_SOURCE_BUF:
 +		/* leave user-provided buf intact */
 +		break;
 +	}
 +}
 +
 +static int grep_source_load_sha1(struct grep_source *gs)
 +{
 +	enum object_type type;
 +
 +	grep_read_lock();
 +	gs->buf = read_sha1_file(gs->identifier, &type, &gs->size);
 +	grep_read_unlock();
 +
 +	if (!gs->buf)
 +		return error(_("'%s': unable to read %s"),
 +			     gs->name,
 +			     sha1_to_hex(gs->identifier));
 +	return 0;
 +}
 +
 +static int grep_source_load_file(struct grep_source *gs)
 +{
 +	const char *filename = gs->identifier;
 +	struct stat st;
 +	char *data;
 +	size_t size;
 +	int i;
 +
 +	if (lstat(filename, &st) < 0) {
 +	err_ret:
 +		if (errno != ENOENT)
 +			error(_("'%s': %s"), filename, strerror(errno));
 +		return -1;
 +	}
 +	if (!S_ISREG(st.st_mode))
 +		return -1;
 +	size = xsize_t(st.st_size);
 +	i = open(filename, O_RDONLY);
 +	if (i < 0)
 +		goto err_ret;
 +	data = xmalloc(size + 1);
 +	if (st.st_size != read_in_full(i, data, size)) {
 +		error(_("'%s': short read %s"), filename, strerror(errno));
 +		close(i);
 +		free(data);
 +		return -1;
 +	}
 +	close(i);
 +	data[size] = 0;
 +
 +	gs->buf = data;
 +	gs->size = size;
 +	return 0;
 +}
 +
 +int grep_source_load(struct grep_source *gs)
 +{
 +	if (gs->buf)
 +		return 0;
 +
 +	switch (gs->type) {
 +	case GREP_SOURCE_FILE:
 +		return grep_source_load_file(gs);
 +	case GREP_SOURCE_SHA1:
 +		return grep_source_load_sha1(gs);
 +	case GREP_SOURCE_BUF:
 +		return gs->buf ? 0 : -1;
 +	}
 +	die("BUG: invalid grep_source type");
 +}
 +
 +void grep_source_load_driver(struct grep_source *gs)
 +{
 +	if (gs->driver)
 +		return;
 +
 +	grep_attr_lock();
 +	gs->driver = userdiff_find_by_path(gs->name);
 +	if (!gs->driver)
 +		gs->driver = userdiff_find_by_name("default");
 +	grep_attr_unlock();
 +}
 +
 +int grep_source_is_binary(struct grep_source *gs)
 +{
 +	grep_source_load_driver(gs);
 +	if (gs->driver->binary != -1)
 +		return gs->driver->binary;
 +
 +	if (!grep_source_load(gs))
 +		return buffer_is_binary(gs->buf, gs->size);
 +
 +	return 0;
  }
diff --combined grep.h
index 36e49d8255,5b083affe1..ed7de6bec8
--- a/grep.h
+++ b/grep.h
@@@ -7,9 -7,6 +7,9 @@@
  typedef int pcre;
  typedef int pcre_extra;
  #endif
 +#include "kwset.h"
 +#include "thread-utils.h"
 +#include "userdiff.h"
  
  enum grep_pat_token {
  	GREP_PATTERN,
@@@ -38,13 -35,12 +38,13 @@@ struct grep_pat 
  	const char *origin;
  	int no;
  	enum grep_pat_token token;
- 	const char *pattern;
+ 	char *pattern;
  	size_t patternlen;
  	enum grep_header_field field;
  	regex_t regexp;
  	pcre *pcre_regexp;
  	pcre_extra *pcre_extra_info;
 +	kwset_t kws;
  	unsigned fixed:1;
  	unsigned ignore_case:1;
  	unsigned word_regexp:1;
@@@ -102,7 -98,6 +102,7 @@@ struct grep_opt 
  	int color;
  	int max_depth;
  	int funcname;
 +	int funcbody;
  	char color_context[COLOR_MAXLEN];
  	char color_filename[COLOR_MAXLEN];
  	char color_function[COLOR_MAXLEN];
@@@ -115,8 -110,6 +115,8 @@@
  	unsigned post_context;
  	unsigned last_shown;
  	int show_hunk_mark;
 +	int file_break;
 +	int heading;
  	void *priv;
  
  	void (*output)(struct grep_opt *opt, const void *data, size_t size);
@@@ -128,61 -121,9 +128,61 @@@ extern void append_grep_pattern(struct 
  extern void append_header_grep_pattern(struct grep_opt *, enum grep_header_field, const char *);
  extern void compile_grep_patterns(struct grep_opt *opt);
  extern void free_grep_patterns(struct grep_opt *opt);
 -extern int grep_buffer(struct grep_opt *opt, const char *name, char *buf, unsigned long size);
 +extern int grep_buffer(struct grep_opt *opt, char *buf, unsigned long size);
 +
 +struct grep_source {
 +	char *name;
 +
 +	enum grep_source_type {
 +		GREP_SOURCE_SHA1,
 +		GREP_SOURCE_FILE,
 +		GREP_SOURCE_BUF,
 +	} type;
 +	void *identifier;
 +
 +	char *buf;
 +	unsigned long size;
 +
 +	struct userdiff_driver *driver;
 +};
 +
 +void grep_source_init(struct grep_source *gs, enum grep_source_type type,
 +		      const char *name, const void *identifier);
 +int grep_source_load(struct grep_source *gs);
 +void grep_source_clear_data(struct grep_source *gs);
 +void grep_source_clear(struct grep_source *gs);
 +void grep_source_load_driver(struct grep_source *gs);
 +int grep_source_is_binary(struct grep_source *gs);
 +
 +int grep_source(struct grep_opt *opt, struct grep_source *gs);
  
  extern struct grep_opt *grep_opt_dup(const struct grep_opt *opt);
  extern int grep_threads_ok(const struct grep_opt *opt);
  
 +#ifndef NO_PTHREADS
 +/*
 + * Mutex used around access to the attributes machinery if
 + * opt->use_threads.  Must be initialized/destroyed by callers!
 + */
 +extern int grep_use_locks;
 +extern pthread_mutex_t grep_attr_mutex;
 +extern pthread_mutex_t grep_read_mutex;
 +
 +static inline void grep_read_lock(void)
 +{
 +	if (grep_use_locks)
 +		pthread_mutex_lock(&grep_read_mutex);
 +}
 +
 +static inline void grep_read_unlock(void)
 +{
 +	if (grep_use_locks)
 +		pthread_mutex_unlock(&grep_read_mutex);
 +}
 +
 +#else
 +#define grep_read_lock()
 +#define grep_read_unlock()
 +#endif
 +
  #endif
diff --combined t/t7810-grep.sh
index d9ad633310,bc9a522085..24e9b1974d
--- a/t/t7810-grep.sh
+++ b/t/t7810-grep.sh
@@@ -47,13 -47,6 +47,13 @@@ test_expect_success setup 
  	echo vvv >t/v &&
  	mkdir t/a &&
  	echo vvv >t/a/v &&
 +	{
 +		echo "line without leading space1"
 +		echo " line with leading space1"
 +		echo " line with leading space2"
 +		echo " line with leading space3"
 +		echo "line without leading space2"
 +	} >space &&
  	git add . &&
  	test_tick &&
  	git commit -m initial
@@@ -252,28 -245,6 +252,28 @@@ d
  	'
  done
  
 +cat >expected <<EOF
 +file
 +EOF
 +test_expect_success 'grep -l -C' '
 +	git grep -l -C1 foo >actual &&
 +	test_cmp expected actual
 +'
 +
 +cat >expected <<EOF
 +file:5
 +EOF
 +test_expect_success 'grep -l -C' '
 +	git grep -c -C1 foo >actual &&
 +	test_cmp expected actual
 +'
 +
 +test_expect_success 'grep -L -C' '
 +	git ls-files >expected &&
 +	git grep -L -C1 nonexistent_string >actual &&
 +	test_cmp expected actual
 +'
 +
  cat >expected <<EOF
  file:foo mmap bar_mmap
  EOF
@@@ -351,6 -322,11 +351,11 @@@ test_expect_success 'grep -f, multiple 
  	test_cmp expected actual
  '
  
+ test_expect_success 'grep, multiple patterns' '
+ 	git grep "$(cat patterns)" >actual &&
+ 	test_cmp expected actual
+ '
+ 
  cat >expected <<EOF
  file:foo mmap bar
  file:foo_mmap bar
@@@ -538,34 -514,6 +543,34 @@@ test_expect_success 'grep -p -B5' 
  	test_cmp expected actual
  '
  
 +cat >expected <<EOF
 +hello.c=int main(int argc, const char **argv)
 +hello.c-{
 +hello.c-	printf("Hello world.\n");
 +hello.c:	return 0;
 +hello.c-	/* char ?? */
 +hello.c-}
 +EOF
 +
 +test_expect_success 'grep -W' '
 +	git grep -W return >actual &&
 +	test_cmp expected actual
 +'
 +
 +cat >expected <<EOF
 +hello.c=	printf("Hello world.\n");
 +hello.c:	return 0;
 +hello.c-	/* char ?? */
 +EOF
 +
 +test_expect_success 'grep -W with userdiff' '
 +	test_when_finished "rm -f .gitattributes" &&
 +	git config diff.custom.xfuncname "(printf.*|})$" &&
 +	echo "hello.c diff=custom" >.gitattributes &&
 +	git grep -W return >actual &&
 +	test_cmp expected actual
 +'
 +
  test_expect_success 'grep from a subdirectory to search wider area (1)' '
  	mkdir -p s &&
  	(
@@@ -597,6 -545,7 +602,6 @@@ test_expect_success 'outside of git rep
  	mkdir -p non/git/sub &&
  	echo hello >non/git/file1 &&
  	echo world >non/git/sub/file2 &&
 -	echo ".*o*" >non/git/.gitignore &&
  	{
  		echo file1:hello &&
  		echo sub/file2:world
@@@ -613,23 -562,6 +618,23 @@@
  		test_must_fail git grep o &&
  		git grep --no-index o >../../actual.sub &&
  		test_cmp ../../expect.sub ../../actual.sub
 +	) &&
 +
 +	echo ".*o*" >non/git/.gitignore &&
 +	(
 +		GIT_CEILING_DIRECTORIES="$(pwd)/non/git" &&
 +		export GIT_CEILING_DIRECTORIES &&
 +		cd non/git &&
 +		test_must_fail git grep o &&
 +		git grep --no-index --exclude-standard o >../actual.full &&
 +		test_cmp ../expect.full ../actual.full &&
 +
 +		{
 +			echo ".gitignore:.*o*"
 +			cat ../expect.full
 +		} >../expect.with.ignored &&
 +		git grep --no-index --no-exclude o >../actual.full &&
 +		test_cmp ../expect.with.ignored ../actual.full
  	)
  '
  
@@@ -642,10 -574,6 +647,10 @@@ test_expect_success 'inside git reposit
  	{
  		echo file1:hello &&
  		echo sub/file2:world
 +	} >is/expect.unignored &&
 +	{
 +		echo ".gitignore:.*o*" &&
 +		cat is/expect.unignored
  	} >is/expect.full &&
  	: >is/expect.empty &&
  	echo file2:world >is/expect.sub &&
@@@ -654,24 -582,12 +659,24 @@@
  		git init &&
  		test_must_fail git grep o >../actual.full &&
  		test_cmp ../expect.empty ../actual.full &&
 +
 +		git grep --untracked o >../actual.unignored &&
 +		test_cmp ../expect.unignored ../actual.unignored &&
 +
  		git grep --no-index o >../actual.full &&
  		test_cmp ../expect.full ../actual.full &&
 +
 +		git grep --no-index --exclude-standard o >../actual.unignored &&
 +		test_cmp ../expect.unignored ../actual.unignored &&
 +
  		cd sub &&
  		test_must_fail git grep o >../../actual.sub &&
  		test_cmp ../../expect.empty ../../actual.sub &&
 +
  		git grep --no-index o >../../actual.sub &&
 +		test_cmp ../../expect.sub ../../actual.sub &&
 +
 +		git grep --untracked o >../../actual.sub &&
  		test_cmp ../../expect.sub ../../actual.sub
  	)
  '
@@@ -805,115 -721,4 +810,115 @@@ test_expect_success LIBPCRE 'grep -G -
  	test_cmp expected actual
  '
  
 +test_config() {
 +	git config "$1" "$2" &&
 +	test_when_finished "git config --unset $1"
 +}
 +
 +cat >expected <<EOF
 +hello.c<RED>:<RESET>int main(int argc, const char **argv)
 +hello.c<RED>-<RESET>{
 +<RED>--<RESET>
 +hello.c<RED>:<RESET>	/* char ?? */
 +hello.c<RED>-<RESET>}
 +<RED>--<RESET>
 +hello_world<RED>:<RESET>Hello_world
 +hello_world<RED>-<RESET>HeLLo_world
 +EOF
 +
 +test_expect_success 'grep --color, separator' '
 +	test_config color.grep.context		normal &&
 +	test_config color.grep.filename		normal &&
 +	test_config color.grep.function		normal &&
 +	test_config color.grep.linenumber	normal &&
 +	test_config color.grep.match		normal &&
 +	test_config color.grep.selected		normal &&
 +	test_config color.grep.separator	red &&
 +
 +	git grep --color=always -A1 -e char -e lo_w hello.c hello_world |
 +	test_decode_color >actual &&
 +	test_cmp expected actual
 +'
 +
 +cat >expected <<EOF
 +hello.c:int main(int argc, const char **argv)
 +hello.c:	/* char ?? */
 +
 +hello_world:Hello_world
 +EOF
 +
 +test_expect_success 'grep --break' '
 +	git grep --break -e char -e lo_w hello.c hello_world >actual &&
 +	test_cmp expected actual
 +'
 +
 +cat >expected <<EOF
 +hello.c:int main(int argc, const char **argv)
 +hello.c-{
 +--
 +hello.c:	/* char ?? */
 +hello.c-}
 +
 +hello_world:Hello_world
 +hello_world-HeLLo_world
 +EOF
 +
 +test_expect_success 'grep --break with context' '
 +	git grep --break -A1 -e char -e lo_w hello.c hello_world >actual &&
 +	test_cmp expected actual
 +'
 +
 +cat >expected <<EOF
 +hello.c
 +int main(int argc, const char **argv)
 +	/* char ?? */
 +hello_world
 +Hello_world
 +EOF
 +
 +test_expect_success 'grep --heading' '
 +	git grep --heading -e char -e lo_w hello.c hello_world >actual &&
 +	test_cmp expected actual
 +'
 +
 +cat >expected <<EOF
 +<BOLD;GREEN>hello.c<RESET>
 +2:int main(int argc, const <BLACK;BYELLOW>char<RESET> **argv)
 +6:	/* <BLACK;BYELLOW>char<RESET> ?? */
 +
 +<BOLD;GREEN>hello_world<RESET>
 +3:Hel<BLACK;BYELLOW>lo_w<RESET>orld
 +EOF
 +
 +test_expect_success 'mimic ack-grep --group' '
 +	test_config color.grep.context		normal &&
 +	test_config color.grep.filename		"bold green" &&
 +	test_config color.grep.function		normal &&
 +	test_config color.grep.linenumber	normal &&
 +	test_config color.grep.match		"black yellow" &&
 +	test_config color.grep.selected		normal &&
 +	test_config color.grep.separator	normal &&
 +
 +	git grep --break --heading -n --color \
 +		-e char -e lo_w hello.c hello_world |
 +	test_decode_color >actual &&
 +	test_cmp expected actual
 +'
 +
 +cat >expected <<EOF
 +space: line with leading space1
 +space: line with leading space2
 +space: line with leading space3
 +EOF
 +
 +test_expect_success LIBPCRE 'grep -E "^ "' '
 +	git grep -E "^ " space >actual &&
 +	test_cmp expected actual
 +'
 +
 +test_expect_success LIBPCRE 'grep -P "^ "' '
 +	git grep -P "^ " space >actual &&
 +	test_cmp expected actual
 +'
 +
  test_done