Andrew's git / gitweb.git / grep.c
?
grep.con commit config: don't implicitly use gitdir or commondir (dc8441f)
   1#include "cache.h"
   2#include "config.h"
   3#include "grep.h"
   4#include "userdiff.h"
   5#include "xdiff-interface.h"
   6#include "diff.h"
   7#include "diffcore.h"
   8#include "commit.h"
   9#include "quote.h"
  10
  11static int grep_source_load(struct grep_source *gs);
  12static int grep_source_is_binary(struct grep_source *gs);
  13
  14static struct grep_opt grep_defaults;
  15
  16static void std_output(struct grep_opt *opt, const void *buf, size_t size)
  17{
  18        fwrite(buf, size, 1, stdout);
  19}
  20
  21/*
  22 * Initialize the grep_defaults template with hardcoded defaults.
  23 * We could let the compiler do this, but without C99 initializers
  24 * the code gets unwieldy and unreadable, so...
  25 */
  26void init_grep_defaults(void)
  27{
  28        struct grep_opt *opt = &grep_defaults;
  29        static int run_once;
  30
  31        if (run_once)
  32                return;
  33        run_once++;
  34
  35        memset(opt, 0, sizeof(*opt));
  36        opt->relative = 1;
  37        opt->pathname = 1;
  38        opt->regflags = REG_NEWLINE;
  39        opt->max_depth = -1;
  40        opt->pattern_type_option = GREP_PATTERN_TYPE_UNSPECIFIED;
  41        opt->extended_regexp_option = 0;
  42        color_set(opt->color_context, "");
  43        color_set(opt->color_filename, "");
  44        color_set(opt->color_function, "");
  45        color_set(opt->color_lineno, "");
  46        color_set(opt->color_match_context, GIT_COLOR_BOLD_RED);
  47        color_set(opt->color_match_selected, GIT_COLOR_BOLD_RED);
  48        color_set(opt->color_selected, "");
  49        color_set(opt->color_sep, GIT_COLOR_CYAN);
  50        opt->color = -1;
  51        opt->output = std_output;
  52}
  53
  54static int parse_pattern_type_arg(const char *opt, const char *arg)
  55{
  56        if (!strcmp(arg, "default"))
  57                return GREP_PATTERN_TYPE_UNSPECIFIED;
  58        else if (!strcmp(arg, "basic"))
  59                return GREP_PATTERN_TYPE_BRE;
  60        else if (!strcmp(arg, "extended"))
  61                return GREP_PATTERN_TYPE_ERE;
  62        else if (!strcmp(arg, "fixed"))
  63                return GREP_PATTERN_TYPE_FIXED;
  64        else if (!strcmp(arg, "perl"))
  65                return GREP_PATTERN_TYPE_PCRE;
  66        die("bad %s argument: %s", opt, arg);
  67}
  68
  69/*
  70 * Read the configuration file once and store it in
  71 * the grep_defaults template.
  72 */
  73int grep_config(const char *var, const char *value, void *cb)
  74{
  75        struct grep_opt *opt = &grep_defaults;
  76        char *color = NULL;
  77
  78        if (userdiff_config(var, value) < 0)
  79                return -1;
  80
  81        if (!strcmp(var, "grep.extendedregexp")) {
  82                if (git_config_bool(var, value))
  83                        opt->extended_regexp_option = 1;
  84                else
  85                        opt->extended_regexp_option = 0;
  86                return 0;
  87        }
  88
  89        if (!strcmp(var, "grep.patterntype")) {
  90                opt->pattern_type_option = parse_pattern_type_arg(var, value);
  91                return 0;
  92        }
  93
  94        if (!strcmp(var, "grep.linenumber")) {
  95                opt->linenum = git_config_bool(var, value);
  96                return 0;
  97        }
  98
  99        if (!strcmp(var, "grep.fullname")) {
 100                opt->relative = !git_config_bool(var, value);
 101                return 0;
 102        }
 103
 104        if (!strcmp(var, "color.grep"))
 105                opt->color = git_config_colorbool(var, value);
 106        else if (!strcmp(var, "color.grep.context"))
 107                color = opt->color_context;
 108        else if (!strcmp(var, "color.grep.filename"))
 109                color = opt->color_filename;
 110        else if (!strcmp(var, "color.grep.function"))
 111                color = opt->color_function;
 112        else if (!strcmp(var, "color.grep.linenumber"))
 113                color = opt->color_lineno;
 114        else if (!strcmp(var, "color.grep.matchcontext"))
 115                color = opt->color_match_context;
 116        else if (!strcmp(var, "color.grep.matchselected"))
 117                color = opt->color_match_selected;
 118        else if (!strcmp(var, "color.grep.selected"))
 119                color = opt->color_selected;
 120        else if (!strcmp(var, "color.grep.separator"))
 121                color = opt->color_sep;
 122        else if (!strcmp(var, "color.grep.match")) {
 123                int rc = 0;
 124                if (!value)
 125                        return config_error_nonbool(var);
 126                rc |= color_parse(value, opt->color_match_context);
 127                rc |= color_parse(value, opt->color_match_selected);
 128                return rc;
 129        }
 130
 131        if (color) {
 132                if (!value)
 133                        return config_error_nonbool(var);
 134                return color_parse(value, color);
 135        }
 136        return 0;
 137}
 138
 139/*
 140 * Initialize one instance of grep_opt and copy the
 141 * default values from the template we read the configuration
 142 * information in an earlier call to git_config(grep_config).
 143 */
 144void grep_init(struct grep_opt *opt, const char *prefix)
 145{
 146        struct grep_opt *def = &grep_defaults;
 147
 148        memset(opt, 0, sizeof(*opt));
 149        opt->prefix = prefix;
 150        opt->prefix_length = (prefix && *prefix) ? strlen(prefix) : 0;
 151        opt->pattern_tail = &opt->pattern_list;
 152        opt->header_tail = &opt->header_list;
 153
 154        opt->color = def->color;
 155        opt->extended_regexp_option = def->extended_regexp_option;
 156        opt->pattern_type_option = def->pattern_type_option;
 157        opt->linenum = def->linenum;
 158        opt->max_depth = def->max_depth;
 159        opt->pathname = def->pathname;
 160        opt->regflags = def->regflags;
 161        opt->relative = def->relative;
 162        opt->output = def->output;
 163
 164        color_set(opt->color_context, def->color_context);
 165        color_set(opt->color_filename, def->color_filename);
 166        color_set(opt->color_function, def->color_function);
 167        color_set(opt->color_lineno, def->color_lineno);
 168        color_set(opt->color_match_context, def->color_match_context);
 169        color_set(opt->color_match_selected, def->color_match_selected);
 170        color_set(opt->color_selected, def->color_selected);
 171        color_set(opt->color_sep, def->color_sep);
 172}
 173
 174static void grep_set_pattern_type_option(enum grep_pattern_type pattern_type, struct grep_opt *opt)
 175{
 176        switch (pattern_type) {
 177        case GREP_PATTERN_TYPE_UNSPECIFIED:
 178                /* fall through */
 179
 180        case GREP_PATTERN_TYPE_BRE:
 181                opt->fixed = 0;
 182                opt->pcre = 0;
 183                opt->regflags &= ~REG_EXTENDED;
 184                break;
 185
 186        case GREP_PATTERN_TYPE_ERE:
 187                opt->fixed = 0;
 188                opt->pcre = 0;
 189                opt->regflags |= REG_EXTENDED;
 190                break;
 191
 192        case GREP_PATTERN_TYPE_FIXED:
 193                opt->fixed = 1;
 194                opt->pcre = 0;
 195                opt->regflags &= ~REG_EXTENDED;
 196                break;
 197
 198        case GREP_PATTERN_TYPE_PCRE:
 199                opt->fixed = 0;
 200                opt->pcre = 1;
 201                opt->regflags &= ~REG_EXTENDED;
 202                break;
 203        }
 204}
 205
 206void grep_commit_pattern_type(enum grep_pattern_type pattern_type, struct grep_opt *opt)
 207{
 208        if (pattern_type != GREP_PATTERN_TYPE_UNSPECIFIED)
 209                grep_set_pattern_type_option(pattern_type, opt);
 210        else if (opt->pattern_type_option != GREP_PATTERN_TYPE_UNSPECIFIED)
 211                grep_set_pattern_type_option(opt->pattern_type_option, opt);
 212        else if (opt->extended_regexp_option)
 213                grep_set_pattern_type_option(GREP_PATTERN_TYPE_ERE, opt);
 214}
 215
 216static struct grep_pat *create_grep_pat(const char *pat, size_t patlen,
 217                                        const char *origin, int no,
 218                                        enum grep_pat_token t,
 219                                        enum grep_header_field field)
 220{
 221        struct grep_pat *p = xcalloc(1, sizeof(*p));
 222        p->pattern = xmemdupz(pat, patlen);
 223        p->patternlen = patlen;
 224        p->origin = origin;
 225        p->no = no;
 226        p->token = t;
 227        p->field = field;
 228        return p;
 229}
 230
 231static void do_append_grep_pat(struct grep_pat ***tail, struct grep_pat *p)
 232{
 233        **tail = p;
 234        *tail = &p->next;
 235        p->next = NULL;
 236
 237        switch (p->token) {
 238        case GREP_PATTERN: /* atom */
 239        case GREP_PATTERN_HEAD:
 240        case GREP_PATTERN_BODY:
 241                for (;;) {
 242                        struct grep_pat *new_pat;
 243                        size_t len = 0;
 244                        char *cp = p->pattern + p->patternlen, *nl = NULL;
 245                        while (++len <= p->patternlen) {
 246                                if (*(--cp) == '\n') {
 247                                        nl = cp;
 248                                        break;
 249                                }
 250                        }
 251                        if (!nl)
 252                                break;
 253                        new_pat = create_grep_pat(nl + 1, len - 1, p->origin,
 254                                                  p->no, p->token, p->field);
 255                        new_pat->next = p->next;
 256                        if (!p->next)
 257                                *tail = &new_pat->next;
 258                        p->next = new_pat;
 259                        *nl = '\0';
 260                        p->patternlen -= len;
 261                }
 262                break;
 263        default:
 264                break;
 265        }
 266}
 267
 268void append_header_grep_pattern(struct grep_opt *opt,
 269                                enum grep_header_field field, const char *pat)
 270{
 271        struct grep_pat *p = create_grep_pat(pat, strlen(pat), "header", 0,
 272                                             GREP_PATTERN_HEAD, field);
 273        if (field == GREP_HEADER_REFLOG)
 274                opt->use_reflog_filter = 1;
 275        do_append_grep_pat(&opt->header_tail, p);
 276}
 277
 278void append_grep_pattern(struct grep_opt *opt, const char *pat,
 279                         const char *origin, int no, enum grep_pat_token t)
 280{
 281        append_grep_pat(opt, pat, strlen(pat), origin, no, t);
 282}
 283
 284void append_grep_pat(struct grep_opt *opt, const char *pat, size_t patlen,
 285                     const char *origin, int no, enum grep_pat_token t)
 286{
 287        struct grep_pat *p = create_grep_pat(pat, patlen, origin, no, t, 0);
 288        do_append_grep_pat(&opt->pattern_tail, p);
 289}
 290
 291struct grep_opt *grep_opt_dup(const struct grep_opt *opt)
 292{
 293        struct grep_pat *pat;
 294        struct grep_opt *ret = xmalloc(sizeof(struct grep_opt));
 295        *ret = *opt;
 296
 297        ret->pattern_list = NULL;
 298        ret->pattern_tail = &ret->pattern_list;
 299
 300        for(pat = opt->pattern_list; pat != NULL; pat = pat->next)
 301        {
 302                if(pat->token == GREP_PATTERN_HEAD)
 303                        append_header_grep_pattern(ret, pat->field,
 304                                                   pat->pattern);
 305                else
 306                        append_grep_pat(ret, pat->pattern, pat->patternlen,
 307                                        pat->origin, pat->no, pat->token);
 308        }
 309
 310        return ret;
 311}
 312
 313static NORETURN void compile_regexp_failed(const struct grep_pat *p,
 314                const char *error)
 315{
 316        char where[1024];
 317
 318        if (p->no)
 319                xsnprintf(where, sizeof(where), "In '%s' at %d, ", p->origin, p->no);
 320        else if (p->origin)
 321                xsnprintf(where, sizeof(where), "%s, ", p->origin);
 322        else
 323                where[0] = 0;
 324
 325        die("%s'%s': %s", where, p->pattern, error);
 326}
 327
 328#ifdef USE_LIBPCRE
 329static void compile_pcre_regexp(struct grep_pat *p, const struct grep_opt *opt)
 330{
 331        const char *error;
 332        int erroffset;
 333        int options = PCRE_MULTILINE;
 334
 335        if (opt->ignore_case) {
 336                if (has_non_ascii(p->pattern))
 337                        p->pcre_tables = pcre_maketables();
 338                options |= PCRE_CASELESS;
 339        }
 340        if (is_utf8_locale() && has_non_ascii(p->pattern))
 341                options |= PCRE_UTF8;
 342
 343        p->pcre_regexp = pcre_compile(p->pattern, options, &error, &erroffset,
 344                                      p->pcre_tables);
 345        if (!p->pcre_regexp)
 346                compile_regexp_failed(p, error);
 347
 348        p->pcre_extra_info = pcre_study(p->pcre_regexp, 0, &error);
 349        if (!p->pcre_extra_info && error)
 350                die("%s", error);
 351}
 352
 353static int pcrematch(struct grep_pat *p, const char *line, const char *eol,
 354                regmatch_t *match, int eflags)
 355{
 356        int ovector[30], ret, flags = 0;
 357
 358        if (eflags & REG_NOTBOL)
 359                flags |= PCRE_NOTBOL;
 360
 361        ret = pcre_exec(p->pcre_regexp, p->pcre_extra_info, line, eol - line,
 362                        0, flags, ovector, ARRAY_SIZE(ovector));
 363        if (ret < 0 && ret != PCRE_ERROR_NOMATCH)
 364                die("pcre_exec failed with error code %d", ret);
 365        if (ret > 0) {
 366                ret = 0;
 367                match->rm_so = ovector[0];
 368                match->rm_eo = ovector[1];
 369        }
 370
 371        return ret;
 372}
 373
 374static void free_pcre_regexp(struct grep_pat *p)
 375{
 376        pcre_free(p->pcre_regexp);
 377        pcre_free(p->pcre_extra_info);
 378        pcre_free((void *)p->pcre_tables);
 379}
 380#else /* !USE_LIBPCRE */
 381static void compile_pcre_regexp(struct grep_pat *p, const struct grep_opt *opt)
 382{
 383        die("cannot use Perl-compatible regexes when not compiled with USE_LIBPCRE");
 384}
 385
 386static int pcrematch(struct grep_pat *p, const char *line, const char *eol,
 387                regmatch_t *match, int eflags)
 388{
 389        return 1;
 390}
 391
 392static void free_pcre_regexp(struct grep_pat *p)
 393{
 394}
 395#endif /* !USE_LIBPCRE */
 396
 397static int is_fixed(const char *s, size_t len)
 398{
 399        size_t i;
 400
 401        /* regcomp cannot accept patterns with NULs so we
 402         * consider any pattern containing a NUL fixed.
 403         */
 404        if (memchr(s, 0, len))
 405                return 1;
 406
 407        for (i = 0; i < len; i++) {
 408                if (is_regex_special(s[i]))
 409                        return 0;
 410        }
 411
 412        return 1;
 413}
 414
 415static void compile_fixed_regexp(struct grep_pat *p, struct grep_opt *opt)
 416{
 417        struct strbuf sb = STRBUF_INIT;
 418        int err;
 419        int regflags;
 420
 421        basic_regex_quote_buf(&sb, p->pattern);
 422        regflags = opt->regflags & ~REG_EXTENDED;
 423        if (opt->ignore_case)
 424                regflags |= REG_ICASE;
 425        err = regcomp(&p->regexp, sb.buf, regflags);
 426        if (opt->debug)
 427                fprintf(stderr, "fixed %s\n", sb.buf);
 428        strbuf_release(&sb);
 429        if (err) {
 430                char errbuf[1024];
 431                regerror(err, &p->regexp, errbuf, sizeof(errbuf));
 432                regfree(&p->regexp);
 433                compile_regexp_failed(p, errbuf);
 434        }
 435}
 436
 437static void compile_regexp(struct grep_pat *p, struct grep_opt *opt)
 438{
 439        int icase, ascii_only;
 440        int err;
 441
 442        p->word_regexp = opt->word_regexp;
 443        p->ignore_case = opt->ignore_case;
 444        icase          = opt->regflags & REG_ICASE || p->ignore_case;
 445        ascii_only     = !has_non_ascii(p->pattern);
 446
 447        /*
 448         * Even when -F (fixed) asks us to do a non-regexp search, we
 449         * may not be able to correctly case-fold when -i
 450         * (ignore-case) is asked (in which case, we'll synthesize a
 451         * regexp to match the pattern that matches regexp special
 452         * characters literally, while ignoring case differences).  On
 453         * the other hand, even without -F, if the pattern does not
 454         * have any regexp special characters and there is no need for
 455         * case-folding search, we can internally turn it into a
 456         * simple string match using kws.  p->fixed tells us if we
 457         * want to use kws.
 458         */
 459        if (opt->fixed || is_fixed(p->pattern, p->patternlen))
 460                p->fixed = !icase || ascii_only;
 461        else
 462                p->fixed = 0;
 463
 464        if (p->fixed) {
 465                p->kws = kwsalloc(icase ? tolower_trans_tbl : NULL);
 466                kwsincr(p->kws, p->pattern, p->patternlen);
 467                kwsprep(p->kws);
 468                return;
 469        } else if (opt->fixed) {
 470                /*
 471                 * We come here when the pattern has the non-ascii
 472                 * characters we cannot case-fold, and asked to
 473                 * ignore-case.
 474                 */
 475                compile_fixed_regexp(p, opt);
 476                return;
 477        }
 478
 479        if (opt->pcre) {
 480                compile_pcre_regexp(p, opt);
 481                return;
 482        }
 483
 484        err = regcomp(&p->regexp, p->pattern, opt->regflags);
 485        if (err) {
 486                char errbuf[1024];
 487                regerror(err, &p->regexp, errbuf, 1024);
 488                regfree(&p->regexp);
 489                compile_regexp_failed(p, errbuf);
 490        }
 491}
 492
 493static struct grep_expr *compile_pattern_or(struct grep_pat **);
 494static struct grep_expr *compile_pattern_atom(struct grep_pat **list)
 495{
 496        struct grep_pat *p;
 497        struct grep_expr *x;
 498
 499        p = *list;
 500        if (!p)
 501                return NULL;
 502        switch (p->token) {
 503        case GREP_PATTERN: /* atom */
 504        case GREP_PATTERN_HEAD:
 505        case GREP_PATTERN_BODY:
 506                x = xcalloc(1, sizeof (struct grep_expr));
 507                x->node = GREP_NODE_ATOM;
 508                x->u.atom = p;
 509                *list = p->next;
 510                return x;
 511        case GREP_OPEN_PAREN:
 512                *list = p->next;
 513                x = compile_pattern_or(list);
 514                if (!*list || (*list)->token != GREP_CLOSE_PAREN)
 515                        die("unmatched parenthesis");
 516                *list = (*list)->next;
 517                return x;
 518        default:
 519                return NULL;
 520        }
 521}
 522
 523static struct grep_expr *compile_pattern_not(struct grep_pat **list)
 524{
 525        struct grep_pat *p;
 526        struct grep_expr *x;
 527
 528        p = *list;
 529        if (!p)
 530                return NULL;
 531        switch (p->token) {
 532        case GREP_NOT:
 533                if (!p->next)
 534                        die("--not not followed by pattern expression");
 535                *list = p->next;
 536                x = xcalloc(1, sizeof (struct grep_expr));
 537                x->node = GREP_NODE_NOT;
 538                x->u.unary = compile_pattern_not(list);
 539                if (!x->u.unary)
 540                        die("--not followed by non pattern expression");
 541                return x;
 542        default:
 543                return compile_pattern_atom(list);
 544        }
 545}
 546
 547static struct grep_expr *compile_pattern_and(struct grep_pat **list)
 548{
 549        struct grep_pat *p;
 550        struct grep_expr *x, *y, *z;
 551
 552        x = compile_pattern_not(list);
 553        p = *list;
 554        if (p && p->token == GREP_AND) {
 555                if (!p->next)
 556                        die("--and not followed by pattern expression");
 557                *list = p->next;
 558                y = compile_pattern_and(list);
 559                if (!y)
 560                        die("--and not followed by pattern expression");
 561                z = xcalloc(1, sizeof (struct grep_expr));
 562                z->node = GREP_NODE_AND;
 563                z->u.binary.left = x;
 564                z->u.binary.right = y;
 565                return z;
 566        }
 567        return x;
 568}
 569
 570static struct grep_expr *compile_pattern_or(struct grep_pat **list)
 571{
 572        struct grep_pat *p;
 573        struct grep_expr *x, *y, *z;
 574
 575        x = compile_pattern_and(list);
 576        p = *list;
 577        if (x && p && p->token != GREP_CLOSE_PAREN) {
 578                y = compile_pattern_or(list);
 579                if (!y)
 580                        die("not a pattern expression %s", p->pattern);
 581                z = xcalloc(1, sizeof (struct grep_expr));
 582                z->node = GREP_NODE_OR;
 583                z->u.binary.left = x;
 584                z->u.binary.right = y;
 585                return z;
 586        }
 587        return x;
 588}
 589
 590static struct grep_expr *compile_pattern_expr(struct grep_pat **list)
 591{
 592        return compile_pattern_or(list);
 593}
 594
 595static void indent(int in)
 596{
 597        while (in-- > 0)
 598                fputc(' ', stderr);
 599}
 600
 601static void dump_grep_pat(struct grep_pat *p)
 602{
 603        switch (p->token) {
 604        case GREP_AND: fprintf(stderr, "*and*"); break;
 605        case GREP_OPEN_PAREN: fprintf(stderr, "*(*"); break;
 606        case GREP_CLOSE_PAREN: fprintf(stderr, "*)*"); break;
 607        case GREP_NOT: fprintf(stderr, "*not*"); break;
 608        case GREP_OR: fprintf(stderr, "*or*"); break;
 609
 610        case GREP_PATTERN: fprintf(stderr, "pattern"); break;
 611        case GREP_PATTERN_HEAD: fprintf(stderr, "pattern_head"); break;
 612        case GREP_PATTERN_BODY: fprintf(stderr, "pattern_body"); break;
 613        }
 614
 615        switch (p->token) {
 616        default: break;
 617        case GREP_PATTERN_HEAD:
 618                fprintf(stderr, "<head %d>", p->field); break;
 619        case GREP_PATTERN_BODY:
 620                fprintf(stderr, "<body>"); break;
 621        }
 622        switch (p->token) {
 623        default: break;
 624        case GREP_PATTERN_HEAD:
 625        case GREP_PATTERN_BODY:
 626        case GREP_PATTERN:
 627                fprintf(stderr, "%.*s", (int)p->patternlen, p->pattern);
 628                break;
 629        }
 630        fputc('\n', stderr);
 631}
 632
 633static void dump_grep_expression_1(struct grep_expr *x, int in)
 634{
 635        indent(in);
 636        switch (x->node) {
 637        case GREP_NODE_TRUE:
 638                fprintf(stderr, "true\n");
 639                break;
 640        case GREP_NODE_ATOM:
 641                dump_grep_pat(x->u.atom);
 642                break;
 643        case GREP_NODE_NOT:
 644                fprintf(stderr, "(not\n");
 645                dump_grep_expression_1(x->u.unary, in+1);
 646                indent(in);
 647                fprintf(stderr, ")\n");
 648                break;
 649        case GREP_NODE_AND:
 650                fprintf(stderr, "(and\n");
 651                dump_grep_expression_1(x->u.binary.left, in+1);
 652                dump_grep_expression_1(x->u.binary.right, in+1);
 653                indent(in);
 654                fprintf(stderr, ")\n");
 655                break;
 656        case GREP_NODE_OR:
 657                fprintf(stderr, "(or\n");
 658                dump_grep_expression_1(x->u.binary.left, in+1);
 659                dump_grep_expression_1(x->u.binary.right, in+1);
 660                indent(in);
 661                fprintf(stderr, ")\n");
 662                break;
 663        }
 664}
 665
 666static void dump_grep_expression(struct grep_opt *opt)
 667{
 668        struct grep_expr *x = opt->pattern_expression;
 669
 670        if (opt->all_match)
 671                fprintf(stderr, "[all-match]\n");
 672        dump_grep_expression_1(x, 0);
 673        fflush(NULL);
 674}
 675
 676static struct grep_expr *grep_true_expr(void)
 677{
 678        struct grep_expr *z = xcalloc(1, sizeof(*z));
 679        z->node = GREP_NODE_TRUE;
 680        return z;
 681}
 682
 683static struct grep_expr *grep_or_expr(struct grep_expr *left, struct grep_expr *right)
 684{
 685        struct grep_expr *z = xcalloc(1, sizeof(*z));
 686        z->node = GREP_NODE_OR;
 687        z->u.binary.left = left;
 688        z->u.binary.right = right;
 689        return z;
 690}
 691
 692static struct grep_expr *prep_header_patterns(struct grep_opt *opt)
 693{
 694        struct grep_pat *p;
 695        struct grep_expr *header_expr;
 696        struct grep_expr *(header_group[GREP_HEADER_FIELD_MAX]);
 697        enum grep_header_field fld;
 698
 699        if (!opt->header_list)
 700                return NULL;
 701
 702        for (p = opt->header_list; p; p = p->next) {
 703                if (p->token != GREP_PATTERN_HEAD)
 704                        die("BUG: a non-header pattern in grep header list.");
 705                if (p->field < GREP_HEADER_FIELD_MIN ||
 706                    GREP_HEADER_FIELD_MAX <= p->field)
 707                        die("BUG: unknown header field %d", p->field);
 708                compile_regexp(p, opt);
 709        }
 710
 711        for (fld = 0; fld < GREP_HEADER_FIELD_MAX; fld++)
 712                header_group[fld] = NULL;
 713
 714        for (p = opt->header_list; p; p = p->next) {
 715                struct grep_expr *h;
 716                struct grep_pat *pp = p;
 717
 718                h = compile_pattern_atom(&pp);
 719                if (!h || pp != p->next)
 720                        die("BUG: malformed header expr");
 721                if (!header_group[p->field]) {
 722                        header_group[p->field] = h;
 723                        continue;
 724                }
 725                header_group[p->field] = grep_or_expr(h, header_group[p->field]);
 726        }
 727
 728        header_expr = NULL;
 729
 730        for (fld = 0; fld < GREP_HEADER_FIELD_MAX; fld++) {
 731                if (!header_group[fld])
 732                        continue;
 733                if (!header_expr)
 734                        header_expr = grep_true_expr();
 735                header_expr = grep_or_expr(header_group[fld], header_expr);
 736        }
 737        return header_expr;
 738}
 739
 740static struct grep_expr *grep_splice_or(struct grep_expr *x, struct grep_expr *y)
 741{
 742        struct grep_expr *z = x;
 743
 744        while (x) {
 745                assert(x->node == GREP_NODE_OR);
 746                if (x->u.binary.right &&
 747                    x->u.binary.right->node == GREP_NODE_TRUE) {
 748                        x->u.binary.right = y;
 749                        break;
 750                }
 751                x = x->u.binary.right;
 752        }
 753        return z;
 754}
 755
 756static void compile_grep_patterns_real(struct grep_opt *opt)
 757{
 758        struct grep_pat *p;
 759        struct grep_expr *header_expr = prep_header_patterns(opt);
 760
 761        for (p = opt->pattern_list; p; p = p->next) {
 762                switch (p->token) {
 763                case GREP_PATTERN: /* atom */
 764                case GREP_PATTERN_HEAD:
 765                case GREP_PATTERN_BODY:
 766                        compile_regexp(p, opt);
 767                        break;
 768                default:
 769                        opt->extended = 1;
 770                        break;
 771                }
 772        }
 773
 774        if (opt->all_match || header_expr)
 775                opt->extended = 1;
 776        else if (!opt->extended && !opt->debug)
 777                return;
 778
 779        p = opt->pattern_list;
 780        if (p)
 781                opt->pattern_expression = compile_pattern_expr(&p);
 782        if (p)
 783                die("incomplete pattern expression: %s", p->pattern);
 784
 785        if (!header_expr)
 786                return;
 787
 788        if (!opt->pattern_expression)
 789                opt->pattern_expression = header_expr;
 790        else if (opt->all_match)
 791                opt->pattern_expression = grep_splice_or(header_expr,
 792                                                         opt->pattern_expression);
 793        else
 794                opt->pattern_expression = grep_or_expr(opt->pattern_expression,
 795                                                       header_expr);
 796        opt->all_match = 1;
 797}
 798
 799void compile_grep_patterns(struct grep_opt *opt)
 800{
 801        compile_grep_patterns_real(opt);
 802        if (opt->debug)
 803                dump_grep_expression(opt);
 804}
 805
 806static void free_pattern_expr(struct grep_expr *x)
 807{
 808        switch (x->node) {
 809        case GREP_NODE_TRUE:
 810        case GREP_NODE_ATOM:
 811                break;
 812        case GREP_NODE_NOT:
 813                free_pattern_expr(x->u.unary);
 814                break;
 815        case GREP_NODE_AND:
 816        case GREP_NODE_OR:
 817                free_pattern_expr(x->u.binary.left);
 818                free_pattern_expr(x->u.binary.right);
 819                break;
 820        }
 821        free(x);
 822}
 823
 824void free_grep_patterns(struct grep_opt *opt)
 825{
 826        struct grep_pat *p, *n;
 827
 828        for (p = opt->pattern_list; p; p = n) {
 829                n = p->next;
 830                switch (p->token) {
 831                case GREP_PATTERN: /* atom */
 832                case GREP_PATTERN_HEAD:
 833                case GREP_PATTERN_BODY:
 834                        if (p->kws)
 835                                kwsfree(p->kws);
 836                        else if (p->pcre_regexp)
 837                                free_pcre_regexp(p);
 838                        else
 839                                regfree(&p->regexp);
 840                        free(p->pattern);
 841                        break;
 842                default:
 843                        break;
 844                }
 845                free(p);
 846        }
 847
 848        if (!opt->extended)
 849                return;
 850        free_pattern_expr(opt->pattern_expression);
 851}
 852
 853static char *end_of_line(char *cp, unsigned long *left)
 854{
 855        unsigned long l = *left;
 856        while (l && *cp != '\n') {
 857                l--;
 858                cp++;
 859        }
 860        *left = l;
 861        return cp;
 862}
 863
 864static int word_char(char ch)
 865{
 866        return isalnum(ch) || ch == '_';
 867}
 868
 869static void output_color(struct grep_opt *opt, const void *data, size_t size,
 870                         const char *color)
 871{
 872        if (want_color(opt->color) && color && color[0]) {
 873                opt->output(opt, color, strlen(color));
 874                opt->output(opt, data, size);
 875                opt->output(opt, GIT_COLOR_RESET, strlen(GIT_COLOR_RESET));
 876        } else
 877                opt->output(opt, data, size);
 878}
 879
 880static void output_sep(struct grep_opt *opt, char sign)
 881{
 882        if (opt->null_following_name)
 883                opt->output(opt, "\0", 1);
 884        else
 885                output_color(opt, &sign, 1, opt->color_sep);
 886}
 887
 888static void show_name(struct grep_opt *opt, const char *name)
 889{
 890        output_color(opt, name, strlen(name), opt->color_filename);
 891        opt->output(opt, opt->null_following_name ? "\0" : "\n", 1);
 892}
 893
 894static int fixmatch(struct grep_pat *p, char *line, char *eol,
 895                    regmatch_t *match)
 896{
 897        struct kwsmatch kwsm;
 898        size_t offset = kwsexec(p->kws, line, eol - line, &kwsm);
 899        if (offset == -1) {
 900                match->rm_so = match->rm_eo = -1;
 901                return REG_NOMATCH;
 902        } else {
 903                match->rm_so = offset;
 904                match->rm_eo = match->rm_so + kwsm.size[0];
 905                return 0;
 906        }
 907}
 908
 909static int patmatch(struct grep_pat *p, char *line, char *eol,
 910                    regmatch_t *match, int eflags)
 911{
 912        int hit;
 913
 914        if (p->fixed)
 915                hit = !fixmatch(p, line, eol, match);
 916        else if (p->pcre_regexp)
 917                hit = !pcrematch(p, line, eol, match, eflags);
 918        else
 919                hit = !regexec_buf(&p->regexp, line, eol - line, 1, match,
 920                                   eflags);
 921
 922        return hit;
 923}
 924
 925static int strip_timestamp(char *bol, char **eol_p)
 926{
 927        char *eol = *eol_p;
 928        int ch;
 929
 930        while (bol < --eol) {
 931                if (*eol != '>')
 932                        continue;
 933                *eol_p = ++eol;
 934                ch = *eol;
 935                *eol = '\0';
 936                return ch;
 937        }
 938        return 0;
 939}
 940
 941static struct {
 942        const char *field;
 943        size_t len;
 944} header_field[] = {
 945        { "author ", 7 },
 946        { "committer ", 10 },
 947        { "reflog ", 7 },
 948};
 949
 950static int match_one_pattern(struct grep_pat *p, char *bol, char *eol,
 951                             enum grep_context ctx,
 952                             regmatch_t *pmatch, int eflags)
 953{
 954        int hit = 0;
 955        int saved_ch = 0;
 956        const char *start = bol;
 957
 958        if ((p->token != GREP_PATTERN) &&
 959            ((p->token == GREP_PATTERN_HEAD) != (ctx == GREP_CONTEXT_HEAD)))
 960                return 0;
 961
 962        if (p->token == GREP_PATTERN_HEAD) {
 963                const char *field;
 964                size_t len;
 965                assert(p->field < ARRAY_SIZE(header_field));
 966                field = header_field[p->field].field;
 967                len = header_field[p->field].len;
 968                if (strncmp(bol, field, len))
 969                        return 0;
 970                bol += len;
 971                switch (p->field) {
 972                case GREP_HEADER_AUTHOR:
 973                case GREP_HEADER_COMMITTER:
 974                        saved_ch = strip_timestamp(bol, &eol);
 975                        break;
 976                default:
 977                        break;
 978                }
 979        }
 980
 981 again:
 982        hit = patmatch(p, bol, eol, pmatch, eflags);
 983
 984        if (hit && p->word_regexp) {
 985                if ((pmatch[0].rm_so < 0) ||
 986                    (eol - bol) < pmatch[0].rm_so ||
 987                    (pmatch[0].rm_eo < 0) ||
 988                    (eol - bol) < pmatch[0].rm_eo)
 989                        die("regexp returned nonsense");
 990
 991                /* Match beginning must be either beginning of the
 992                 * line, or at word boundary (i.e. the last char must
 993                 * not be a word char).  Similarly, match end must be
 994                 * either end of the line, or at word boundary
 995                 * (i.e. the next char must not be a word char).
 996                 */
 997                if ( ((pmatch[0].rm_so == 0) ||
 998                      !word_char(bol[pmatch[0].rm_so-1])) &&
 999                     ((pmatch[0].rm_eo == (eol-bol)) ||
1000                      !word_char(bol[pmatch[0].rm_eo])) )
1001                        ;
1002                else
1003                        hit = 0;
1004
1005                /* Words consist of at least one character. */
1006                if (pmatch->rm_so == pmatch->rm_eo)
1007                        hit = 0;
1008
1009                if (!hit && pmatch[0].rm_so + bol + 1 < eol) {
1010                        /* There could be more than one match on the
1011                         * line, and the first match might not be
1012                         * strict word match.  But later ones could be!
1013                         * Forward to the next possible start, i.e. the
1014                         * next position following a non-word char.
1015                         */
1016                        bol = pmatch[0].rm_so + bol + 1;
1017                        while (word_char(bol[-1]) && bol < eol)
1018                                bol++;
1019                        eflags |= REG_NOTBOL;
1020                        if (bol < eol)
1021                                goto again;
1022                }
1023        }
1024        if (p->token == GREP_PATTERN_HEAD && saved_ch)
1025                *eol = saved_ch;
1026        if (hit) {
1027                pmatch[0].rm_so += bol - start;
1028                pmatch[0].rm_eo += bol - start;
1029        }
1030        return hit;
1031}
1032
1033static int match_expr_eval(struct grep_expr *x, char *bol, char *eol,
1034                           enum grep_context ctx, int collect_hits)
1035{
1036        int h = 0;
1037        regmatch_t match;
1038
1039        if (!x)
1040                die("Not a valid grep expression");
1041        switch (x->node) {
1042        case GREP_NODE_TRUE:
1043                h = 1;
1044                break;
1045        case GREP_NODE_ATOM:
1046                h = match_one_pattern(x->u.atom, bol, eol, ctx, &match, 0);
1047                break;
1048        case GREP_NODE_NOT:
1049                h = !match_expr_eval(x->u.unary, bol, eol, ctx, 0);
1050                break;
1051        case GREP_NODE_AND:
1052                if (!match_expr_eval(x->u.binary.left, bol, eol, ctx, 0))
1053                        return 0;
1054                h = match_expr_eval(x->u.binary.right, bol, eol, ctx, 0);
1055                break;
1056        case GREP_NODE_OR:
1057                if (!collect_hits)
1058                        return (match_expr_eval(x->u.binary.left,
1059                                                bol, eol, ctx, 0) ||
1060                                match_expr_eval(x->u.binary.right,
1061                                                bol, eol, ctx, 0));
1062                h = match_expr_eval(x->u.binary.left, bol, eol, ctx, 0);
1063                x->u.binary.left->hit |= h;
1064                h |= match_expr_eval(x->u.binary.right, bol, eol, ctx, 1);
1065                break;
1066        default:
1067                die("Unexpected node type (internal error) %d", x->node);
1068        }
1069        if (collect_hits)
1070                x->hit |= h;
1071        return h;
1072}
1073
1074static int match_expr(struct grep_opt *opt, char *bol, char *eol,
1075                      enum grep_context ctx, int collect_hits)
1076{
1077        struct grep_expr *x = opt->pattern_expression;
1078        return match_expr_eval(x, bol, eol, ctx, collect_hits);
1079}
1080
1081static int match_line(struct grep_opt *opt, char *bol, char *eol,
1082                      enum grep_context ctx, int collect_hits)
1083{
1084        struct grep_pat *p;
1085        regmatch_t match;
1086
1087        if (opt->extended)
1088                return match_expr(opt, bol, eol, ctx, collect_hits);
1089
1090        /* we do not call with collect_hits without being extended */
1091        for (p = opt->pattern_list; p; p = p->next) {
1092                if (match_one_pattern(p, bol, eol, ctx, &match, 0))
1093                        return 1;
1094        }
1095        return 0;
1096}
1097
1098static int match_next_pattern(struct grep_pat *p, char *bol, char *eol,
1099                              enum grep_context ctx,
1100                              regmatch_t *pmatch, int eflags)
1101{
1102        regmatch_t match;
1103
1104        if (!match_one_pattern(p, bol, eol, ctx, &match, eflags))
1105                return 0;
1106        if (match.rm_so < 0 || match.rm_eo < 0)
1107                return 0;
1108        if (pmatch->rm_so >= 0 && pmatch->rm_eo >= 0) {
1109                if (match.rm_so > pmatch->rm_so)
1110                        return 1;
1111                if (match.rm_so == pmatch->rm_so && match.rm_eo < pmatch->rm_eo)
1112                        return 1;
1113        }
1114        pmatch->rm_so = match.rm_so;
1115        pmatch->rm_eo = match.rm_eo;
1116        return 1;
1117}
1118
1119static int next_match(struct grep_opt *opt, char *bol, char *eol,
1120                      enum grep_context ctx, regmatch_t *pmatch, int eflags)
1121{
1122        struct grep_pat *p;
1123        int hit = 0;
1124
1125        pmatch->rm_so = pmatch->rm_eo = -1;
1126        if (bol < eol) {
1127                for (p = opt->pattern_list; p; p = p->next) {
1128                        switch (p->token) {
1129                        case GREP_PATTERN: /* atom */
1130                        case GREP_PATTERN_HEAD:
1131                        case GREP_PATTERN_BODY:
1132                                hit |= match_next_pattern(p, bol, eol, ctx,
1133                                                          pmatch, eflags);
1134                                break;
1135                        default:
1136                                break;
1137                        }
1138                }
1139        }
1140        return hit;
1141}
1142
1143static void show_line(struct grep_opt *opt, char *bol, char *eol,
1144                      const char *name, unsigned lno, char sign)
1145{
1146        int rest = eol - bol;
1147        const char *match_color, *line_color = NULL;
1148
1149        if (opt->file_break && opt->last_shown == 0) {
1150                if (opt->show_hunk_mark)
1151                        opt->output(opt, "\n", 1);
1152        } else if (opt->pre_context || opt->post_context || opt->funcbody) {
1153                if (opt->last_shown == 0) {
1154                        if (opt->show_hunk_mark) {
1155                                output_color(opt, "--", 2, opt->color_sep);
1156                                opt->output(opt, "\n", 1);
1157                        }
1158                } else if (lno > opt->last_shown + 1) {
1159                        output_color(opt, "--", 2, opt->color_sep);
1160                        opt->output(opt, "\n", 1);
1161                }
1162        }
1163        if (opt->heading && opt->last_shown == 0) {
1164                output_color(opt, name, strlen(name), opt->color_filename);
1165                opt->output(opt, "\n", 1);
1166        }
1167        opt->last_shown = lno;
1168
1169        if (!opt->heading && opt->pathname) {
1170                output_color(opt, name, strlen(name), opt->color_filename);
1171                output_sep(opt, sign);
1172        }
1173        if (opt->linenum) {
1174                char buf[32];
1175                xsnprintf(buf, sizeof(buf), "%d", lno);
1176                output_color(opt, buf, strlen(buf), opt->color_lineno);
1177                output_sep(opt, sign);
1178        }
1179        if (opt->color) {
1180                regmatch_t match;
1181                enum grep_context ctx = GREP_CONTEXT_BODY;
1182                int ch = *eol;
1183                int eflags = 0;
1184
1185                if (sign == ':')
1186                        match_color = opt->color_match_selected;
1187                else
1188                        match_color = opt->color_match_context;
1189                if (sign == ':')
1190                        line_color = opt->color_selected;
1191                else if (sign == '-')
1192                        line_color = opt->color_context;
1193                else if (sign == '=')
1194                        line_color = opt->color_function;
1195                *eol = '\0';
1196                while (next_match(opt, bol, eol, ctx, &match, eflags)) {
1197                        if (match.rm_so == match.rm_eo)
1198                                break;
1199
1200                        output_color(opt, bol, match.rm_so, line_color);
1201                        output_color(opt, bol + match.rm_so,
1202                                     match.rm_eo - match.rm_so, match_color);
1203                        bol += match.rm_eo;
1204                        rest -= match.rm_eo;
1205                        eflags = REG_NOTBOL;
1206                }
1207                *eol = ch;
1208        }
1209        output_color(opt, bol, rest, line_color);
1210        opt->output(opt, "\n", 1);
1211}
1212
1213#ifndef NO_PTHREADS
1214int grep_use_locks;
1215
1216/*
1217 * This lock protects access to the gitattributes machinery, which is
1218 * not thread-safe.
1219 */
1220pthread_mutex_t grep_attr_mutex;
1221
1222static inline void grep_attr_lock(void)
1223{
1224        if (grep_use_locks)
1225                pthread_mutex_lock(&grep_attr_mutex);
1226}
1227
1228static inline void grep_attr_unlock(void)
1229{
1230        if (grep_use_locks)
1231                pthread_mutex_unlock(&grep_attr_mutex);
1232}
1233
1234/*
1235 * Same as git_attr_mutex, but protecting the thread-unsafe object db access.
1236 */
1237pthread_mutex_t grep_read_mutex;
1238
1239#else
1240#define grep_attr_lock()
1241#define grep_attr_unlock()
1242#endif
1243
1244static int match_funcname(struct grep_opt *opt, struct grep_source *gs, char *bol, char *eol)
1245{
1246        xdemitconf_t *xecfg = opt->priv;
1247        if (xecfg && !xecfg->find_func) {
1248                grep_source_load_driver(gs);
1249                if (gs->driver->funcname.pattern) {
1250                        const struct userdiff_funcname *pe = &gs->driver->funcname;
1251                        xdiff_set_find_func(xecfg, pe->pattern, pe->cflags);
1252                } else {
1253                        xecfg = opt->priv = NULL;
1254                }
1255        }
1256
1257        if (xecfg) {
1258                char buf[1];
1259                return xecfg->find_func(bol, eol - bol, buf, 1,
1260                                        xecfg->find_func_priv) >= 0;
1261        }
1262
1263        if (bol == eol)
1264                return 0;
1265        if (isalpha(*bol) || *bol == '_' || *bol == '$')
1266                return 1;
1267        return 0;
1268}
1269
1270static void show_funcname_line(struct grep_opt *opt, struct grep_source *gs,
1271                               char *bol, unsigned lno)
1272{
1273        while (bol > gs->buf) {
1274                char *eol = --bol;
1275
1276                while (bol > gs->buf && bol[-1] != '\n')
1277                        bol--;
1278                lno--;
1279
1280                if (lno <= opt->last_shown)
1281                        break;
1282
1283                if (match_funcname(opt, gs, bol, eol)) {
1284                        show_line(opt, bol, eol, gs->name, lno, '=');
1285                        break;
1286                }
1287        }
1288}
1289
1290static void show_pre_context(struct grep_opt *opt, struct grep_source *gs,
1291                             char *bol, char *end, unsigned lno)
1292{
1293        unsigned cur = lno, from = 1, funcname_lno = 0;
1294        int funcname_needed = !!opt->funcname;
1295
1296        if (opt->funcbody && !match_funcname(opt, gs, bol, end))
1297                funcname_needed = 2;
1298
1299        if (opt->pre_context < lno)
1300                from = lno - opt->pre_context;
1301        if (from <= opt->last_shown)
1302                from = opt->last_shown + 1;
1303
1304        /* Rewind. */
1305        while (bol > gs->buf &&
1306               cur > (funcname_needed == 2 ? opt->last_shown + 1 : from)) {
1307                char *eol = --bol;
1308
1309                while (bol > gs->buf && bol[-1] != '\n')
1310                        bol--;
1311                cur--;
1312                if (funcname_needed && match_funcname(opt, gs, bol, eol)) {
1313                        funcname_lno = cur;
1314                        funcname_needed = 0;
1315                }
1316        }
1317
1318        /* We need to look even further back to find a function signature. */
1319        if (opt->funcname && funcname_needed)
1320                show_funcname_line(opt, gs, bol, cur);
1321
1322        /* Back forward. */
1323        while (cur < lno) {
1324                char *eol = bol, sign = (cur == funcname_lno) ? '=' : '-';
1325
1326                while (*eol != '\n')
1327                        eol++;
1328                show_line(opt, bol, eol, gs->name, cur, sign);
1329                bol = eol + 1;
1330                cur++;
1331        }
1332}
1333
1334static int should_lookahead(struct grep_opt *opt)
1335{
1336        struct grep_pat *p;
1337
1338        if (opt->extended)
1339                return 0; /* punt for too complex stuff */
1340        if (opt->invert)
1341                return 0;
1342        for (p = opt->pattern_list; p; p = p->next) {
1343                if (p->token != GREP_PATTERN)
1344                        return 0; /* punt for "header only" and stuff */
1345        }
1346        return 1;
1347}
1348
1349static int look_ahead(struct grep_opt *opt,
1350                      unsigned long *left_p,
1351                      unsigned *lno_p,
1352                      char **bol_p)
1353{
1354        unsigned lno = *lno_p;
1355        char *bol = *bol_p;
1356        struct grep_pat *p;
1357        char *sp, *last_bol;
1358        regoff_t earliest = -1;
1359
1360        for (p = opt->pattern_list; p; p = p->next) {
1361                int hit;
1362                regmatch_t m;
1363
1364                hit = patmatch(p, bol, bol + *left_p, &m, 0);
1365                if (!hit || m.rm_so < 0 || m.rm_eo < 0)
1366                        continue;
1367                if (earliest < 0 || m.rm_so < earliest)
1368                        earliest = m.rm_so;
1369        }
1370
1371        if (earliest < 0) {
1372                *bol_p = bol + *left_p;
1373                *left_p = 0;
1374                return 1;
1375        }
1376        for (sp = bol + earliest; bol < sp && sp[-1] != '\n'; sp--)
1377                ; /* find the beginning of the line */
1378        last_bol = sp;
1379
1380        for (sp = bol; sp < last_bol; sp++) {
1381                if (*sp == '\n')
1382                        lno++;
1383        }
1384        *left_p -= last_bol - bol;
1385        *bol_p = last_bol;
1386        *lno_p = lno;
1387        return 0;
1388}
1389
1390static int fill_textconv_grep(struct userdiff_driver *driver,
1391                              struct grep_source *gs)
1392{
1393        struct diff_filespec *df;
1394        char *buf;
1395        size_t size;
1396
1397        if (!driver || !driver->textconv)
1398                return grep_source_load(gs);
1399
1400        /*
1401         * The textconv interface is intimately tied to diff_filespecs, so we
1402         * have to pretend to be one. If we could unify the grep_source
1403         * and diff_filespec structs, this mess could just go away.
1404         */
1405        df = alloc_filespec(gs->path);
1406        switch (gs->type) {
1407        case GREP_SOURCE_SHA1:
1408                fill_filespec(df, gs->identifier, 1, 0100644);
1409                break;
1410        case GREP_SOURCE_FILE:
1411                fill_filespec(df, null_sha1, 0, 0100644);
1412                break;
1413        default:
1414                die("BUG: attempt to textconv something without a path?");
1415        }
1416
1417        /*
1418         * fill_textconv is not remotely thread-safe; it may load objects
1419         * behind the scenes, and it modifies the global diff tempfile
1420         * structure.
1421         */
1422        grep_read_lock();
1423        size = fill_textconv(driver, df, &buf);
1424        grep_read_unlock();
1425        free_filespec(df);
1426
1427        /*
1428         * The normal fill_textconv usage by the diff machinery would just keep
1429         * the textconv'd buf separate from the diff_filespec. But much of the
1430         * grep code passes around a grep_source and assumes that its "buf"
1431         * pointer is the beginning of the thing we are searching. So let's
1432         * install our textconv'd version into the grep_source, taking care not
1433         * to leak any existing buffer.
1434         */
1435        grep_source_clear_data(gs);
1436        gs->buf = buf;
1437        gs->size = size;
1438
1439        return 0;
1440}
1441
1442static int is_empty_line(const char *bol, const char *eol)
1443{
1444        while (bol < eol && isspace(*bol))
1445                bol++;
1446        return bol == eol;
1447}
1448
1449static int grep_source_1(struct grep_opt *opt, struct grep_source *gs, int collect_hits)
1450{
1451        char *bol;
1452        char *peek_bol = NULL;
1453        unsigned long left;
1454        unsigned lno = 1;
1455        unsigned last_hit = 0;
1456        int binary_match_only = 0;
1457        unsigned count = 0;
1458        int try_lookahead = 0;
1459        int show_function = 0;
1460        struct userdiff_driver *textconv = NULL;
1461        enum grep_context ctx = GREP_CONTEXT_HEAD;
1462        xdemitconf_t xecfg;
1463
1464        if (!opt->output)
1465                opt->output = std_output;
1466
1467        if (opt->pre_context || opt->post_context || opt->file_break ||
1468            opt->funcbody) {
1469                /* Show hunk marks, except for the first file. */
1470                if (opt->last_shown)
1471                        opt->show_hunk_mark = 1;
1472                /*
1473                 * If we're using threads then we can't easily identify
1474                 * the first file.  Always put hunk marks in that case
1475                 * and skip the very first one later in work_done().
1476                 */
1477                if (opt->output != std_output)
1478                        opt->show_hunk_mark = 1;
1479        }
1480        opt->last_shown = 0;
1481
1482        if (opt->allow_textconv) {
1483                grep_source_load_driver(gs);
1484                /*
1485                 * We might set up the shared textconv cache data here, which
1486                 * is not thread-safe.
1487                 */
1488                grep_attr_lock();
1489                textconv = userdiff_get_textconv(gs->driver);
1490                grep_attr_unlock();
1491        }
1492
1493        /*
1494         * We know the result of a textconv is text, so we only have to care
1495         * about binary handling if we are not using it.
1496         */
1497        if (!textconv) {
1498                switch (opt->binary) {
1499                case GREP_BINARY_DEFAULT:
1500                        if (grep_source_is_binary(gs))
1501                                binary_match_only = 1;
1502                        break;
1503                case GREP_BINARY_NOMATCH:
1504                        if (grep_source_is_binary(gs))
1505                                return 0; /* Assume unmatch */
1506                        break;
1507                case GREP_BINARY_TEXT:
1508                        break;
1509                default:
1510                        die("BUG: unknown binary handling mode");
1511                }
1512        }
1513
1514        memset(&xecfg, 0, sizeof(xecfg));
1515        opt->priv = &xecfg;
1516
1517        try_lookahead = should_lookahead(opt);
1518
1519        if (fill_textconv_grep(textconv, gs) < 0)
1520                return 0;
1521
1522        bol = gs->buf;
1523        left = gs->size;
1524        while (left) {
1525                char *eol, ch;
1526                int hit;
1527
1528                /*
1529                 * look_ahead() skips quickly to the line that possibly
1530                 * has the next hit; don't call it if we need to do
1531                 * something more than just skipping the current line
1532                 * in response to an unmatch for the current line.  E.g.
1533                 * inside a post-context window, we will show the current
1534                 * line as a context around the previous hit when it
1535                 * doesn't hit.
1536                 */
1537                if (try_lookahead
1538                    && !(last_hit
1539                         && (show_function ||
1540                             lno <= last_hit + opt->post_context))
1541                    && look_ahead(opt, &left, &lno, &bol))
1542                        break;
1543                eol = end_of_line(bol, &left);
1544                ch = *eol;
1545                *eol = 0;
1546
1547                if ((ctx == GREP_CONTEXT_HEAD) && (eol == bol))
1548                        ctx = GREP_CONTEXT_BODY;
1549
1550                hit = match_line(opt, bol, eol, ctx, collect_hits);
1551                *eol = ch;
1552
1553                if (collect_hits)
1554                        goto next_line;
1555
1556                /* "grep -v -e foo -e bla" should list lines
1557                 * that do not have either, so inversion should
1558                 * be done outside.
1559                 */
1560                if (opt->invert)
1561                        hit = !hit;
1562                if (opt->unmatch_name_only) {
1563                        if (hit)
1564                                return 0;
1565                        goto next_line;
1566                }
1567                if (hit) {
1568                        count++;
1569                        if (opt->status_only)
1570                                return 1;
1571                        if (opt->name_only) {
1572                                show_name(opt, gs->name);
1573                                return 1;
1574                        }
1575                        if (opt->count)
1576                                goto next_line;
1577                        if (binary_match_only) {
1578                                opt->output(opt, "Binary file ", 12);
1579                                output_color(opt, gs->name, strlen(gs->name),
1580                                             opt->color_filename);
1581                                opt->output(opt, " matches\n", 9);
1582                                return 1;
1583                        }
1584                        /* Hit at this line.  If we haven't shown the
1585                         * pre-context lines, we would need to show them.
1586                         */
1587                        if (opt->pre_context || opt->funcbody)
1588                                show_pre_context(opt, gs, bol, eol, lno);
1589                        else if (opt->funcname)
1590                                show_funcname_line(opt, gs, bol, lno);
1591                        show_line(opt, bol, eol, gs->name, lno, ':');
1592                        last_hit = lno;
1593                        if (opt->funcbody)
1594                                show_function = 1;
1595                        goto next_line;
1596                }
1597                if (show_function && (!peek_bol || peek_bol < bol)) {
1598                        unsigned long peek_left = left;
1599                        char *peek_eol = eol;
1600
1601                        /*
1602                         * Trailing empty lines are not interesting.
1603                         * Peek past them to see if they belong to the
1604                         * body of the current function.
1605                         */
1606                        peek_bol = bol;
1607                        while (is_empty_line(peek_bol, peek_eol)) {
1608                                peek_bol = peek_eol + 1;
1609                                peek_eol = end_of_line(peek_bol, &peek_left);
1610                        }
1611
1612                        if (match_funcname(opt, gs, peek_bol, peek_eol))
1613                                show_function = 0;
1614                }
1615                if (show_function ||
1616                    (last_hit && lno <= last_hit + opt->post_context)) {
1617                        /* If the last hit is within the post context,
1618                         * we need to show this line.
1619                         */
1620                        show_line(opt, bol, eol, gs->name, lno, '-');
1621                }
1622
1623        next_line:
1624                bol = eol + 1;
1625                if (!left)
1626                        break;
1627                left--;
1628                lno++;
1629        }
1630
1631        if (collect_hits)
1632                return 0;
1633
1634        if (opt->status_only)
1635                return 0;
1636        if (opt->unmatch_name_only) {
1637                /* We did not see any hit, so we want to show this */
1638                show_name(opt, gs->name);
1639                return 1;
1640        }
1641
1642        xdiff_clear_find_func(&xecfg);
1643        opt->priv = NULL;
1644
1645        /* NEEDSWORK:
1646         * The real "grep -c foo *.c" gives many "bar.c:0" lines,
1647         * which feels mostly useless but sometimes useful.  Maybe
1648         * make it another option?  For now suppress them.
1649         */
1650        if (opt->count && count) {
1651                char buf[32];
1652                if (opt->pathname) {
1653                        output_color(opt, gs->name, strlen(gs->name),
1654                                     opt->color_filename);
1655                        output_sep(opt, ':');
1656                }
1657                xsnprintf(buf, sizeof(buf), "%u\n", count);
1658                opt->output(opt, buf, strlen(buf));
1659                return 1;
1660        }
1661        return !!last_hit;
1662}
1663
1664static void clr_hit_marker(struct grep_expr *x)
1665{
1666        /* All-hit markers are meaningful only at the very top level
1667         * OR node.
1668         */
1669        while (1) {
1670                x->hit = 0;
1671                if (x->node != GREP_NODE_OR)
1672                        return;
1673                x->u.binary.left->hit = 0;
1674                x = x->u.binary.right;
1675        }
1676}
1677
1678static int chk_hit_marker(struct grep_expr *x)
1679{
1680        /* Top level nodes have hit markers.  See if they all are hits */
1681        while (1) {
1682                if (x->node != GREP_NODE_OR)
1683                        return x->hit;
1684                if (!x->u.binary.left->hit)
1685                        return 0;
1686                x = x->u.binary.right;
1687        }
1688}
1689
1690int grep_source(struct grep_opt *opt, struct grep_source *gs)
1691{
1692        /*
1693         * we do not have to do the two-pass grep when we do not check
1694         * buffer-wide "all-match".
1695         */
1696        if (!opt->all_match)
1697                return grep_source_1(opt, gs, 0);
1698
1699        /* Otherwise the toplevel "or" terms hit a bit differently.
1700         * We first clear hit markers from them.
1701         */
1702        clr_hit_marker(opt->pattern_expression);
1703        grep_source_1(opt, gs, 1);
1704
1705        if (!chk_hit_marker(opt->pattern_expression))
1706                return 0;
1707
1708        return grep_source_1(opt, gs, 0);
1709}
1710
1711int grep_buffer(struct grep_opt *opt, char *buf, unsigned long size)
1712{
1713        struct grep_source gs;
1714        int r;
1715
1716        grep_source_init(&gs, GREP_SOURCE_BUF, NULL, NULL, NULL);
1717        gs.buf = buf;
1718        gs.size = size;
1719
1720        r = grep_source(opt, &gs);
1721
1722        grep_source_clear(&gs);
1723        return r;
1724}
1725
1726void grep_source_init(struct grep_source *gs, enum grep_source_type type,
1727                      const char *name, const char *path,
1728                      const void *identifier)
1729{
1730        gs->type = type;
1731        gs->name = xstrdup_or_null(name);
1732        gs->path = xstrdup_or_null(path);
1733        gs->buf = NULL;
1734        gs->size = 0;
1735        gs->driver = NULL;
1736
1737        switch (type) {
1738        case GREP_SOURCE_FILE:
1739                gs->identifier = xstrdup(identifier);
1740                break;
1741        case GREP_SOURCE_SUBMODULE:
1742                if (!identifier) {
1743                        gs->identifier = NULL;
1744                        break;
1745                }
1746                /*
1747                 * FALL THROUGH
1748                 * If the identifier is non-NULL (in the submodule case) it
1749                 * will be a SHA1 that needs to be copied.
1750                 */
1751        case GREP_SOURCE_SHA1:
1752                gs->identifier = xmalloc(20);
1753                hashcpy(gs->identifier, identifier);
1754                break;
1755        case GREP_SOURCE_BUF:
1756                gs->identifier = NULL;
1757                break;
1758        }
1759}
1760
1761void grep_source_clear(struct grep_source *gs)
1762{
1763        free(gs->name);
1764        gs->name = NULL;
1765        free(gs->path);
1766        gs->path = NULL;
1767        free(gs->identifier);
1768        gs->identifier = NULL;
1769        grep_source_clear_data(gs);
1770}
1771
1772void grep_source_clear_data(struct grep_source *gs)
1773{
1774        switch (gs->type) {
1775        case GREP_SOURCE_FILE:
1776        case GREP_SOURCE_SHA1:
1777        case GREP_SOURCE_SUBMODULE:
1778                free(gs->buf);
1779                gs->buf = NULL;
1780                gs->size = 0;
1781                break;
1782        case GREP_SOURCE_BUF:
1783                /* leave user-provided buf intact */
1784                break;
1785        }
1786}
1787
1788static int grep_source_load_sha1(struct grep_source *gs)
1789{
1790        enum object_type type;
1791
1792        grep_read_lock();
1793        gs->buf = read_sha1_file(gs->identifier, &type, &gs->size);
1794        grep_read_unlock();
1795
1796        if (!gs->buf)
1797                return error(_("'%s': unable to read %s"),
1798                             gs->name,
1799                             sha1_to_hex(gs->identifier));
1800        return 0;
1801}
1802
1803static int grep_source_load_file(struct grep_source *gs)
1804{
1805        const char *filename = gs->identifier;
1806        struct stat st;
1807        char *data;
1808        size_t size;
1809        int i;
1810
1811        if (lstat(filename, &st) < 0) {
1812        err_ret:
1813                if (errno != ENOENT)
1814                        error_errno(_("failed to stat '%s'"), filename);
1815                return -1;
1816        }
1817        if (!S_ISREG(st.st_mode))
1818                return -1;
1819        size = xsize_t(st.st_size);
1820        i = open(filename, O_RDONLY);
1821        if (i < 0)
1822                goto err_ret;
1823        data = xmallocz(size);
1824        if (st.st_size != read_in_full(i, data, size)) {
1825                error_errno(_("'%s': short read"), filename);
1826                close(i);
1827                free(data);
1828                return -1;
1829        }
1830        close(i);
1831
1832        gs->buf = data;
1833        gs->size = size;
1834        return 0;
1835}
1836
1837static int grep_source_load(struct grep_source *gs)
1838{
1839        if (gs->buf)
1840                return 0;
1841
1842        switch (gs->type) {
1843        case GREP_SOURCE_FILE:
1844                return grep_source_load_file(gs);
1845        case GREP_SOURCE_SHA1:
1846                return grep_source_load_sha1(gs);
1847        case GREP_SOURCE_BUF:
1848                return gs->buf ? 0 : -1;
1849        case GREP_SOURCE_SUBMODULE:
1850                break;
1851        }
1852        die("BUG: invalid grep_source type to load");
1853}
1854
1855void grep_source_load_driver(struct grep_source *gs)
1856{
1857        if (gs->driver)
1858                return;
1859
1860        grep_attr_lock();
1861        if (gs->path)
1862                gs->driver = userdiff_find_by_path(gs->path);
1863        if (!gs->driver)
1864                gs->driver = userdiff_find_by_name("default");
1865        grep_attr_unlock();
1866}
1867
1868static int grep_source_is_binary(struct grep_source *gs)
1869{
1870        grep_source_load_driver(gs);
1871        if (gs->driver->binary != -1)
1872                return gs->driver->binary;
1873
1874        if (!grep_source_load(gs))
1875                return buffer_is_binary(gs->buf, gs->size);
1876
1877        return 0;
1878}