convert.con commit Merge branch 'ds/include-exclude' (9755f70)
   1#include "cache.h"
   2#include "config.h"
   3#include "object-store.h"
   4#include "attr.h"
   5#include "run-command.h"
   6#include "quote.h"
   7#include "sigchain.h"
   8#include "pkt-line.h"
   9#include "sub-process.h"
  10#include "utf8.h"
  11#include "ll-merge.h"
  12
  13/*
  14 * convert.c - convert a file when checking it out and checking it in.
  15 *
  16 * This should use the pathname to decide on whether it wants to do some
  17 * more interesting conversions (automatic gzip/unzip, general format
  18 * conversions etc etc), but by default it just does automatic CRLF<->LF
  19 * translation when the "text" attribute or "auto_crlf" option is set.
  20 */
  21
  22/* Stat bits: When BIN is set, the txt bits are unset */
  23#define CONVERT_STAT_BITS_TXT_LF    0x1
  24#define CONVERT_STAT_BITS_TXT_CRLF  0x2
  25#define CONVERT_STAT_BITS_BIN       0x4
  26
  27enum crlf_action {
  28        CRLF_UNDEFINED,
  29        CRLF_BINARY,
  30        CRLF_TEXT,
  31        CRLF_TEXT_INPUT,
  32        CRLF_TEXT_CRLF,
  33        CRLF_AUTO,
  34        CRLF_AUTO_INPUT,
  35        CRLF_AUTO_CRLF
  36};
  37
  38struct text_stat {
  39        /* NUL, CR, LF and CRLF counts */
  40        unsigned nul, lonecr, lonelf, crlf;
  41
  42        /* These are just approximations! */
  43        unsigned printable, nonprintable;
  44};
  45
  46static void gather_stats(const char *buf, unsigned long size, struct text_stat *stats)
  47{
  48        unsigned long i;
  49
  50        memset(stats, 0, sizeof(*stats));
  51
  52        for (i = 0; i < size; i++) {
  53                unsigned char c = buf[i];
  54                if (c == '\r') {
  55                        if (i+1 < size && buf[i+1] == '\n') {
  56                                stats->crlf++;
  57                                i++;
  58                        } else
  59                                stats->lonecr++;
  60                        continue;
  61                }
  62                if (c == '\n') {
  63                        stats->lonelf++;
  64                        continue;
  65                }
  66                if (c == 127)
  67                        /* DEL */
  68                        stats->nonprintable++;
  69                else if (c < 32) {
  70                        switch (c) {
  71                                /* BS, HT, ESC and FF */
  72                        case '\b': case '\t': case '\033': case '\014':
  73                                stats->printable++;
  74                                break;
  75                        case 0:
  76                                stats->nul++;
  77                                /* fall through */
  78                        default:
  79                                stats->nonprintable++;
  80                        }
  81                }
  82                else
  83                        stats->printable++;
  84        }
  85
  86        /* If file ends with EOF then don't count this EOF as non-printable. */
  87        if (size >= 1 && buf[size-1] == '\032')
  88                stats->nonprintable--;
  89}
  90
  91/*
  92 * The same heuristics as diff.c::mmfile_is_binary()
  93 * We treat files with bare CR as binary
  94 */
  95static int convert_is_binary(const struct text_stat *stats)
  96{
  97        if (stats->lonecr)
  98                return 1;
  99        if (stats->nul)
 100                return 1;
 101        if ((stats->printable >> 7) < stats->nonprintable)
 102                return 1;
 103        return 0;
 104}
 105
 106static unsigned int gather_convert_stats(const char *data, unsigned long size)
 107{
 108        struct text_stat stats;
 109        int ret = 0;
 110        if (!data || !size)
 111                return 0;
 112        gather_stats(data, size, &stats);
 113        if (convert_is_binary(&stats))
 114                ret |= CONVERT_STAT_BITS_BIN;
 115        if (stats.crlf)
 116                ret |= CONVERT_STAT_BITS_TXT_CRLF;
 117        if (stats.lonelf)
 118                ret |=  CONVERT_STAT_BITS_TXT_LF;
 119
 120        return ret;
 121}
 122
 123static const char *gather_convert_stats_ascii(const char *data, unsigned long size)
 124{
 125        unsigned int convert_stats = gather_convert_stats(data, size);
 126
 127        if (convert_stats & CONVERT_STAT_BITS_BIN)
 128                return "-text";
 129        switch (convert_stats) {
 130        case CONVERT_STAT_BITS_TXT_LF:
 131                return "lf";
 132        case CONVERT_STAT_BITS_TXT_CRLF:
 133                return "crlf";
 134        case CONVERT_STAT_BITS_TXT_LF | CONVERT_STAT_BITS_TXT_CRLF:
 135                return "mixed";
 136        default:
 137                return "none";
 138        }
 139}
 140
 141const char *get_cached_convert_stats_ascii(const struct index_state *istate,
 142                                           const char *path)
 143{
 144        const char *ret;
 145        unsigned long sz;
 146        void *data = read_blob_data_from_index(istate, path, &sz);
 147        ret = gather_convert_stats_ascii(data, sz);
 148        free(data);
 149        return ret;
 150}
 151
 152const char *get_wt_convert_stats_ascii(const char *path)
 153{
 154        const char *ret = "";
 155        struct strbuf sb = STRBUF_INIT;
 156        if (strbuf_read_file(&sb, path, 0) >= 0)
 157                ret = gather_convert_stats_ascii(sb.buf, sb.len);
 158        strbuf_release(&sb);
 159        return ret;
 160}
 161
 162static int text_eol_is_crlf(void)
 163{
 164        if (auto_crlf == AUTO_CRLF_TRUE)
 165                return 1;
 166        else if (auto_crlf == AUTO_CRLF_INPUT)
 167                return 0;
 168        if (core_eol == EOL_CRLF)
 169                return 1;
 170        if (core_eol == EOL_UNSET && EOL_NATIVE == EOL_CRLF)
 171                return 1;
 172        return 0;
 173}
 174
 175static enum eol output_eol(enum crlf_action crlf_action)
 176{
 177        switch (crlf_action) {
 178        case CRLF_BINARY:
 179                return EOL_UNSET;
 180        case CRLF_TEXT_CRLF:
 181                return EOL_CRLF;
 182        case CRLF_TEXT_INPUT:
 183                return EOL_LF;
 184        case CRLF_UNDEFINED:
 185        case CRLF_AUTO_CRLF:
 186                return EOL_CRLF;
 187        case CRLF_AUTO_INPUT:
 188                return EOL_LF;
 189        case CRLF_TEXT:
 190        case CRLF_AUTO:
 191                /* fall through */
 192                return text_eol_is_crlf() ? EOL_CRLF : EOL_LF;
 193        }
 194        warning(_("illegal crlf_action %d"), (int)crlf_action);
 195        return core_eol;
 196}
 197
 198static void check_global_conv_flags_eol(const char *path, enum crlf_action crlf_action,
 199                            struct text_stat *old_stats, struct text_stat *new_stats,
 200                            int conv_flags)
 201{
 202        if (old_stats->crlf && !new_stats->crlf ) {
 203                /*
 204                 * CRLFs would not be restored by checkout
 205                 */
 206                if (conv_flags & CONV_EOL_RNDTRP_DIE)
 207                        die(_("CRLF would be replaced by LF in %s"), path);
 208                else if (conv_flags & CONV_EOL_RNDTRP_WARN)
 209                        warning(_("CRLF will be replaced by LF in %s.\n"
 210                                  "The file will have its original line"
 211                                  " endings in your working directory"), path);
 212        } else if (old_stats->lonelf && !new_stats->lonelf ) {
 213                /*
 214                 * CRLFs would be added by checkout
 215                 */
 216                if (conv_flags & CONV_EOL_RNDTRP_DIE)
 217                        die(_("LF would be replaced by CRLF in %s"), path);
 218                else if (conv_flags & CONV_EOL_RNDTRP_WARN)
 219                        warning(_("LF will be replaced by CRLF in %s.\n"
 220                                  "The file will have its original line"
 221                                  " endings in your working directory"), path);
 222        }
 223}
 224
 225static int has_crlf_in_index(const struct index_state *istate, const char *path)
 226{
 227        unsigned long sz;
 228        void *data;
 229        const char *crp;
 230        int has_crlf = 0;
 231
 232        data = read_blob_data_from_index(istate, path, &sz);
 233        if (!data)
 234                return 0;
 235
 236        crp = memchr(data, '\r', sz);
 237        if (crp) {
 238                unsigned int ret_stats;
 239                ret_stats = gather_convert_stats(data, sz);
 240                if (!(ret_stats & CONVERT_STAT_BITS_BIN) &&
 241                    (ret_stats & CONVERT_STAT_BITS_TXT_CRLF))
 242                        has_crlf = 1;
 243        }
 244        free(data);
 245        return has_crlf;
 246}
 247
 248static int will_convert_lf_to_crlf(struct text_stat *stats,
 249                                   enum crlf_action crlf_action)
 250{
 251        if (output_eol(crlf_action) != EOL_CRLF)
 252                return 0;
 253        /* No "naked" LF? Nothing to convert, regardless. */
 254        if (!stats->lonelf)
 255                return 0;
 256
 257        if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
 258                /* If we have any CR or CRLF line endings, we do not touch it */
 259                /* This is the new safer autocrlf-handling */
 260                if (stats->lonecr || stats->crlf)
 261                        return 0;
 262
 263                if (convert_is_binary(stats))
 264                        return 0;
 265        }
 266        return 1;
 267
 268}
 269
 270static int validate_encoding(const char *path, const char *enc,
 271                      const char *data, size_t len, int die_on_error)
 272{
 273        /* We only check for UTF here as UTF?? can be an alias for UTF-?? */
 274        if (istarts_with(enc, "UTF")) {
 275                /*
 276                 * Check for detectable errors in UTF encodings
 277                 */
 278                if (has_prohibited_utf_bom(enc, data, len)) {
 279                        const char *error_msg = _(
 280                                "BOM is prohibited in '%s' if encoded as %s");
 281                        /*
 282                         * This advice is shown for UTF-??BE and UTF-??LE encodings.
 283                         * We cut off the last two characters of the encoding name
 284                         * to generate the encoding name suitable for BOMs.
 285                         */
 286                        const char *advise_msg = _(
 287                                "The file '%s' contains a byte order "
 288                                "mark (BOM). Please use UTF-%s as "
 289                                "working-tree-encoding.");
 290                        const char *stripped = NULL;
 291                        char *upper = xstrdup_toupper(enc);
 292                        upper[strlen(upper)-2] = '\0';
 293                        if (!skip_prefix(upper, "UTF-", &stripped))
 294                                skip_prefix(stripped, "UTF", &stripped);
 295                        advise(advise_msg, path, stripped);
 296                        free(upper);
 297                        if (die_on_error)
 298                                die(error_msg, path, enc);
 299                        else {
 300                                return error(error_msg, path, enc);
 301                        }
 302
 303                } else if (is_missing_required_utf_bom(enc, data, len)) {
 304                        const char *error_msg = _(
 305                                "BOM is required in '%s' if encoded as %s");
 306                        const char *advise_msg = _(
 307                                "The file '%s' is missing a byte order "
 308                                "mark (BOM). Please use UTF-%sBE or UTF-%sLE "
 309                                "(depending on the byte order) as "
 310                                "working-tree-encoding.");
 311                        const char *stripped = NULL;
 312                        char *upper = xstrdup_toupper(enc);
 313                        if (!skip_prefix(upper, "UTF-", &stripped))
 314                                skip_prefix(stripped, "UTF", &stripped);
 315                        advise(advise_msg, path, stripped, stripped);
 316                        free(upper);
 317                        if (die_on_error)
 318                                die(error_msg, path, enc);
 319                        else {
 320                                return error(error_msg, path, enc);
 321                        }
 322                }
 323
 324        }
 325        return 0;
 326}
 327
 328static void trace_encoding(const char *context, const char *path,
 329                           const char *encoding, const char *buf, size_t len)
 330{
 331        static struct trace_key coe = TRACE_KEY_INIT(WORKING_TREE_ENCODING);
 332        struct strbuf trace = STRBUF_INIT;
 333        int i;
 334
 335        strbuf_addf(&trace, "%s (%s, considered %s):\n", context, path, encoding);
 336        for (i = 0; i < len && buf; ++i) {
 337                strbuf_addf(
 338                        &trace, "| \033[2m%2i:\033[0m %2x \033[2m%c\033[0m%c",
 339                        i,
 340                        (unsigned char) buf[i],
 341                        (buf[i] > 32 && buf[i] < 127 ? buf[i] : ' '),
 342                        ((i+1) % 8 && (i+1) < len ? ' ' : '\n')
 343                );
 344        }
 345        strbuf_addchars(&trace, '\n', 1);
 346
 347        trace_strbuf(&coe, &trace);
 348        strbuf_release(&trace);
 349}
 350
 351static int check_roundtrip(const char *enc_name)
 352{
 353        /*
 354         * check_roundtrip_encoding contains a string of comma and/or
 355         * space separated encodings (eg. "UTF-16, ASCII, CP1125").
 356         * Search for the given encoding in that string.
 357         */
 358        const char *found = strcasestr(check_roundtrip_encoding, enc_name);
 359        const char *next;
 360        int len;
 361        if (!found)
 362                return 0;
 363        next = found + strlen(enc_name);
 364        len = strlen(check_roundtrip_encoding);
 365        return (found && (
 366                        /*
 367                         * check that the found encoding is at the
 368                         * beginning of check_roundtrip_encoding or
 369                         * that it is prefixed with a space or comma
 370                         */
 371                        found == check_roundtrip_encoding || (
 372                                (isspace(found[-1]) || found[-1] == ',')
 373                        )
 374                ) && (
 375                        /*
 376                         * check that the found encoding is at the
 377                         * end of check_roundtrip_encoding or
 378                         * that it is suffixed with a space or comma
 379                         */
 380                        next == check_roundtrip_encoding + len || (
 381                                next < check_roundtrip_encoding + len &&
 382                                (isspace(next[0]) || next[0] == ',')
 383                        )
 384                ));
 385}
 386
 387static const char *default_encoding = "UTF-8";
 388
 389static int encode_to_git(const char *path, const char *src, size_t src_len,
 390                         struct strbuf *buf, const char *enc, int conv_flags)
 391{
 392        char *dst;
 393        size_t dst_len;
 394        int die_on_error = conv_flags & CONV_WRITE_OBJECT;
 395
 396        /*
 397         * No encoding is specified or there is nothing to encode.
 398         * Tell the caller that the content was not modified.
 399         */
 400        if (!enc || (src && !src_len))
 401                return 0;
 402
 403        /*
 404         * Looks like we got called from "would_convert_to_git()".
 405         * This means Git wants to know if it would encode (= modify!)
 406         * the content. Let's answer with "yes", since an encoding was
 407         * specified.
 408         */
 409        if (!buf && !src)
 410                return 1;
 411
 412        if (validate_encoding(path, enc, src, src_len, die_on_error))
 413                return 0;
 414
 415        trace_encoding("source", path, enc, src, src_len);
 416        dst = reencode_string_len(src, src_len, default_encoding, enc,
 417                                  &dst_len);
 418        if (!dst) {
 419                /*
 420                 * We could add the blob "as-is" to Git. However, on checkout
 421                 * we would try to reencode to the original encoding. This
 422                 * would fail and we would leave the user with a messed-up
 423                 * working tree. Let's try to avoid this by screaming loud.
 424                 */
 425                const char* msg = _("failed to encode '%s' from %s to %s");
 426                if (die_on_error)
 427                        die(msg, path, enc, default_encoding);
 428                else {
 429                        error(msg, path, enc, default_encoding);
 430                        return 0;
 431                }
 432        }
 433        trace_encoding("destination", path, default_encoding, dst, dst_len);
 434
 435        /*
 436         * UTF supports lossless conversion round tripping [1] and conversions
 437         * between UTF and other encodings are mostly round trip safe as
 438         * Unicode aims to be a superset of all other character encodings.
 439         * However, certain encodings (e.g. SHIFT-JIS) are known to have round
 440         * trip issues [2]. Check the round trip conversion for all encodings
 441         * listed in core.checkRoundtripEncoding.
 442         *
 443         * The round trip check is only performed if content is written to Git.
 444         * This ensures that no information is lost during conversion to/from
 445         * the internal UTF-8 representation.
 446         *
 447         * Please note, the code below is not tested because I was not able to
 448         * generate a faulty round trip without an iconv error. Iconv errors
 449         * are already caught above.
 450         *
 451         * [1] http://unicode.org/faq/utf_bom.html#gen2
 452         * [2] https://support.microsoft.com/en-us/help/170559/prb-conversion-problem-between-shift-jis-and-unicode
 453         */
 454        if (die_on_error && check_roundtrip(enc)) {
 455                char *re_src;
 456                size_t re_src_len;
 457
 458                re_src = reencode_string_len(dst, dst_len,
 459                                             enc, default_encoding,
 460                                             &re_src_len);
 461
 462                trace_printf("Checking roundtrip encoding for %s...\n", enc);
 463                trace_encoding("reencoded source", path, enc,
 464                               re_src, re_src_len);
 465
 466                if (!re_src || src_len != re_src_len ||
 467                    memcmp(src, re_src, src_len)) {
 468                        const char* msg = _("encoding '%s' from %s to %s and "
 469                                            "back is not the same");
 470                        die(msg, path, enc, default_encoding);
 471                }
 472
 473                free(re_src);
 474        }
 475
 476        strbuf_attach(buf, dst, dst_len, dst_len + 1);
 477        return 1;
 478}
 479
 480static int encode_to_worktree(const char *path, const char *src, size_t src_len,
 481                              struct strbuf *buf, const char *enc)
 482{
 483        char *dst;
 484        size_t dst_len;
 485
 486        /*
 487         * No encoding is specified or there is nothing to encode.
 488         * Tell the caller that the content was not modified.
 489         */
 490        if (!enc || (src && !src_len))
 491                return 0;
 492
 493        dst = reencode_string_len(src, src_len, enc, default_encoding,
 494                                  &dst_len);
 495        if (!dst) {
 496                error(_("failed to encode '%s' from %s to %s"),
 497                      path, default_encoding, enc);
 498                return 0;
 499        }
 500
 501        strbuf_attach(buf, dst, dst_len, dst_len + 1);
 502        return 1;
 503}
 504
 505static int crlf_to_git(const struct index_state *istate,
 506                       const char *path, const char *src, size_t len,
 507                       struct strbuf *buf,
 508                       enum crlf_action crlf_action, int conv_flags)
 509{
 510        struct text_stat stats;
 511        char *dst;
 512        int convert_crlf_into_lf;
 513
 514        if (crlf_action == CRLF_BINARY ||
 515            (src && !len))
 516                return 0;
 517
 518        /*
 519         * If we are doing a dry-run and have no source buffer, there is
 520         * nothing to analyze; we must assume we would convert.
 521         */
 522        if (!buf && !src)
 523                return 1;
 524
 525        gather_stats(src, len, &stats);
 526        /* Optimization: No CRLF? Nothing to convert, regardless. */
 527        convert_crlf_into_lf = !!stats.crlf;
 528
 529        if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
 530                if (convert_is_binary(&stats))
 531                        return 0;
 532                /*
 533                 * If the file in the index has any CR in it, do not
 534                 * convert.  This is the new safer autocrlf handling,
 535                 * unless we want to renormalize in a merge or
 536                 * cherry-pick.
 537                 */
 538                if ((!(conv_flags & CONV_EOL_RENORMALIZE)) &&
 539                    has_crlf_in_index(istate, path))
 540                        convert_crlf_into_lf = 0;
 541        }
 542        if (((conv_flags & CONV_EOL_RNDTRP_WARN) ||
 543             ((conv_flags & CONV_EOL_RNDTRP_DIE) && len))) {
 544                struct text_stat new_stats;
 545                memcpy(&new_stats, &stats, sizeof(new_stats));
 546                /* simulate "git add" */
 547                if (convert_crlf_into_lf) {
 548                        new_stats.lonelf += new_stats.crlf;
 549                        new_stats.crlf = 0;
 550                }
 551                /* simulate "git checkout" */
 552                if (will_convert_lf_to_crlf(&new_stats, crlf_action)) {
 553                        new_stats.crlf += new_stats.lonelf;
 554                        new_stats.lonelf = 0;
 555                }
 556                check_global_conv_flags_eol(path, crlf_action, &stats, &new_stats, conv_flags);
 557        }
 558        if (!convert_crlf_into_lf)
 559                return 0;
 560
 561        /*
 562         * At this point all of our source analysis is done, and we are sure we
 563         * would convert. If we are in dry-run mode, we can give an answer.
 564         */
 565        if (!buf)
 566                return 1;
 567
 568        /* only grow if not in place */
 569        if (strbuf_avail(buf) + buf->len < len)
 570                strbuf_grow(buf, len - buf->len);
 571        dst = buf->buf;
 572        if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
 573                /*
 574                 * If we guessed, we already know we rejected a file with
 575                 * lone CR, and we can strip a CR without looking at what
 576                 * follow it.
 577                 */
 578                do {
 579                        unsigned char c = *src++;
 580                        if (c != '\r')
 581                                *dst++ = c;
 582                } while (--len);
 583        } else {
 584                do {
 585                        unsigned char c = *src++;
 586                        if (! (c == '\r' && (1 < len && *src == '\n')))
 587                                *dst++ = c;
 588                } while (--len);
 589        }
 590        strbuf_setlen(buf, dst - buf->buf);
 591        return 1;
 592}
 593
 594static int crlf_to_worktree(const char *src, size_t len,
 595                            struct strbuf *buf, enum crlf_action crlf_action)
 596{
 597        char *to_free = NULL;
 598        struct text_stat stats;
 599
 600        if (!len || output_eol(crlf_action) != EOL_CRLF)
 601                return 0;
 602
 603        gather_stats(src, len, &stats);
 604        if (!will_convert_lf_to_crlf(&stats, crlf_action))
 605                return 0;
 606
 607        /* are we "faking" in place editing ? */
 608        if (src == buf->buf)
 609                to_free = strbuf_detach(buf, NULL);
 610
 611        strbuf_grow(buf, len + stats.lonelf);
 612        for (;;) {
 613                const char *nl = memchr(src, '\n', len);
 614                if (!nl)
 615                        break;
 616                if (nl > src && nl[-1] == '\r') {
 617                        strbuf_add(buf, src, nl + 1 - src);
 618                } else {
 619                        strbuf_add(buf, src, nl - src);
 620                        strbuf_addstr(buf, "\r\n");
 621                }
 622                len -= nl + 1 - src;
 623                src  = nl + 1;
 624        }
 625        strbuf_add(buf, src, len);
 626
 627        free(to_free);
 628        return 1;
 629}
 630
 631struct filter_params {
 632        const char *src;
 633        unsigned long size;
 634        int fd;
 635        const char *cmd;
 636        const char *path;
 637};
 638
 639static int filter_buffer_or_fd(int in, int out, void *data)
 640{
 641        /*
 642         * Spawn cmd and feed the buffer contents through its stdin.
 643         */
 644        struct child_process child_process = CHILD_PROCESS_INIT;
 645        struct filter_params *params = (struct filter_params *)data;
 646        int write_err, status;
 647        const char *argv[] = { NULL, NULL };
 648
 649        /* apply % substitution to cmd */
 650        struct strbuf cmd = STRBUF_INIT;
 651        struct strbuf path = STRBUF_INIT;
 652        struct strbuf_expand_dict_entry dict[] = {
 653                { "f", NULL, },
 654                { NULL, NULL, },
 655        };
 656
 657        /* quote the path to preserve spaces, etc. */
 658        sq_quote_buf(&path, params->path);
 659        dict[0].value = path.buf;
 660
 661        /* expand all %f with the quoted path */
 662        strbuf_expand(&cmd, params->cmd, strbuf_expand_dict_cb, &dict);
 663        strbuf_release(&path);
 664
 665        argv[0] = cmd.buf;
 666
 667        child_process.argv = argv;
 668        child_process.use_shell = 1;
 669        child_process.in = -1;
 670        child_process.out = out;
 671
 672        if (start_command(&child_process)) {
 673                strbuf_release(&cmd);
 674                return error(_("cannot fork to run external filter '%s'"),
 675                             params->cmd);
 676        }
 677
 678        sigchain_push(SIGPIPE, SIG_IGN);
 679
 680        if (params->src) {
 681                write_err = (write_in_full(child_process.in,
 682                                           params->src, params->size) < 0);
 683                if (errno == EPIPE)
 684                        write_err = 0;
 685        } else {
 686                write_err = copy_fd(params->fd, child_process.in);
 687                if (write_err == COPY_WRITE_ERROR && errno == EPIPE)
 688                        write_err = 0;
 689        }
 690
 691        if (close(child_process.in))
 692                write_err = 1;
 693        if (write_err)
 694                error(_("cannot feed the input to external filter '%s'"),
 695                      params->cmd);
 696
 697        sigchain_pop(SIGPIPE);
 698
 699        status = finish_command(&child_process);
 700        if (status)
 701                error(_("external filter '%s' failed %d"), params->cmd, status);
 702
 703        strbuf_release(&cmd);
 704        return (write_err || status);
 705}
 706
 707static int apply_single_file_filter(const char *path, const char *src, size_t len, int fd,
 708                                    struct strbuf *dst, const char *cmd)
 709{
 710        /*
 711         * Create a pipeline to have the command filter the buffer's
 712         * contents.
 713         *
 714         * (child --> cmd) --> us
 715         */
 716        int err = 0;
 717        struct strbuf nbuf = STRBUF_INIT;
 718        struct async async;
 719        struct filter_params params;
 720
 721        memset(&async, 0, sizeof(async));
 722        async.proc = filter_buffer_or_fd;
 723        async.data = &params;
 724        async.out = -1;
 725        params.src = src;
 726        params.size = len;
 727        params.fd = fd;
 728        params.cmd = cmd;
 729        params.path = path;
 730
 731        fflush(NULL);
 732        if (start_async(&async))
 733                return 0;       /* error was already reported */
 734
 735        if (strbuf_read(&nbuf, async.out, 0) < 0) {
 736                err = error(_("read from external filter '%s' failed"), cmd);
 737        }
 738        if (close(async.out)) {
 739                err = error(_("read from external filter '%s' failed"), cmd);
 740        }
 741        if (finish_async(&async)) {
 742                err = error(_("external filter '%s' failed"), cmd);
 743        }
 744
 745        if (!err) {
 746                strbuf_swap(dst, &nbuf);
 747        }
 748        strbuf_release(&nbuf);
 749        return !err;
 750}
 751
 752#define CAP_CLEAN    (1u<<0)
 753#define CAP_SMUDGE   (1u<<1)
 754#define CAP_DELAY    (1u<<2)
 755
 756struct cmd2process {
 757        struct subprocess_entry subprocess; /* must be the first member! */
 758        unsigned int supported_capabilities;
 759};
 760
 761static int subprocess_map_initialized;
 762static struct hashmap subprocess_map;
 763
 764static int start_multi_file_filter_fn(struct subprocess_entry *subprocess)
 765{
 766        static int versions[] = {2, 0};
 767        static struct subprocess_capability capabilities[] = {
 768                { "clean",  CAP_CLEAN  },
 769                { "smudge", CAP_SMUDGE },
 770                { "delay",  CAP_DELAY  },
 771                { NULL, 0 }
 772        };
 773        struct cmd2process *entry = (struct cmd2process *)subprocess;
 774        return subprocess_handshake(subprocess, "git-filter", versions, NULL,
 775                                    capabilities,
 776                                    &entry->supported_capabilities);
 777}
 778
 779static void handle_filter_error(const struct strbuf *filter_status,
 780                                struct cmd2process *entry,
 781                                const unsigned int wanted_capability)
 782{
 783        if (!strcmp(filter_status->buf, "error"))
 784                ; /* The filter signaled a problem with the file. */
 785        else if (!strcmp(filter_status->buf, "abort") && wanted_capability) {
 786                /*
 787                 * The filter signaled a permanent problem. Don't try to filter
 788                 * files with the same command for the lifetime of the current
 789                 * Git process.
 790                 */
 791                 entry->supported_capabilities &= ~wanted_capability;
 792        } else {
 793                /*
 794                 * Something went wrong with the protocol filter.
 795                 * Force shutdown and restart if another blob requires filtering.
 796                 */
 797                error(_("external filter '%s' failed"), entry->subprocess.cmd);
 798                subprocess_stop(&subprocess_map, &entry->subprocess);
 799                free(entry);
 800        }
 801}
 802
 803static int apply_multi_file_filter(const char *path, const char *src, size_t len,
 804                                   int fd, struct strbuf *dst, const char *cmd,
 805                                   const unsigned int wanted_capability,
 806                                   struct delayed_checkout *dco)
 807{
 808        int err;
 809        int can_delay = 0;
 810        struct cmd2process *entry;
 811        struct child_process *process;
 812        struct strbuf nbuf = STRBUF_INIT;
 813        struct strbuf filter_status = STRBUF_INIT;
 814        const char *filter_type;
 815
 816        if (!subprocess_map_initialized) {
 817                subprocess_map_initialized = 1;
 818                hashmap_init(&subprocess_map, cmd2process_cmp, NULL, 0);
 819                entry = NULL;
 820        } else {
 821                entry = (struct cmd2process *)subprocess_find_entry(&subprocess_map, cmd);
 822        }
 823
 824        fflush(NULL);
 825
 826        if (!entry) {
 827                entry = xmalloc(sizeof(*entry));
 828                entry->supported_capabilities = 0;
 829
 830                if (subprocess_start(&subprocess_map, &entry->subprocess, cmd, start_multi_file_filter_fn)) {
 831                        free(entry);
 832                        return 0;
 833                }
 834        }
 835        process = &entry->subprocess.process;
 836
 837        if (!(entry->supported_capabilities & wanted_capability))
 838                return 0;
 839
 840        if (wanted_capability & CAP_CLEAN)
 841                filter_type = "clean";
 842        else if (wanted_capability & CAP_SMUDGE)
 843                filter_type = "smudge";
 844        else
 845                die(_("unexpected filter type"));
 846
 847        sigchain_push(SIGPIPE, SIG_IGN);
 848
 849        assert(strlen(filter_type) < LARGE_PACKET_DATA_MAX - strlen("command=\n"));
 850        err = packet_write_fmt_gently(process->in, "command=%s\n", filter_type);
 851        if (err)
 852                goto done;
 853
 854        err = strlen(path) > LARGE_PACKET_DATA_MAX - strlen("pathname=\n");
 855        if (err) {
 856                error(_("path name too long for external filter"));
 857                goto done;
 858        }
 859
 860        err = packet_write_fmt_gently(process->in, "pathname=%s\n", path);
 861        if (err)
 862                goto done;
 863
 864        if ((entry->supported_capabilities & CAP_DELAY) &&
 865            dco && dco->state == CE_CAN_DELAY) {
 866                can_delay = 1;
 867                err = packet_write_fmt_gently(process->in, "can-delay=1\n");
 868                if (err)
 869                        goto done;
 870        }
 871
 872        err = packet_flush_gently(process->in);
 873        if (err)
 874                goto done;
 875
 876        if (fd >= 0)
 877                err = write_packetized_from_fd(fd, process->in);
 878        else
 879                err = write_packetized_from_buf(src, len, process->in);
 880        if (err)
 881                goto done;
 882
 883        err = subprocess_read_status(process->out, &filter_status);
 884        if (err)
 885                goto done;
 886
 887        if (can_delay && !strcmp(filter_status.buf, "delayed")) {
 888                string_list_insert(&dco->filters, cmd);
 889                string_list_insert(&dco->paths, path);
 890        } else {
 891                /* The filter got the blob and wants to send us a response. */
 892                err = strcmp(filter_status.buf, "success");
 893                if (err)
 894                        goto done;
 895
 896                err = read_packetized_to_strbuf(process->out, &nbuf) < 0;
 897                if (err)
 898                        goto done;
 899
 900                err = subprocess_read_status(process->out, &filter_status);
 901                if (err)
 902                        goto done;
 903
 904                err = strcmp(filter_status.buf, "success");
 905        }
 906
 907done:
 908        sigchain_pop(SIGPIPE);
 909
 910        if (err)
 911                handle_filter_error(&filter_status, entry, wanted_capability);
 912        else
 913                strbuf_swap(dst, &nbuf);
 914        strbuf_release(&nbuf);
 915        return !err;
 916}
 917
 918
 919int async_query_available_blobs(const char *cmd, struct string_list *available_paths)
 920{
 921        int err;
 922        char *line;
 923        struct cmd2process *entry;
 924        struct child_process *process;
 925        struct strbuf filter_status = STRBUF_INIT;
 926
 927        assert(subprocess_map_initialized);
 928        entry = (struct cmd2process *)subprocess_find_entry(&subprocess_map, cmd);
 929        if (!entry) {
 930                error(_("external filter '%s' is not available anymore although "
 931                        "not all paths have been filtered"), cmd);
 932                return 0;
 933        }
 934        process = &entry->subprocess.process;
 935        sigchain_push(SIGPIPE, SIG_IGN);
 936
 937        err = packet_write_fmt_gently(
 938                process->in, "command=list_available_blobs\n");
 939        if (err)
 940                goto done;
 941
 942        err = packet_flush_gently(process->in);
 943        if (err)
 944                goto done;
 945
 946        while ((line = packet_read_line(process->out, NULL))) {
 947                const char *path;
 948                if (skip_prefix(line, "pathname=", &path))
 949                        string_list_insert(available_paths, xstrdup(path));
 950                else
 951                        ; /* ignore unknown keys */
 952        }
 953
 954        err = subprocess_read_status(process->out, &filter_status);
 955        if (err)
 956                goto done;
 957
 958        err = strcmp(filter_status.buf, "success");
 959
 960done:
 961        sigchain_pop(SIGPIPE);
 962
 963        if (err)
 964                handle_filter_error(&filter_status, entry, 0);
 965        return !err;
 966}
 967
 968static struct convert_driver {
 969        const char *name;
 970        struct convert_driver *next;
 971        const char *smudge;
 972        const char *clean;
 973        const char *process;
 974        int required;
 975} *user_convert, **user_convert_tail;
 976
 977static int apply_filter(const char *path, const char *src, size_t len,
 978                        int fd, struct strbuf *dst, struct convert_driver *drv,
 979                        const unsigned int wanted_capability,
 980                        struct delayed_checkout *dco)
 981{
 982        const char *cmd = NULL;
 983
 984        if (!drv)
 985                return 0;
 986
 987        if (!dst)
 988                return 1;
 989
 990        if ((wanted_capability & CAP_CLEAN) && !drv->process && drv->clean)
 991                cmd = drv->clean;
 992        else if ((wanted_capability & CAP_SMUDGE) && !drv->process && drv->smudge)
 993                cmd = drv->smudge;
 994
 995        if (cmd && *cmd)
 996                return apply_single_file_filter(path, src, len, fd, dst, cmd);
 997        else if (drv->process && *drv->process)
 998                return apply_multi_file_filter(path, src, len, fd, dst,
 999                        drv->process, wanted_capability, dco);
1000
1001        return 0;
1002}
1003
1004static int read_convert_config(const char *var, const char *value, void *cb)
1005{
1006        const char *key, *name;
1007        int namelen;
1008        struct convert_driver *drv;
1009
1010        /*
1011         * External conversion drivers are configured using
1012         * "filter.<name>.variable".
1013         */
1014        if (parse_config_key(var, "filter", &name, &namelen, &key) < 0 || !name)
1015                return 0;
1016        for (drv = user_convert; drv; drv = drv->next)
1017                if (!strncmp(drv->name, name, namelen) && !drv->name[namelen])
1018                        break;
1019        if (!drv) {
1020                drv = xcalloc(1, sizeof(struct convert_driver));
1021                drv->name = xmemdupz(name, namelen);
1022                *user_convert_tail = drv;
1023                user_convert_tail = &(drv->next);
1024        }
1025
1026        /*
1027         * filter.<name>.smudge and filter.<name>.clean specifies
1028         * the command line:
1029         *
1030         *      command-line
1031         *
1032         * The command-line will not be interpolated in any way.
1033         */
1034
1035        if (!strcmp("smudge", key))
1036                return git_config_string(&drv->smudge, var, value);
1037
1038        if (!strcmp("clean", key))
1039                return git_config_string(&drv->clean, var, value);
1040
1041        if (!strcmp("process", key))
1042                return git_config_string(&drv->process, var, value);
1043
1044        if (!strcmp("required", key)) {
1045                drv->required = git_config_bool(var, value);
1046                return 0;
1047        }
1048
1049        return 0;
1050}
1051
1052static int count_ident(const char *cp, unsigned long size)
1053{
1054        /*
1055         * "$Id: 0000000000000000000000000000000000000000 $" <=> "$Id$"
1056         */
1057        int cnt = 0;
1058        char ch;
1059
1060        while (size) {
1061                ch = *cp++;
1062                size--;
1063                if (ch != '$')
1064                        continue;
1065                if (size < 3)
1066                        break;
1067                if (memcmp("Id", cp, 2))
1068                        continue;
1069                ch = cp[2];
1070                cp += 3;
1071                size -= 3;
1072                if (ch == '$')
1073                        cnt++; /* $Id$ */
1074                if (ch != ':')
1075                        continue;
1076
1077                /*
1078                 * "$Id: ... "; scan up to the closing dollar sign and discard.
1079                 */
1080                while (size) {
1081                        ch = *cp++;
1082                        size--;
1083                        if (ch == '$') {
1084                                cnt++;
1085                                break;
1086                        }
1087                        if (ch == '\n')
1088                                break;
1089                }
1090        }
1091        return cnt;
1092}
1093
1094static int ident_to_git(const char *src, size_t len,
1095                        struct strbuf *buf, int ident)
1096{
1097        char *dst, *dollar;
1098
1099        if (!ident || (src && !count_ident(src, len)))
1100                return 0;
1101
1102        if (!buf)
1103                return 1;
1104
1105        /* only grow if not in place */
1106        if (strbuf_avail(buf) + buf->len < len)
1107                strbuf_grow(buf, len - buf->len);
1108        dst = buf->buf;
1109        for (;;) {
1110                dollar = memchr(src, '$', len);
1111                if (!dollar)
1112                        break;
1113                memmove(dst, src, dollar + 1 - src);
1114                dst += dollar + 1 - src;
1115                len -= dollar + 1 - src;
1116                src  = dollar + 1;
1117
1118                if (len > 3 && !memcmp(src, "Id:", 3)) {
1119                        dollar = memchr(src + 3, '$', len - 3);
1120                        if (!dollar)
1121                                break;
1122                        if (memchr(src + 3, '\n', dollar - src - 3)) {
1123                                /* Line break before the next dollar. */
1124                                continue;
1125                        }
1126
1127                        memcpy(dst, "Id$", 3);
1128                        dst += 3;
1129                        len -= dollar + 1 - src;
1130                        src  = dollar + 1;
1131                }
1132        }
1133        memmove(dst, src, len);
1134        strbuf_setlen(buf, dst + len - buf->buf);
1135        return 1;
1136}
1137
1138static int ident_to_worktree(const char *src, size_t len,
1139                             struct strbuf *buf, int ident)
1140{
1141        struct object_id oid;
1142        char *to_free = NULL, *dollar, *spc;
1143        int cnt;
1144
1145        if (!ident)
1146                return 0;
1147
1148        cnt = count_ident(src, len);
1149        if (!cnt)
1150                return 0;
1151
1152        /* are we "faking" in place editing ? */
1153        if (src == buf->buf)
1154                to_free = strbuf_detach(buf, NULL);
1155        hash_object_file(src, len, "blob", &oid);
1156
1157        strbuf_grow(buf, len + cnt * (the_hash_algo->hexsz + 3));
1158        for (;;) {
1159                /* step 1: run to the next '$' */
1160                dollar = memchr(src, '$', len);
1161                if (!dollar)
1162                        break;
1163                strbuf_add(buf, src, dollar + 1 - src);
1164                len -= dollar + 1 - src;
1165                src  = dollar + 1;
1166
1167                /* step 2: does it looks like a bit like Id:xxx$ or Id$ ? */
1168                if (len < 3 || memcmp("Id", src, 2))
1169                        continue;
1170
1171                /* step 3: skip over Id$ or Id:xxxxx$ */
1172                if (src[2] == '$') {
1173                        src += 3;
1174                        len -= 3;
1175                } else if (src[2] == ':') {
1176                        /*
1177                         * It's possible that an expanded Id has crept its way into the
1178                         * repository, we cope with that by stripping the expansion out.
1179                         * This is probably not a good idea, since it will cause changes
1180                         * on checkout, which won't go away by stash, but let's keep it
1181                         * for git-style ids.
1182                         */
1183                        dollar = memchr(src + 3, '$', len - 3);
1184                        if (!dollar) {
1185                                /* incomplete keyword, no more '$', so just quit the loop */
1186                                break;
1187                        }
1188
1189                        if (memchr(src + 3, '\n', dollar - src - 3)) {
1190                                /* Line break before the next dollar. */
1191                                continue;
1192                        }
1193
1194                        spc = memchr(src + 4, ' ', dollar - src - 4);
1195                        if (spc && spc < dollar-1) {
1196                                /* There are spaces in unexpected places.
1197                                 * This is probably an id from some other
1198                                 * versioning system. Keep it for now.
1199                                 */
1200                                continue;
1201                        }
1202
1203                        len -= dollar + 1 - src;
1204                        src  = dollar + 1;
1205                } else {
1206                        /* it wasn't a "Id$" or "Id:xxxx$" */
1207                        continue;
1208                }
1209
1210                /* step 4: substitute */
1211                strbuf_addstr(buf, "Id: ");
1212                strbuf_addstr(buf, oid_to_hex(&oid));
1213                strbuf_addstr(buf, " $");
1214        }
1215        strbuf_add(buf, src, len);
1216
1217        free(to_free);
1218        return 1;
1219}
1220
1221static const char *git_path_check_encoding(struct attr_check_item *check)
1222{
1223        const char *value = check->value;
1224
1225        if (ATTR_UNSET(value) || !strlen(value))
1226                return NULL;
1227
1228        if (ATTR_TRUE(value) || ATTR_FALSE(value)) {
1229                die(_("true/false are no valid working-tree-encodings"));
1230        }
1231
1232        /* Don't encode to the default encoding */
1233        if (same_encoding(value, default_encoding))
1234                return NULL;
1235
1236        return value;
1237}
1238
1239static enum crlf_action git_path_check_crlf(struct attr_check_item *check)
1240{
1241        const char *value = check->value;
1242
1243        if (ATTR_TRUE(value))
1244                return CRLF_TEXT;
1245        else if (ATTR_FALSE(value))
1246                return CRLF_BINARY;
1247        else if (ATTR_UNSET(value))
1248                ;
1249        else if (!strcmp(value, "input"))
1250                return CRLF_TEXT_INPUT;
1251        else if (!strcmp(value, "auto"))
1252                return CRLF_AUTO;
1253        return CRLF_UNDEFINED;
1254}
1255
1256static enum eol git_path_check_eol(struct attr_check_item *check)
1257{
1258        const char *value = check->value;
1259
1260        if (ATTR_UNSET(value))
1261                ;
1262        else if (!strcmp(value, "lf"))
1263                return EOL_LF;
1264        else if (!strcmp(value, "crlf"))
1265                return EOL_CRLF;
1266        return EOL_UNSET;
1267}
1268
1269static struct convert_driver *git_path_check_convert(struct attr_check_item *check)
1270{
1271        const char *value = check->value;
1272        struct convert_driver *drv;
1273
1274        if (ATTR_TRUE(value) || ATTR_FALSE(value) || ATTR_UNSET(value))
1275                return NULL;
1276        for (drv = user_convert; drv; drv = drv->next)
1277                if (!strcmp(value, drv->name))
1278                        return drv;
1279        return NULL;
1280}
1281
1282static int git_path_check_ident(struct attr_check_item *check)
1283{
1284        const char *value = check->value;
1285
1286        return !!ATTR_TRUE(value);
1287}
1288
1289struct conv_attrs {
1290        struct convert_driver *drv;
1291        enum crlf_action attr_action; /* What attr says */
1292        enum crlf_action crlf_action; /* When no attr is set, use core.autocrlf */
1293        int ident;
1294        const char *working_tree_encoding; /* Supported encoding or default encoding if NULL */
1295};
1296
1297static struct attr_check *check;
1298
1299static void convert_attrs(const struct index_state *istate,
1300                          struct conv_attrs *ca, const char *path)
1301{
1302        struct attr_check_item *ccheck = NULL;
1303
1304        if (!check) {
1305                check = attr_check_initl("crlf", "ident", "filter",
1306                                         "eol", "text", "working-tree-encoding",
1307                                         NULL);
1308                user_convert_tail = &user_convert;
1309                git_config(read_convert_config, NULL);
1310        }
1311
1312        git_check_attr(istate, path, check);
1313        ccheck = check->items;
1314        ca->crlf_action = git_path_check_crlf(ccheck + 4);
1315        if (ca->crlf_action == CRLF_UNDEFINED)
1316                ca->crlf_action = git_path_check_crlf(ccheck + 0);
1317        ca->ident = git_path_check_ident(ccheck + 1);
1318        ca->drv = git_path_check_convert(ccheck + 2);
1319        if (ca->crlf_action != CRLF_BINARY) {
1320                enum eol eol_attr = git_path_check_eol(ccheck + 3);
1321                if (ca->crlf_action == CRLF_AUTO && eol_attr == EOL_LF)
1322                        ca->crlf_action = CRLF_AUTO_INPUT;
1323                else if (ca->crlf_action == CRLF_AUTO && eol_attr == EOL_CRLF)
1324                        ca->crlf_action = CRLF_AUTO_CRLF;
1325                else if (eol_attr == EOL_LF)
1326                        ca->crlf_action = CRLF_TEXT_INPUT;
1327                else if (eol_attr == EOL_CRLF)
1328                        ca->crlf_action = CRLF_TEXT_CRLF;
1329        }
1330        ca->working_tree_encoding = git_path_check_encoding(ccheck + 5);
1331
1332        /* Save attr and make a decision for action */
1333        ca->attr_action = ca->crlf_action;
1334        if (ca->crlf_action == CRLF_TEXT)
1335                ca->crlf_action = text_eol_is_crlf() ? CRLF_TEXT_CRLF : CRLF_TEXT_INPUT;
1336        if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_FALSE)
1337                ca->crlf_action = CRLF_BINARY;
1338        if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_TRUE)
1339                ca->crlf_action = CRLF_AUTO_CRLF;
1340        if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_INPUT)
1341                ca->crlf_action = CRLF_AUTO_INPUT;
1342}
1343
1344void reset_parsed_attributes(void)
1345{
1346        struct convert_driver *drv, *next;
1347
1348        attr_check_free(check);
1349        check = NULL;
1350        reset_merge_attributes();
1351
1352        for (drv = user_convert; drv; drv = next) {
1353                next = drv->next;
1354                free((void *)drv->name);
1355                free(drv);
1356        }
1357        user_convert = NULL;
1358        user_convert_tail = NULL;
1359}
1360
1361int would_convert_to_git_filter_fd(const struct index_state *istate, const char *path)
1362{
1363        struct conv_attrs ca;
1364
1365        convert_attrs(istate, &ca, path);
1366        if (!ca.drv)
1367                return 0;
1368
1369        /*
1370         * Apply a filter to an fd only if the filter is required to succeed.
1371         * We must die if the filter fails, because the original data before
1372         * filtering is not available.
1373         */
1374        if (!ca.drv->required)
1375                return 0;
1376
1377        return apply_filter(path, NULL, 0, -1, NULL, ca.drv, CAP_CLEAN, NULL);
1378}
1379
1380const char *get_convert_attr_ascii(const struct index_state *istate, const char *path)
1381{
1382        struct conv_attrs ca;
1383
1384        convert_attrs(istate, &ca, path);
1385        switch (ca.attr_action) {
1386        case CRLF_UNDEFINED:
1387                return "";
1388        case CRLF_BINARY:
1389                return "-text";
1390        case CRLF_TEXT:
1391                return "text";
1392        case CRLF_TEXT_INPUT:
1393                return "text eol=lf";
1394        case CRLF_TEXT_CRLF:
1395                return "text eol=crlf";
1396        case CRLF_AUTO:
1397                return "text=auto";
1398        case CRLF_AUTO_CRLF:
1399                return "text=auto eol=crlf";
1400        case CRLF_AUTO_INPUT:
1401                return "text=auto eol=lf";
1402        }
1403        return "";
1404}
1405
1406int convert_to_git(const struct index_state *istate,
1407                   const char *path, const char *src, size_t len,
1408                   struct strbuf *dst, int conv_flags)
1409{
1410        int ret = 0;
1411        struct conv_attrs ca;
1412
1413        convert_attrs(istate, &ca, path);
1414
1415        ret |= apply_filter(path, src, len, -1, dst, ca.drv, CAP_CLEAN, NULL);
1416        if (!ret && ca.drv && ca.drv->required)
1417                die(_("%s: clean filter '%s' failed"), path, ca.drv->name);
1418
1419        if (ret && dst) {
1420                src = dst->buf;
1421                len = dst->len;
1422        }
1423
1424        ret |= encode_to_git(path, src, len, dst, ca.working_tree_encoding, conv_flags);
1425        if (ret && dst) {
1426                src = dst->buf;
1427                len = dst->len;
1428        }
1429
1430        if (!(conv_flags & CONV_EOL_KEEP_CRLF)) {
1431                ret |= crlf_to_git(istate, path, src, len, dst, ca.crlf_action, conv_flags);
1432                if (ret && dst) {
1433                        src = dst->buf;
1434                        len = dst->len;
1435                }
1436        }
1437        return ret | ident_to_git(src, len, dst, ca.ident);
1438}
1439
1440void convert_to_git_filter_fd(const struct index_state *istate,
1441                              const char *path, int fd, struct strbuf *dst,
1442                              int conv_flags)
1443{
1444        struct conv_attrs ca;
1445        convert_attrs(istate, &ca, path);
1446
1447        assert(ca.drv);
1448        assert(ca.drv->clean || ca.drv->process);
1449
1450        if (!apply_filter(path, NULL, 0, fd, dst, ca.drv, CAP_CLEAN, NULL))
1451                die(_("%s: clean filter '%s' failed"), path, ca.drv->name);
1452
1453        encode_to_git(path, dst->buf, dst->len, dst, ca.working_tree_encoding, conv_flags);
1454        crlf_to_git(istate, path, dst->buf, dst->len, dst, ca.crlf_action, conv_flags);
1455        ident_to_git(dst->buf, dst->len, dst, ca.ident);
1456}
1457
1458static int convert_to_working_tree_internal(const struct index_state *istate,
1459                                            const char *path, const char *src,
1460                                            size_t len, struct strbuf *dst,
1461                                            int normalizing, struct delayed_checkout *dco)
1462{
1463        int ret = 0, ret_filter = 0;
1464        struct conv_attrs ca;
1465
1466        convert_attrs(istate, &ca, path);
1467
1468        ret |= ident_to_worktree(src, len, dst, ca.ident);
1469        if (ret) {
1470                src = dst->buf;
1471                len = dst->len;
1472        }
1473        /*
1474         * CRLF conversion can be skipped if normalizing, unless there
1475         * is a smudge or process filter (even if the process filter doesn't
1476         * support smudge).  The filters might expect CRLFs.
1477         */
1478        if ((ca.drv && (ca.drv->smudge || ca.drv->process)) || !normalizing) {
1479                ret |= crlf_to_worktree(src, len, dst, ca.crlf_action);
1480                if (ret) {
1481                        src = dst->buf;
1482                        len = dst->len;
1483                }
1484        }
1485
1486        ret |= encode_to_worktree(path, src, len, dst, ca.working_tree_encoding);
1487        if (ret) {
1488                src = dst->buf;
1489                len = dst->len;
1490        }
1491
1492        ret_filter = apply_filter(
1493                path, src, len, -1, dst, ca.drv, CAP_SMUDGE, dco);
1494        if (!ret_filter && ca.drv && ca.drv->required)
1495                die(_("%s: smudge filter %s failed"), path, ca.drv->name);
1496
1497        return ret | ret_filter;
1498}
1499
1500int async_convert_to_working_tree(const struct index_state *istate,
1501                                  const char *path, const char *src,
1502                                  size_t len, struct strbuf *dst,
1503                                  void *dco)
1504{
1505        return convert_to_working_tree_internal(istate, path, src, len, dst, 0, dco);
1506}
1507
1508int convert_to_working_tree(const struct index_state *istate,
1509                            const char *path, const char *src,
1510                            size_t len, struct strbuf *dst)
1511{
1512        return convert_to_working_tree_internal(istate, path, src, len, dst, 0, NULL);
1513}
1514
1515int renormalize_buffer(const struct index_state *istate, const char *path,
1516                       const char *src, size_t len, struct strbuf *dst)
1517{
1518        int ret = convert_to_working_tree_internal(istate, path, src, len, dst, 1, NULL);
1519        if (ret) {
1520                src = dst->buf;
1521                len = dst->len;
1522        }
1523        return ret | convert_to_git(istate, path, src, len, dst, CONV_EOL_RENORMALIZE);
1524}
1525
1526/*****************************************************************
1527 *
1528 * Streaming conversion support
1529 *
1530 *****************************************************************/
1531
1532typedef int (*filter_fn)(struct stream_filter *,
1533                         const char *input, size_t *isize_p,
1534                         char *output, size_t *osize_p);
1535typedef void (*free_fn)(struct stream_filter *);
1536
1537struct stream_filter_vtbl {
1538        filter_fn filter;
1539        free_fn free;
1540};
1541
1542struct stream_filter {
1543        struct stream_filter_vtbl *vtbl;
1544};
1545
1546static int null_filter_fn(struct stream_filter *filter,
1547                          const char *input, size_t *isize_p,
1548                          char *output, size_t *osize_p)
1549{
1550        size_t count;
1551
1552        if (!input)
1553                return 0; /* we do not keep any states */
1554        count = *isize_p;
1555        if (*osize_p < count)
1556                count = *osize_p;
1557        if (count) {
1558                memmove(output, input, count);
1559                *isize_p -= count;
1560                *osize_p -= count;
1561        }
1562        return 0;
1563}
1564
1565static void null_free_fn(struct stream_filter *filter)
1566{
1567        ; /* nothing -- null instances are shared */
1568}
1569
1570static struct stream_filter_vtbl null_vtbl = {
1571        null_filter_fn,
1572        null_free_fn,
1573};
1574
1575static struct stream_filter null_filter_singleton = {
1576        &null_vtbl,
1577};
1578
1579int is_null_stream_filter(struct stream_filter *filter)
1580{
1581        return filter == &null_filter_singleton;
1582}
1583
1584
1585/*
1586 * LF-to-CRLF filter
1587 */
1588
1589struct lf_to_crlf_filter {
1590        struct stream_filter filter;
1591        unsigned has_held:1;
1592        char held;
1593};
1594
1595static int lf_to_crlf_filter_fn(struct stream_filter *filter,
1596                                const char *input, size_t *isize_p,
1597                                char *output, size_t *osize_p)
1598{
1599        size_t count, o = 0;
1600        struct lf_to_crlf_filter *lf_to_crlf = (struct lf_to_crlf_filter *)filter;
1601
1602        /*
1603         * We may be holding onto the CR to see if it is followed by a
1604         * LF, in which case we would need to go to the main loop.
1605         * Otherwise, just emit it to the output stream.
1606         */
1607        if (lf_to_crlf->has_held && (lf_to_crlf->held != '\r' || !input)) {
1608                output[o++] = lf_to_crlf->held;
1609                lf_to_crlf->has_held = 0;
1610        }
1611
1612        /* We are told to drain */
1613        if (!input) {
1614                *osize_p -= o;
1615                return 0;
1616        }
1617
1618        count = *isize_p;
1619        if (count || lf_to_crlf->has_held) {
1620                size_t i;
1621                int was_cr = 0;
1622
1623                if (lf_to_crlf->has_held) {
1624                        was_cr = 1;
1625                        lf_to_crlf->has_held = 0;
1626                }
1627
1628                for (i = 0; o < *osize_p && i < count; i++) {
1629                        char ch = input[i];
1630
1631                        if (ch == '\n') {
1632                                output[o++] = '\r';
1633                        } else if (was_cr) {
1634                                /*
1635                                 * Previous round saw CR and it is not followed
1636                                 * by a LF; emit the CR before processing the
1637                                 * current character.
1638                                 */
1639                                output[o++] = '\r';
1640                        }
1641
1642                        /*
1643                         * We may have consumed the last output slot,
1644                         * in which case we need to break out of this
1645                         * loop; hold the current character before
1646                         * returning.
1647                         */
1648                        if (*osize_p <= o) {
1649                                lf_to_crlf->has_held = 1;
1650                                lf_to_crlf->held = ch;
1651                                continue; /* break but increment i */
1652                        }
1653
1654                        if (ch == '\r') {
1655                                was_cr = 1;
1656                                continue;
1657                        }
1658
1659                        was_cr = 0;
1660                        output[o++] = ch;
1661                }
1662
1663                *osize_p -= o;
1664                *isize_p -= i;
1665
1666                if (!lf_to_crlf->has_held && was_cr) {
1667                        lf_to_crlf->has_held = 1;
1668                        lf_to_crlf->held = '\r';
1669                }
1670        }
1671        return 0;
1672}
1673
1674static void lf_to_crlf_free_fn(struct stream_filter *filter)
1675{
1676        free(filter);
1677}
1678
1679static struct stream_filter_vtbl lf_to_crlf_vtbl = {
1680        lf_to_crlf_filter_fn,
1681        lf_to_crlf_free_fn,
1682};
1683
1684static struct stream_filter *lf_to_crlf_filter(void)
1685{
1686        struct lf_to_crlf_filter *lf_to_crlf = xcalloc(1, sizeof(*lf_to_crlf));
1687
1688        lf_to_crlf->filter.vtbl = &lf_to_crlf_vtbl;
1689        return (struct stream_filter *)lf_to_crlf;
1690}
1691
1692/*
1693 * Cascade filter
1694 */
1695#define FILTER_BUFFER 1024
1696struct cascade_filter {
1697        struct stream_filter filter;
1698        struct stream_filter *one;
1699        struct stream_filter *two;
1700        char buf[FILTER_BUFFER];
1701        int end, ptr;
1702};
1703
1704static int cascade_filter_fn(struct stream_filter *filter,
1705                             const char *input, size_t *isize_p,
1706                             char *output, size_t *osize_p)
1707{
1708        struct cascade_filter *cas = (struct cascade_filter *) filter;
1709        size_t filled = 0;
1710        size_t sz = *osize_p;
1711        size_t to_feed, remaining;
1712
1713        /*
1714         * input -- (one) --> buf -- (two) --> output
1715         */
1716        while (filled < sz) {
1717                remaining = sz - filled;
1718
1719                /* do we already have something to feed two with? */
1720                if (cas->ptr < cas->end) {
1721                        to_feed = cas->end - cas->ptr;
1722                        if (stream_filter(cas->two,
1723                                          cas->buf + cas->ptr, &to_feed,
1724                                          output + filled, &remaining))
1725                                return -1;
1726                        cas->ptr += (cas->end - cas->ptr) - to_feed;
1727                        filled = sz - remaining;
1728                        continue;
1729                }
1730
1731                /* feed one from upstream and have it emit into our buffer */
1732                to_feed = input ? *isize_p : 0;
1733                if (input && !to_feed)
1734                        break;
1735                remaining = sizeof(cas->buf);
1736                if (stream_filter(cas->one,
1737                                  input, &to_feed,
1738                                  cas->buf, &remaining))
1739                        return -1;
1740                cas->end = sizeof(cas->buf) - remaining;
1741                cas->ptr = 0;
1742                if (input) {
1743                        size_t fed = *isize_p - to_feed;
1744                        *isize_p -= fed;
1745                        input += fed;
1746                }
1747
1748                /* do we know that we drained one completely? */
1749                if (input || cas->end)
1750                        continue;
1751
1752                /* tell two to drain; we have nothing more to give it */
1753                to_feed = 0;
1754                remaining = sz - filled;
1755                if (stream_filter(cas->two,
1756                                  NULL, &to_feed,
1757                                  output + filled, &remaining))
1758                        return -1;
1759                if (remaining == (sz - filled))
1760                        break; /* completely drained two */
1761                filled = sz - remaining;
1762        }
1763        *osize_p -= filled;
1764        return 0;
1765}
1766
1767static void cascade_free_fn(struct stream_filter *filter)
1768{
1769        struct cascade_filter *cas = (struct cascade_filter *)filter;
1770        free_stream_filter(cas->one);
1771        free_stream_filter(cas->two);
1772        free(filter);
1773}
1774
1775static struct stream_filter_vtbl cascade_vtbl = {
1776        cascade_filter_fn,
1777        cascade_free_fn,
1778};
1779
1780static struct stream_filter *cascade_filter(struct stream_filter *one,
1781                                            struct stream_filter *two)
1782{
1783        struct cascade_filter *cascade;
1784
1785        if (!one || is_null_stream_filter(one))
1786                return two;
1787        if (!two || is_null_stream_filter(two))
1788                return one;
1789
1790        cascade = xmalloc(sizeof(*cascade));
1791        cascade->one = one;
1792        cascade->two = two;
1793        cascade->end = cascade->ptr = 0;
1794        cascade->filter.vtbl = &cascade_vtbl;
1795        return (struct stream_filter *)cascade;
1796}
1797
1798/*
1799 * ident filter
1800 */
1801#define IDENT_DRAINING (-1)
1802#define IDENT_SKIPPING (-2)
1803struct ident_filter {
1804        struct stream_filter filter;
1805        struct strbuf left;
1806        int state;
1807        char ident[GIT_MAX_HEXSZ + 5]; /* ": x40 $" */
1808};
1809
1810static int is_foreign_ident(const char *str)
1811{
1812        int i;
1813
1814        if (!skip_prefix(str, "$Id: ", &str))
1815                return 0;
1816        for (i = 0; str[i]; i++) {
1817                if (isspace(str[i]) && str[i+1] != '$')
1818                        return 1;
1819        }
1820        return 0;
1821}
1822
1823static void ident_drain(struct ident_filter *ident, char **output_p, size_t *osize_p)
1824{
1825        size_t to_drain = ident->left.len;
1826
1827        if (*osize_p < to_drain)
1828                to_drain = *osize_p;
1829        if (to_drain) {
1830                memcpy(*output_p, ident->left.buf, to_drain);
1831                strbuf_remove(&ident->left, 0, to_drain);
1832                *output_p += to_drain;
1833                *osize_p -= to_drain;
1834        }
1835        if (!ident->left.len)
1836                ident->state = 0;
1837}
1838
1839static int ident_filter_fn(struct stream_filter *filter,
1840                           const char *input, size_t *isize_p,
1841                           char *output, size_t *osize_p)
1842{
1843        struct ident_filter *ident = (struct ident_filter *)filter;
1844        static const char head[] = "$Id";
1845
1846        if (!input) {
1847                /* drain upon eof */
1848                switch (ident->state) {
1849                default:
1850                        strbuf_add(&ident->left, head, ident->state);
1851                        /* fallthrough */
1852                case IDENT_SKIPPING:
1853                        /* fallthrough */
1854                case IDENT_DRAINING:
1855                        ident_drain(ident, &output, osize_p);
1856                }
1857                return 0;
1858        }
1859
1860        while (*isize_p || (ident->state == IDENT_DRAINING)) {
1861                int ch;
1862
1863                if (ident->state == IDENT_DRAINING) {
1864                        ident_drain(ident, &output, osize_p);
1865                        if (!*osize_p)
1866                                break;
1867                        continue;
1868                }
1869
1870                ch = *(input++);
1871                (*isize_p)--;
1872
1873                if (ident->state == IDENT_SKIPPING) {
1874                        /*
1875                         * Skipping until '$' or LF, but keeping them
1876                         * in case it is a foreign ident.
1877                         */
1878                        strbuf_addch(&ident->left, ch);
1879                        if (ch != '\n' && ch != '$')
1880                                continue;
1881                        if (ch == '$' && !is_foreign_ident(ident->left.buf)) {
1882                                strbuf_setlen(&ident->left, sizeof(head) - 1);
1883                                strbuf_addstr(&ident->left, ident->ident);
1884                        }
1885                        ident->state = IDENT_DRAINING;
1886                        continue;
1887                }
1888
1889                if (ident->state < sizeof(head) &&
1890                    head[ident->state] == ch) {
1891                        ident->state++;
1892                        continue;
1893                }
1894
1895                if (ident->state)
1896                        strbuf_add(&ident->left, head, ident->state);
1897                if (ident->state == sizeof(head) - 1) {
1898                        if (ch != ':' && ch != '$') {
1899                                strbuf_addch(&ident->left, ch);
1900                                ident->state = 0;
1901                                continue;
1902                        }
1903
1904                        if (ch == ':') {
1905                                strbuf_addch(&ident->left, ch);
1906                                ident->state = IDENT_SKIPPING;
1907                        } else {
1908                                strbuf_addstr(&ident->left, ident->ident);
1909                                ident->state = IDENT_DRAINING;
1910                        }
1911                        continue;
1912                }
1913
1914                strbuf_addch(&ident->left, ch);
1915                ident->state = IDENT_DRAINING;
1916        }
1917        return 0;
1918}
1919
1920static void ident_free_fn(struct stream_filter *filter)
1921{
1922        struct ident_filter *ident = (struct ident_filter *)filter;
1923        strbuf_release(&ident->left);
1924        free(filter);
1925}
1926
1927static struct stream_filter_vtbl ident_vtbl = {
1928        ident_filter_fn,
1929        ident_free_fn,
1930};
1931
1932static struct stream_filter *ident_filter(const struct object_id *oid)
1933{
1934        struct ident_filter *ident = xmalloc(sizeof(*ident));
1935
1936        xsnprintf(ident->ident, sizeof(ident->ident),
1937                  ": %s $", oid_to_hex(oid));
1938        strbuf_init(&ident->left, 0);
1939        ident->filter.vtbl = &ident_vtbl;
1940        ident->state = 0;
1941        return (struct stream_filter *)ident;
1942}
1943
1944/*
1945 * Return an appropriately constructed filter for the path, or NULL if
1946 * the contents cannot be filtered without reading the whole thing
1947 * in-core.
1948 *
1949 * Note that you would be crazy to set CRLF, smuge/clean or ident to a
1950 * large binary blob you would want us not to slurp into the memory!
1951 */
1952struct stream_filter *get_stream_filter(const struct index_state *istate,
1953                                        const char *path,
1954                                        const struct object_id *oid)
1955{
1956        struct conv_attrs ca;
1957        struct stream_filter *filter = NULL;
1958
1959        convert_attrs(istate, &ca, path);
1960        if (ca.drv && (ca.drv->process || ca.drv->smudge || ca.drv->clean))
1961                return NULL;
1962
1963        if (ca.working_tree_encoding)
1964                return NULL;
1965
1966        if (ca.crlf_action == CRLF_AUTO || ca.crlf_action == CRLF_AUTO_CRLF)
1967                return NULL;
1968
1969        if (ca.ident)
1970                filter = ident_filter(oid);
1971
1972        if (output_eol(ca.crlf_action) == EOL_CRLF)
1973                filter = cascade_filter(filter, lf_to_crlf_filter());
1974        else
1975                filter = cascade_filter(filter, &null_filter_singleton);
1976
1977        return filter;
1978}
1979
1980void free_stream_filter(struct stream_filter *filter)
1981{
1982        filter->vtbl->free(filter);
1983}
1984
1985int stream_filter(struct stream_filter *filter,
1986                  const char *input, size_t *isize_p,
1987                  char *output, size_t *osize_p)
1988{
1989        return filter->vtbl->filter(filter, input, isize_p, output, osize_p);
1990}