Merge branches 'jc/convert', 'jc/bigfile' and 'jc/replacing' into jc/streaming

* jc/convert:
convert: make it harder to screw up adding a conversion attribute
convert: make it safer to add conversion attributes
convert: give saner names to crlf/eol variables, types and functions
convert: rename the "eol" global variable to "core_eol"

* jc/bigfile:
Bigfile: teach "git add" to send a large file straight to a pack
index_fd(): split into two helper functions
index_fd(): turn write_object and format_check arguments into one flag

* jc/replacing:
read_sha1_file(): allow selective bypassing of replacement mechanism
inline lookup_replace_object() calls
read_sha1_file(): get rid of read_sha1_file_repl() madness
t6050: make sure we test not just commit replacement
Declare lookup_replace_object() in cache.h, not in commit.h

diff --combined cache.h
index 2b341166241c7be98fb1012871266ebb910e1226,4e9123b77bcc9eea114edf3db274e3e1a58f03e7,bc380139590700e88d436e8577d3b433aa4e5d7b,5f1f5c33953510bbdc7d31139b5e33bd1aaa7a4c..b1b5bb5896bb9894cd38d90cbe1d1bbd3752e5a4
--- 1/cache.h
--- 2/cache.h
--- 3/cache.h
--- 4/cache.h
+++ b/cache.h
@@@@@ -511,15 -511,15 -511,18 -511,15 +511,18 @@@@@ struct pathspec 
        struct pathspec_item {
                const char *match;
                int len;
  -             unsigned int has_wildcard:1;
  +             unsigned int use_wildcard:1;
        } *items;
    };
    
    extern int init_pathspec(struct pathspec *, const char **);
    extern void free_pathspec(struct pathspec *);
    extern int ce_path_match(const struct cache_entry *ce, const struct pathspec *pathspec);
-- -extern int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, enum object_type type, const char *path, int format_check);
-- -extern int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object);
++ +
++ +#define HASH_WRITE_OBJECT 1
++ +#define HASH_FORMAT_CHECK 2
++ +extern int index_fd(unsigned char *sha1, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags);
++ +extern int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags);
    extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st);
    
    #define REFRESH_REALLY              0x0001  /* ignore_valid */
@@@@@ -606,7 -606,7 -609,7 -606,7 +609,7 @@@@@ enum eol 
    #endif
    };
    
- --extern enum eol eol;
+ ++extern enum eol core_eol;
    
    enum branch_track {
        BRANCH_TRACK_UNSPECIFIED = -1,
@@@@@ -676,24 -676,24 -679,14 -676,24 +679,24 @@@@@ extern char *sha1_pack_name(const unsig
    extern char *sha1_pack_index_name(const unsigned char *sha1);
    extern const char *find_unique_abbrev(const unsigned char *sha1, int);
    extern const unsigned char null_sha1[20];
  - static inline int is_null_sha1(const unsigned char *sha1)
  + 
  + static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2)
    {
  -     return !memcmp(sha1, null_sha1, 20);
  +     int i;
  + 
  +     for (i = 0; i < 20; i++, sha1++, sha2++) {
  +             if (*sha1 != *sha2)
  +                     return *sha1 - *sha2;
  +     }
  + 
  +     return 0;
    }
  - static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2)
  + 
  + static inline int is_null_sha1(const unsigned char *sha1)
    {
  -     return memcmp(sha1, sha2, 20);
  +     return !hashcmp(sha1, null_sha1);
    }
  + 
    static inline void hashcpy(unsigned char *sha_dst, const unsigned char *sha_src)
    {
        memcpy(sha_dst, sha_src, 20);
@@@@@ -756,13 -756,13 -749,13 -756,23 +759,23 @@@@@ char *strip_path_suffix(const char *pat
    int daemon_avoid_alias(const char *path);
    int offset_1st_component(const char *path);
    
--- /* Read and unpack a sha1 file into memory, write memory to a sha1 file */
--- extern int sha1_object_info(const unsigned char *, unsigned long *);
--- extern void *read_sha1_file_repl(const unsigned char *sha1, enum object_type *type, unsigned long *size, const unsigned char **replacement);
+++ /* object replacement */
+++ #define READ_SHA1_FILE_REPLACE 1
+++ extern void *read_sha1_file_extended(const unsigned char *sha1, enum object_type *type, unsigned long *size, unsigned flag);
    static inline void *read_sha1_file(const unsigned char *sha1, enum object_type *type, unsigned long *size)
    {
---     return read_sha1_file_repl(sha1, type, size, NULL);
+++     return read_sha1_file_extended(sha1, type, size, READ_SHA1_FILE_REPLACE);
+++ }
+++ extern const unsigned char *do_lookup_replace_object(const unsigned char *sha1);
+++ static inline const unsigned char *lookup_replace_object(const unsigned char *sha1)
+++ {
+++     if (!read_replace_refs)
+++             return sha1;
+++     return do_lookup_replace_object(sha1);
    }
+++ 
+++ /* Read and unpack a sha1 file into memory, write memory to a sha1 file */
+++ extern int sha1_object_info(const unsigned char *, unsigned long *);
    extern int hash_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1);
    extern int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *return_sha1);
    extern int pretend_sha1_file(void *, unsigned long, enum object_type, unsigned char *);

diff --combined environment.c
index 40185bc854ea2c5b8d2e3deb800dd6f3f44482a9,7fe9f101243d9063ff06fe2c6ea10f220b6a2549,40185bc854ea2c5b8d2e3deb800dd6f3f44482a9,91828201d87daf4c1035ac27bc6e9f07c01c8d90..94d58fd24413ec1d1a5769029bd5149b609f0d4e
--- 1/environment.c
--- 2/environment.c
--- 3/environment.c
--- 4/environment.c
+++ b/environment.c
@@@@@ -42,8 -42,8 -42,8 -42,8 +42,8 @@@@@ const char *editor_program
    const char *askpass_program;
    const char *excludes_file;
    enum auto_crlf auto_crlf = AUTO_CRLF_FALSE;
--- int read_replace_refs = 1;
- - enum eol eol = EOL_UNSET;
+++ int read_replace_refs = 1; /* NEEDSWORK: rename to use_replace_refs */
   -enum eol eol = EOL_UNSET;
+ ++enum eol core_eol = EOL_UNSET;
    enum safe_crlf safe_crlf = SAFE_CRLF_WARN;
    unsigned whitespace_rule_cfg = WS_DEFAULT_RULE;
    enum branch_track git_branch_track = BRANCH_TRACK_REMOTE;

diff --combined sha1_file.c
index 1a7e41070efa6327859d3c09ede07317cc91ba7d,889fe7183065ae8bc12821aadebb69a17bc7635c,f0ca6a1749e2517bf3640f9d9f17e866c4eed47f,7e6e976c23024c575a1eba73ed8dd70424ec09ff..064a33040812ba8782bf602c693abf08613d6ec7
--- 1/sha1_file.c
--- 2/sha1_file.c
--- 3/sha1_file.c
--- 4/sha1_file.c
+++ b/sha1_file.c
@@@@@ -11,6 -11,6 -11,7 -11,6 +11,7 @@@@@
    #include "pack.h"
    #include "blob.h"
    #include "commit.h"
++ +#include "run-command.h"
    #include "tag.h"
    #include "tree.h"
    #include "tree-walk.h"
@@@@@ -31,6 -31,8 -32,8 -31,8 +32,6 @@@@@ static inline uintmax_t sz_fmt(size_t s
    
    const unsigned char null_sha1[20];
    
 ---static int git_open_noatime(const char *name, struct packed_git *p);
 ---
    /*
     * This is meant to hold a *small* number of objects that you would
     * want read_sha1_file() to be able to return, but yet you do not want
@@@@@ -225,7 -227,6 -228,6 -227,6 +226,7 @@@@@ struct alternate_object_database *alt_o
    static struct alternate_object_database **alt_odb_tail;
    
    static void read_info_alternates(const char * alternates, int depth);
 +++static int git_open_noatime(const char *name);
    
    /*
     * Prepare alternate object database registry.
@@@@@ -359,7 -360,7 -361,7 -360,7 +360,7 @@@@@ static void read_info_alternates(const 
        int fd;
    
        sprintf(path, "%s/%s", relative_base, alt_file_name);
 ---    fd = git_open_noatime(path, NULL);
 +++    fd = git_open_noatime(path);
        if (fd < 0)
                return;
        if (fstat(fd, &st) || (st.st_size == 0)) {
@@@@@ -474,7 -475,7 -476,7 -475,7 +475,7 @@@@@ static int check_packed_git_idx(const c
        struct pack_idx_header *hdr;
        size_t idx_size;
        uint32_t version, nr, i, *index;
 ---    int fd = git_open_noatime(path, p);
 +++    int fd = git_open_noatime(path);
        struct stat st;
    
        if (fd < 0)
@@@@@ -756,7 -757,7 -758,7 -757,7 +757,7 @@@@@ static int open_packed_git_1(struct pac
        while (pack_max_fds <= pack_open_fds && unuse_one_window(NULL, -1))
                ; /* nothing */
    
 ---    p->pack_fd = git_open_noatime(p->pack_name, p);
 +++    p->pack_fd = git_open_noatime(p->pack_name);
        if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
                return -1;
        pack_open_fds++;
@@@@@ -1144,7 -1145,7 -1146,7 -1145,7 +1145,7 @@@@@ int check_sha1_signature(const unsigne
        return hashcmp(sha1, real_sha1) ? -1 : 0;
    }
    
 ---static int git_open_noatime(const char *name, struct packed_git *p)
 +++static int git_open_noatime(const char *name)
    {
        static int sha1_file_open_flag = O_NOATIME;
    
@@@@@ -1169,7 -1170,7 -1171,7 -1170,7 +1170,7 @@@@@ static int open_sha1_file(const unsigne
        char *name = sha1_file_name(sha1);
        struct alternate_object_database *alt;
    
 ---    fd = git_open_noatime(name, NULL);
 +++    fd = git_open_noatime(name);
        if (fd >= 0)
                return fd;
    
@@@@@ -1178,7 -1179,7 -1180,7 -1179,7 +1179,7 @@@@@
        for (alt = alt_odb_list; alt; alt = alt->next) {
                name = alt->name;
                fill_sha1_path(name, sha1);
 ---            fd = git_open_noatime(alt->base, NULL);
 +++            fd = git_open_noatime(alt->base);
                if (fd >= 0)
                        return fd;
        }
@@@@@ -1307,7 -1308,7 -1309,7 -1308,7 +1308,7 @@@@@ static void *unpack_sha1_rest(z_stream 
                /*
                 * The above condition must be (bytes <= size), not
                 * (bytes < size).  In other words, even though we
 ---             * expect no more output and set avail_out to zer0,
 +++             * expect no more output and set avail_out to zero,
                 * the input zlib stream may have bytes that express
                 * "this concludes the stream", and we *do* want to
                 * eat that input.
@@@@@ -2205,23 -2206,23 -2207,23 -2206,21 +2206,21 @@@@@ static void *read_object(const unsigne
     * deal with them should arrange to call read_object() and give error
     * messages themselves.
     */
--- void *read_sha1_file_repl(const unsigned char *sha1,
---                       enum object_type *type,
---                       unsigned long *size,
---                       const unsigned char **replacement)
+++ void *read_sha1_file_extended(const unsigned char *sha1,
+++                           enum object_type *type,
+++                           unsigned long *size,
+++                           unsigned flag)
    {
---     const unsigned char *repl = lookup_replace_object(sha1);
        void *data;
        char *path;
        const struct packed_git *p;
+++     const unsigned char *repl = (flag & READ_SHA1_FILE_REPLACE)
+++             ? lookup_replace_object(sha1) : sha1;
    
        errno = 0;
        data = read_object(repl, type, size);
---     if (data) {
---             if (replacement)
---                     *replacement = repl;
+++     if (data)
                return data;
---     }
    
        if (errno && errno != ENOENT)
                die_errno("failed to read object %s", sha1_to_hex(sha1));
@@@@@ -2580,10 -2581,10 -2582,11 -2579,10 +2579,11 @@@@@ static void check_tag(const void *buf, 
    }
    
    static int index_mem(unsigned char *sha1, void *buf, size_t size,
-- -                 int write_object, enum object_type type,
-- -                 const char *path, int format_check)
++ +                 enum object_type type,
++ +                 const char *path, unsigned flags)
    {
        int ret, re_allocated = 0;
++ +    int write_object = flags & HASH_WRITE_OBJECT;
    
        if (!type)
                type = OBJ_BLOB;
@@@@@ -2599,7 -2600,7 -2602,7 -2598,7 +2599,7 @@@@@
                        re_allocated = 1;
                }
        }
-- -    if (format_check) {
++ +    if (flags & HASH_FORMAT_CHECK) {
                if (type == OBJ_TREE)
                        check_tree(buf, size);
                if (type == OBJ_COMMIT)
@@@@@ -2617,44 -2618,44 -2620,141 -2616,44 +2617,141 @@@@@
        return ret;
    }
    
++ +static int index_pipe(unsigned char *sha1, int fd, enum object_type type,
++ +                  const char *path, unsigned flags)
++ +{
++ +    struct strbuf sbuf = STRBUF_INIT;
++ +    int ret;
++ +
++ +    if (strbuf_read(&sbuf, fd, 4096) >= 0)
++ +            ret = index_mem(sha1, sbuf.buf, sbuf.len, type, path, flags);
++ +    else
++ +            ret = -1;
++ +    strbuf_release(&sbuf);
++ +    return ret;
++ +}
++ +
    #define SMALL_FILE_SIZE (32*1024)
    
-- -int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object,
-- -         enum object_type type, const char *path, int format_check)
++ +static int index_core(unsigned char *sha1, int fd, size_t size,
++ +                  enum object_type type, const char *path,
++ +                  unsigned flags)
    {
        int ret;
-- -    size_t size = xsize_t(st->st_size);
    
-- -    if (!S_ISREG(st->st_mode)) {
-- -            struct strbuf sbuf = STRBUF_INIT;
-- -            if (strbuf_read(&sbuf, fd, 4096) >= 0)
-- -                    ret = index_mem(sha1, sbuf.buf, sbuf.len, write_object,
-- -                                    type, path, format_check);
-- -            else
-- -                    ret = -1;
-- -            strbuf_release(&sbuf);
-- -    } else if (!size) {
-- -            ret = index_mem(sha1, NULL, size, write_object, type, path,
-- -                            format_check);
++ +    if (!size) {
++ +            ret = index_mem(sha1, NULL, size, type, path, flags);
        } else if (size <= SMALL_FILE_SIZE) {
                char *buf = xmalloc(size);
                if (size == read_in_full(fd, buf, size))
-- -                    ret = index_mem(sha1, buf, size, write_object, type,
-- -                                    path, format_check);
++ +                    ret = index_mem(sha1, buf, size, type, path, flags);
                else
                        ret = error("short read %s", strerror(errno));
                free(buf);
        } else {
                void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
-- -            ret = index_mem(sha1, buf, size, write_object, type, path,
-- -                            format_check);
++ +            ret = index_mem(sha1, buf, size, type, path, flags);
                munmap(buf, size);
        }
++ +    return ret;
++ +}
++ +
++ +/*
++ + * This creates one packfile per large blob, because the caller
++ + * immediately wants the result sha1, and fast-import can report the
++ + * object name via marks mechanism only by closing the created
++ + * packfile.
++ + *
++ + * This also bypasses the usual "convert-to-git" dance, and that is on
++ + * purpose. We could write a streaming version of the converting
++ + * functions and insert that before feeding the data to fast-import
++ + * (or equivalent in-core API described above), but the primary
++ + * motivation for trying to stream from the working tree file and to
++ + * avoid mmaping it in core is to deal with large binary blobs, and
++ + * by definition they do _not_ want to get any conversion.
++ + */
++ +static int index_stream(unsigned char *sha1, int fd, size_t size,
++ +                    enum object_type type, const char *path,
++ +                    unsigned flags)
++ +{
++ +    struct child_process fast_import;
++ +    char export_marks[512];
++ +    const char *argv[] = { "fast-import", "--quiet", export_marks, NULL };
++ +    char tmpfile[512];
++ +    char fast_import_cmd[512];
++ +    char buf[512];
++ +    int len, tmpfd;
++ +
++ +    strcpy(tmpfile, git_path("hashstream_XXXXXX"));
++ +    tmpfd = git_mkstemp_mode(tmpfile, 0600);
++ +    if (tmpfd < 0)
++ +            die_errno("cannot create tempfile: %s", tmpfile);
++ +    if (close(tmpfd))
++ +            die_errno("cannot close tempfile: %s", tmpfile);
++ +    sprintf(export_marks, "--export-marks=%s", tmpfile);
++ +
++ +    memset(&fast_import, 0, sizeof(fast_import));
++ +    fast_import.in = -1;
++ +    fast_import.argv = argv;
++ +    fast_import.git_cmd = 1;
++ +    if (start_command(&fast_import))
++ +            die_errno("index-stream: git fast-import failed");
++ +
++ +    len = sprintf(fast_import_cmd, "blob\nmark :1\ndata %lu\n",
++ +                  (unsigned long) size);
++ +    write_or_whine(fast_import.in, fast_import_cmd, len,
++ +                   "index-stream: feeding fast-import");
++ +    while (size) {
++ +            char buf[10240];
++ +            size_t sz = size < sizeof(buf) ? size : sizeof(buf);
++ +            size_t actual;
++ +
++ +            actual = read_in_full(fd, buf, sz);
++ +            if (actual < 0)
++ +                    die_errno("index-stream: reading input");
++ +            if (write_in_full(fast_import.in, buf, actual) != actual)
++ +                    die_errno("index-stream: feeding fast-import");
++ +            size -= actual;
++ +    }
++ +    if (close(fast_import.in))
++ +            die_errno("index-stream: closing fast-import");
++ +    if (finish_command(&fast_import))
++ +            die_errno("index-stream: finishing fast-import");
++ +
++ +    tmpfd = open(tmpfile, O_RDONLY);
++ +    if (tmpfd < 0)
++ +            die_errno("index-stream: cannot open fast-import mark");
++ +    len = read(tmpfd, buf, sizeof(buf));
++ +    if (len < 0)
++ +            die_errno("index-stream: reading fast-import mark");
++ +    if (close(tmpfd) < 0)
++ +            die_errno("index-stream: closing fast-import mark");
++ +    if (unlink(tmpfile))
++ +            die_errno("index-stream: unlinking fast-import mark");
++ +    if (len != 44 ||
++ +        memcmp(":1 ", buf, 3) ||
++ +        get_sha1_hex(buf + 3, sha1))
++ +            die_errno("index-stream: unexpected fast-import mark: <%s>", buf);
++ +    return 0;
++ +}
++ +
++ +int index_fd(unsigned char *sha1, int fd, struct stat *st,
++ +         enum object_type type, const char *path, unsigned flags)
++ +{
++ +    int ret;
++ +    size_t size = xsize_t(st->st_size);
++ +
++ +    if (!S_ISREG(st->st_mode))
++ +            ret = index_pipe(sha1, fd, type, path, flags);
++ +    else if (size <= big_file_threshold || type != OBJ_BLOB)
++ +            ret = index_core(sha1, fd, size, type, path, flags);
++ +    else
++ +            ret = index_stream(sha1, fd, size, type, path, flags);
        close(fd);
        return ret;
    }
    
-- -int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object)
++ +int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags)
    {
        int fd;
        struct strbuf sb = STRBUF_INIT;
@@@@@ -2665,7 -2666,7 -2765,7 -2664,7 +2762,7 @@@@@
                if (fd < 0)
                        return error("open(\"%s\"): %s", path,
                                     strerror(errno));
-- -            if (index_fd(sha1, fd, st, write_object, OBJ_BLOB, path, 0) < 0)
++ +            if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0)
                        return error("%s: failed to insert into database",
                                     path);
                break;
@@@@@ -2675,7 -2676,7 -2775,7 -2674,7 +2772,7 @@@@@
                        return error("readlink(\"%s\"): %s", path,
                                     errstr);
                }
-- -            if (!write_object)
++ +            if (!(flags & HASH_WRITE_OBJECT))
                        hash_sha1_file(sb.buf, sb.len, blob_type, sha1);
                else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1))
                        return error("%s: failed to insert into database",