From: Junio C Hamano Date: Sun, 15 May 2011 23:30:13 +0000 (-0700) Subject: Merge branches 'jc/convert', 'jc/bigfile' and 'jc/replacing' into jc/streaming X-Git-Tag: v1.7.7-rc0~115^2~13 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/02071b27f1587c65ddc4779017a3ddceb777de9c?hp=-c Merge branches 'jc/convert', 'jc/bigfile' and 'jc/replacing' into jc/streaming * jc/convert: convert: make it harder to screw up adding a conversion attribute convert: make it safer to add conversion attributes convert: give saner names to crlf/eol variables, types and functions convert: rename the "eol" global variable to "core_eol" * jc/bigfile: Bigfile: teach "git add" to send a large file straight to a pack index_fd(): split into two helper functions index_fd(): turn write_object and format_check arguments into one flag * jc/replacing: read_sha1_file(): allow selective bypassing of replacement mechanism inline lookup_replace_object() calls read_sha1_file(): get rid of read_sha1_file_repl() madness t6050: make sure we test not just commit replacement Declare lookup_replace_object() in cache.h, not in commit.h --- 02071b27f1587c65ddc4779017a3ddceb777de9c diff --combined cache.h index 2b34116624,4e9123b77b,bc38013959,5f1f5c3395..b1b5bb5896 --- a/cache.h +++ b/cache.h @@@@@ -511,15 -511,15 -511,18 -511,15 +511,18 @@@@@ struct pathspec struct pathspec_item { const char *match; int len; - unsigned int has_wildcard:1; + unsigned int use_wildcard:1; } *items; }; extern int init_pathspec(struct pathspec *, const char **); extern void free_pathspec(struct pathspec *); extern int ce_path_match(const struct cache_entry *ce, const struct pathspec *pathspec); -- -extern int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, enum object_type type, const char *path, int format_check); -- -extern int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object); ++ + ++ +#define HASH_WRITE_OBJECT 1 ++ +#define HASH_FORMAT_CHECK 2 ++ +extern int index_fd(unsigned char *sha1, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags); ++ +extern int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags); extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st); #define REFRESH_REALLY 0x0001 /* ignore_valid */ @@@@@ -606,7 -606,7 -609,7 -606,7 +609,7 @@@@@ enum eol #endif }; - --extern enum eol eol; + ++extern enum eol core_eol; enum branch_track { BRANCH_TRACK_UNSPECIFIED = -1, @@@@@ -676,24 -676,24 -679,14 -676,24 +679,24 @@@@@ extern char *sha1_pack_name(const unsig extern char *sha1_pack_index_name(const unsigned char *sha1); extern const char *find_unique_abbrev(const unsigned char *sha1, int); extern const unsigned char null_sha1[20]; - static inline int is_null_sha1(const unsigned char *sha1) + + static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2) { - return !memcmp(sha1, null_sha1, 20); + int i; + + for (i = 0; i < 20; i++, sha1++, sha2++) { + if (*sha1 != *sha2) + return *sha1 - *sha2; + } + + return 0; } - static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2) + + static inline int is_null_sha1(const unsigned char *sha1) { - return memcmp(sha1, sha2, 20); + return !hashcmp(sha1, null_sha1); } + static inline void hashcpy(unsigned char *sha_dst, const unsigned char *sha_src) { memcpy(sha_dst, sha_src, 20); @@@@@ -756,13 -756,13 -749,13 -756,23 +759,23 @@@@@ char *strip_path_suffix(const char *pat int daemon_avoid_alias(const char *path); int offset_1st_component(const char *path); --- /* Read and unpack a sha1 file into memory, write memory to a sha1 file */ --- extern int sha1_object_info(const unsigned char *, unsigned long *); --- extern void *read_sha1_file_repl(const unsigned char *sha1, enum object_type *type, unsigned long *size, const unsigned char **replacement); +++ /* object replacement */ +++ #define READ_SHA1_FILE_REPLACE 1 +++ extern void *read_sha1_file_extended(const unsigned char *sha1, enum object_type *type, unsigned long *size, unsigned flag); static inline void *read_sha1_file(const unsigned char *sha1, enum object_type *type, unsigned long *size) { --- return read_sha1_file_repl(sha1, type, size, NULL); +++ return read_sha1_file_extended(sha1, type, size, READ_SHA1_FILE_REPLACE); +++ } +++ extern const unsigned char *do_lookup_replace_object(const unsigned char *sha1); +++ static inline const unsigned char *lookup_replace_object(const unsigned char *sha1) +++ { +++ if (!read_replace_refs) +++ return sha1; +++ return do_lookup_replace_object(sha1); } +++ +++ /* Read and unpack a sha1 file into memory, write memory to a sha1 file */ +++ extern int sha1_object_info(const unsigned char *, unsigned long *); extern int hash_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1); extern int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *return_sha1); extern int pretend_sha1_file(void *, unsigned long, enum object_type, unsigned char *); diff --combined environment.c index 40185bc854,7fe9f10124,40185bc854,91828201d8..94d58fd244 --- a/environment.c +++ b/environment.c @@@@@ -42,8 -42,8 -42,8 -42,8 +42,8 @@@@@ const char *editor_program const char *askpass_program; const char *excludes_file; enum auto_crlf auto_crlf = AUTO_CRLF_FALSE; --- int read_replace_refs = 1; - - enum eol eol = EOL_UNSET; +++ int read_replace_refs = 1; /* NEEDSWORK: rename to use_replace_refs */ -enum eol eol = EOL_UNSET; + ++enum eol core_eol = EOL_UNSET; enum safe_crlf safe_crlf = SAFE_CRLF_WARN; unsigned whitespace_rule_cfg = WS_DEFAULT_RULE; enum branch_track git_branch_track = BRANCH_TRACK_REMOTE; diff --combined sha1_file.c index 1a7e41070e,889fe71830,f0ca6a1749,7e6e976c23..064a330408 --- a/sha1_file.c +++ b/sha1_file.c @@@@@ -11,6 -11,6 -11,7 -11,6 +11,7 @@@@@ #include "pack.h" #include "blob.h" #include "commit.h" ++ +#include "run-command.h" #include "tag.h" #include "tree.h" #include "tree-walk.h" @@@@@ -31,6 -31,8 -32,8 -31,8 +32,6 @@@@@ static inline uintmax_t sz_fmt(size_t s const unsigned char null_sha1[20]; ---static int git_open_noatime(const char *name, struct packed_git *p); --- /* * This is meant to hold a *small* number of objects that you would * want read_sha1_file() to be able to return, but yet you do not want @@@@@ -225,7 -227,6 -228,6 -227,6 +226,7 @@@@@ struct alternate_object_database *alt_o static struct alternate_object_database **alt_odb_tail; static void read_info_alternates(const char * alternates, int depth); +++static int git_open_noatime(const char *name); /* * Prepare alternate object database registry. @@@@@ -359,7 -360,7 -361,7 -360,7 +360,7 @@@@@ static void read_info_alternates(const int fd; sprintf(path, "%s/%s", relative_base, alt_file_name); --- fd = git_open_noatime(path, NULL); +++ fd = git_open_noatime(path); if (fd < 0) return; if (fstat(fd, &st) || (st.st_size == 0)) { @@@@@ -474,7 -475,7 -476,7 -475,7 +475,7 @@@@@ static int check_packed_git_idx(const c struct pack_idx_header *hdr; size_t idx_size; uint32_t version, nr, i, *index; --- int fd = git_open_noatime(path, p); +++ int fd = git_open_noatime(path); struct stat st; if (fd < 0) @@@@@ -756,7 -757,7 -758,7 -757,7 +757,7 @@@@@ static int open_packed_git_1(struct pac while (pack_max_fds <= pack_open_fds && unuse_one_window(NULL, -1)) ; /* nothing */ --- p->pack_fd = git_open_noatime(p->pack_name, p); +++ p->pack_fd = git_open_noatime(p->pack_name); if (p->pack_fd < 0 || fstat(p->pack_fd, &st)) return -1; pack_open_fds++; @@@@@ -1144,7 -1145,7 -1146,7 -1145,7 +1145,7 @@@@@ int check_sha1_signature(const unsigne return hashcmp(sha1, real_sha1) ? -1 : 0; } ---static int git_open_noatime(const char *name, struct packed_git *p) +++static int git_open_noatime(const char *name) { static int sha1_file_open_flag = O_NOATIME; @@@@@ -1169,7 -1170,7 -1171,7 -1170,7 +1170,7 @@@@@ static int open_sha1_file(const unsigne char *name = sha1_file_name(sha1); struct alternate_object_database *alt; --- fd = git_open_noatime(name, NULL); +++ fd = git_open_noatime(name); if (fd >= 0) return fd; @@@@@ -1178,7 -1179,7 -1180,7 -1179,7 +1179,7 @@@@@ for (alt = alt_odb_list; alt; alt = alt->next) { name = alt->name; fill_sha1_path(name, sha1); --- fd = git_open_noatime(alt->base, NULL); +++ fd = git_open_noatime(alt->base); if (fd >= 0) return fd; } @@@@@ -1307,7 -1308,7 -1309,7 -1308,7 +1308,7 @@@@@ static void *unpack_sha1_rest(z_stream /* * The above condition must be (bytes <= size), not * (bytes < size). In other words, even though we --- * expect no more output and set avail_out to zer0, +++ * expect no more output and set avail_out to zero, * the input zlib stream may have bytes that express * "this concludes the stream", and we *do* want to * eat that input. @@@@@ -2205,23 -2206,23 -2207,23 -2206,21 +2206,21 @@@@@ static void *read_object(const unsigne * deal with them should arrange to call read_object() and give error * messages themselves. */ --- void *read_sha1_file_repl(const unsigned char *sha1, --- enum object_type *type, --- unsigned long *size, --- const unsigned char **replacement) +++ void *read_sha1_file_extended(const unsigned char *sha1, +++ enum object_type *type, +++ unsigned long *size, +++ unsigned flag) { --- const unsigned char *repl = lookup_replace_object(sha1); void *data; char *path; const struct packed_git *p; +++ const unsigned char *repl = (flag & READ_SHA1_FILE_REPLACE) +++ ? lookup_replace_object(sha1) : sha1; errno = 0; data = read_object(repl, type, size); --- if (data) { --- if (replacement) --- *replacement = repl; +++ if (data) return data; --- } if (errno && errno != ENOENT) die_errno("failed to read object %s", sha1_to_hex(sha1)); @@@@@ -2580,10 -2581,10 -2582,11 -2579,10 +2579,11 @@@@@ static void check_tag(const void *buf, } static int index_mem(unsigned char *sha1, void *buf, size_t size, -- - int write_object, enum object_type type, -- - const char *path, int format_check) ++ + enum object_type type, ++ + const char *path, unsigned flags) { int ret, re_allocated = 0; ++ + int write_object = flags & HASH_WRITE_OBJECT; if (!type) type = OBJ_BLOB; @@@@@ -2599,7 -2600,7 -2602,7 -2598,7 +2599,7 @@@@@ re_allocated = 1; } } -- - if (format_check) { ++ + if (flags & HASH_FORMAT_CHECK) { if (type == OBJ_TREE) check_tree(buf, size); if (type == OBJ_COMMIT) @@@@@ -2617,44 -2618,44 -2620,141 -2616,44 +2617,141 @@@@@ return ret; } ++ +static int index_pipe(unsigned char *sha1, int fd, enum object_type type, ++ + const char *path, unsigned flags) ++ +{ ++ + struct strbuf sbuf = STRBUF_INIT; ++ + int ret; ++ + ++ + if (strbuf_read(&sbuf, fd, 4096) >= 0) ++ + ret = index_mem(sha1, sbuf.buf, sbuf.len, type, path, flags); ++ + else ++ + ret = -1; ++ + strbuf_release(&sbuf); ++ + return ret; ++ +} ++ + #define SMALL_FILE_SIZE (32*1024) -- -int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, -- - enum object_type type, const char *path, int format_check) ++ +static int index_core(unsigned char *sha1, int fd, size_t size, ++ + enum object_type type, const char *path, ++ + unsigned flags) { int ret; -- - size_t size = xsize_t(st->st_size); -- - if (!S_ISREG(st->st_mode)) { -- - struct strbuf sbuf = STRBUF_INIT; -- - if (strbuf_read(&sbuf, fd, 4096) >= 0) -- - ret = index_mem(sha1, sbuf.buf, sbuf.len, write_object, -- - type, path, format_check); -- - else -- - ret = -1; -- - strbuf_release(&sbuf); -- - } else if (!size) { -- - ret = index_mem(sha1, NULL, size, write_object, type, path, -- - format_check); ++ + if (!size) { ++ + ret = index_mem(sha1, NULL, size, type, path, flags); } else if (size <= SMALL_FILE_SIZE) { char *buf = xmalloc(size); if (size == read_in_full(fd, buf, size)) -- - ret = index_mem(sha1, buf, size, write_object, type, -- - path, format_check); ++ + ret = index_mem(sha1, buf, size, type, path, flags); else ret = error("short read %s", strerror(errno)); free(buf); } else { void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); -- - ret = index_mem(sha1, buf, size, write_object, type, path, -- - format_check); ++ + ret = index_mem(sha1, buf, size, type, path, flags); munmap(buf, size); } ++ + return ret; ++ +} ++ + ++ +/* ++ + * This creates one packfile per large blob, because the caller ++ + * immediately wants the result sha1, and fast-import can report the ++ + * object name via marks mechanism only by closing the created ++ + * packfile. ++ + * ++ + * This also bypasses the usual "convert-to-git" dance, and that is on ++ + * purpose. We could write a streaming version of the converting ++ + * functions and insert that before feeding the data to fast-import ++ + * (or equivalent in-core API described above), but the primary ++ + * motivation for trying to stream from the working tree file and to ++ + * avoid mmaping it in core is to deal with large binary blobs, and ++ + * by definition they do _not_ want to get any conversion. ++ + */ ++ +static int index_stream(unsigned char *sha1, int fd, size_t size, ++ + enum object_type type, const char *path, ++ + unsigned flags) ++ +{ ++ + struct child_process fast_import; ++ + char export_marks[512]; ++ + const char *argv[] = { "fast-import", "--quiet", export_marks, NULL }; ++ + char tmpfile[512]; ++ + char fast_import_cmd[512]; ++ + char buf[512]; ++ + int len, tmpfd; ++ + ++ + strcpy(tmpfile, git_path("hashstream_XXXXXX")); ++ + tmpfd = git_mkstemp_mode(tmpfile, 0600); ++ + if (tmpfd < 0) ++ + die_errno("cannot create tempfile: %s", tmpfile); ++ + if (close(tmpfd)) ++ + die_errno("cannot close tempfile: %s", tmpfile); ++ + sprintf(export_marks, "--export-marks=%s", tmpfile); ++ + ++ + memset(&fast_import, 0, sizeof(fast_import)); ++ + fast_import.in = -1; ++ + fast_import.argv = argv; ++ + fast_import.git_cmd = 1; ++ + if (start_command(&fast_import)) ++ + die_errno("index-stream: git fast-import failed"); ++ + ++ + len = sprintf(fast_import_cmd, "blob\nmark :1\ndata %lu\n", ++ + (unsigned long) size); ++ + write_or_whine(fast_import.in, fast_import_cmd, len, ++ + "index-stream: feeding fast-import"); ++ + while (size) { ++ + char buf[10240]; ++ + size_t sz = size < sizeof(buf) ? size : sizeof(buf); ++ + size_t actual; ++ + ++ + actual = read_in_full(fd, buf, sz); ++ + if (actual < 0) ++ + die_errno("index-stream: reading input"); ++ + if (write_in_full(fast_import.in, buf, actual) != actual) ++ + die_errno("index-stream: feeding fast-import"); ++ + size -= actual; ++ + } ++ + if (close(fast_import.in)) ++ + die_errno("index-stream: closing fast-import"); ++ + if (finish_command(&fast_import)) ++ + die_errno("index-stream: finishing fast-import"); ++ + ++ + tmpfd = open(tmpfile, O_RDONLY); ++ + if (tmpfd < 0) ++ + die_errno("index-stream: cannot open fast-import mark"); ++ + len = read(tmpfd, buf, sizeof(buf)); ++ + if (len < 0) ++ + die_errno("index-stream: reading fast-import mark"); ++ + if (close(tmpfd) < 0) ++ + die_errno("index-stream: closing fast-import mark"); ++ + if (unlink(tmpfile)) ++ + die_errno("index-stream: unlinking fast-import mark"); ++ + if (len != 44 || ++ + memcmp(":1 ", buf, 3) || ++ + get_sha1_hex(buf + 3, sha1)) ++ + die_errno("index-stream: unexpected fast-import mark: <%s>", buf); ++ + return 0; ++ +} ++ + ++ +int index_fd(unsigned char *sha1, int fd, struct stat *st, ++ + enum object_type type, const char *path, unsigned flags) ++ +{ ++ + int ret; ++ + size_t size = xsize_t(st->st_size); ++ + ++ + if (!S_ISREG(st->st_mode)) ++ + ret = index_pipe(sha1, fd, type, path, flags); ++ + else if (size <= big_file_threshold || type != OBJ_BLOB) ++ + ret = index_core(sha1, fd, size, type, path, flags); ++ + else ++ + ret = index_stream(sha1, fd, size, type, path, flags); close(fd); return ret; } -- -int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object) ++ +int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags) { int fd; struct strbuf sb = STRBUF_INIT; @@@@@ -2665,7 -2666,7 -2765,7 -2664,7 +2762,7 @@@@@ if (fd < 0) return error("open(\"%s\"): %s", path, strerror(errno)); -- - if (index_fd(sha1, fd, st, write_object, OBJ_BLOB, path, 0) < 0) ++ + if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0) return error("%s: failed to insert into database", path); break; @@@@@ -2675,7 -2676,7 -2775,7 -2674,7 +2772,7 @@@@@ return error("readlink(\"%s\"): %s", path, errstr); } -- - if (!write_object) ++ + if (!(flags & HASH_WRITE_OBJECT)) hash_sha1_file(sb.buf, sb.len, blob_type, sha1); else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1)) return error("%s: failed to insert into database",