From: Junio C Hamano Date: Wed, 25 May 2011 23:23:26 +0000 (-0700) Subject: Merge branch 'jc/bigfile' X-Git-Tag: v1.7.6-rc0~31 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/5cfe4256d98af22a570c78c5e3048391a90f5f98?hp=-c Merge branch 'jc/bigfile' * jc/bigfile: Bigfile: teach "git add" to send a large file straight to a pack index_fd(): split into two helper functions index_fd(): turn write_object and format_check arguments into one flag --- 5cfe4256d98af22a570c78c5e3048391a90f5f98 diff --combined cache.h index 28a921dffc,bc38013959..009b365370 --- a/cache.h +++ b/cache.h @@@ -511,15 -511,18 +511,18 @@@ struct pathspec struct pathspec_item { const char *match; int len; - unsigned int has_wildcard:1; + unsigned int use_wildcard:1; } *items; }; extern int init_pathspec(struct pathspec *, const char **); extern void free_pathspec(struct pathspec *); extern int ce_path_match(const struct cache_entry *ce, const struct pathspec *pathspec); - extern int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, enum object_type type, const char *path, int format_check); - extern int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object); + + #define HASH_WRITE_OBJECT 1 + #define HASH_FORMAT_CHECK 2 + extern int index_fd(unsigned char *sha1, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags); + extern int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags); extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st); #define REFRESH_REALLY 0x0001 /* ignore_valid */ @@@ -606,7 -609,7 +609,7 @@@ enum eol #endif }; -extern enum eol eol; +extern enum eol core_eol; enum branch_track { BRANCH_TRACK_UNSPECIFIED = -1, @@@ -676,24 -679,14 +679,24 @@@ extern char *sha1_pack_name(const unsig extern char *sha1_pack_index_name(const unsigned char *sha1); extern const char *find_unique_abbrev(const unsigned char *sha1, int); extern const unsigned char null_sha1[20]; -static inline int is_null_sha1(const unsigned char *sha1) + +static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2) { - return !memcmp(sha1, null_sha1, 20); + int i; + + for (i = 0; i < 20; i++, sha1++, sha2++) { + if (*sha1 != *sha2) + return *sha1 - *sha2; + } + + return 0; } -static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2) + +static inline int is_null_sha1(const unsigned char *sha1) { - return memcmp(sha1, sha2, 20); + return !hashcmp(sha1, null_sha1); } + static inline void hashcpy(unsigned char *sha_dst, const unsigned char *sha_src) { memcpy(sha_dst, sha_src, 20); @@@ -756,23 -749,13 +759,23 @@@ char *strip_path_suffix(const char *pat int daemon_avoid_alias(const char *path); int offset_1st_component(const char *path); -/* Read and unpack a sha1 file into memory, write memory to a sha1 file */ -extern int sha1_object_info(const unsigned char *, unsigned long *); -extern void *read_sha1_file_repl(const unsigned char *sha1, enum object_type *type, unsigned long *size, const unsigned char **replacement); +/* object replacement */ +#define READ_SHA1_FILE_REPLACE 1 +extern void *read_sha1_file_extended(const unsigned char *sha1, enum object_type *type, unsigned long *size, unsigned flag); static inline void *read_sha1_file(const unsigned char *sha1, enum object_type *type, unsigned long *size) { - return read_sha1_file_repl(sha1, type, size, NULL); + return read_sha1_file_extended(sha1, type, size, READ_SHA1_FILE_REPLACE); +} +extern const unsigned char *do_lookup_replace_object(const unsigned char *sha1); +static inline const unsigned char *lookup_replace_object(const unsigned char *sha1) +{ + if (!read_replace_refs) + return sha1; + return do_lookup_replace_object(sha1); } + +/* Read and unpack a sha1 file into memory, write memory to a sha1 file */ +extern int sha1_object_info(const unsigned char *, unsigned long *); extern int hash_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1); extern int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *return_sha1); extern int pretend_sha1_file(void *, unsigned long, enum object_type, unsigned char *); @@@ -810,15 -793,15 +813,15 @@@ struct object_context }; extern int get_sha1(const char *str, unsigned char *sha1); -extern int get_sha1_with_mode_1(const char *str, unsigned char *sha1, unsigned *mode, int gently, const char *prefix); +extern int get_sha1_with_mode_1(const char *str, unsigned char *sha1, unsigned *mode, int only_to_die, const char *prefix); static inline int get_sha1_with_mode(const char *str, unsigned char *sha1, unsigned *mode) { - return get_sha1_with_mode_1(str, sha1, mode, 1, NULL); + return get_sha1_with_mode_1(str, sha1, mode, 0, NULL); } -extern int get_sha1_with_context_1(const char *name, unsigned char *sha1, struct object_context *orc, int gently, const char *prefix); +extern int get_sha1_with_context_1(const char *name, unsigned char *sha1, struct object_context *orc, int only_to_die, const char *prefix); static inline int get_sha1_with_context(const char *str, unsigned char *sha1, struct object_context *orc) { - return get_sha1_with_context_1(str, sha1, orc, 1, NULL); + return get_sha1_with_context_1(str, sha1, orc, 0, NULL); } extern int get_sha1_hex(const char *hex, unsigned char *sha1); extern char *sha1_to_hex(const unsigned char *sha1); /* static buffer result! */ @@@ -985,7 -968,6 +988,7 @@@ extern struct ref *find_ref_by_name(con extern char *git_getpass(const char *prompt); extern struct child_process *git_connect(int fd[2], const char *url, const char *prog, int flags); extern int finish_connect(struct child_process *conn); +extern int git_connection_is_socket(struct child_process *conn); extern int path_match(const char *path, int nr, char **match); struct extra_have_objects { int nr, alloc; @@@ -1023,16 -1005,6 +1026,16 @@@ extern const char *packed_object_info_d /* Dumb servers support */ extern int update_server_info(int); +/* git_config_parse_key() returns these negated: */ +#define CONFIG_INVALID_KEY 1 +#define CONFIG_NO_SECTION_OR_NAME 2 +/* git_config_set(), git_config_set_multivar() return the above or these: */ +#define CONFIG_NO_LOCK -1 +#define CONFIG_INVALID_FILE 3 +#define CONFIG_NO_WRITE 4 +#define CONFIG_NOTHING_SET 5 +#define CONFIG_INVALID_PATTERN 6 + typedef int (*config_fn_t)(const char *, const char *, void *); extern int git_default_config(const char *, const char *, void *); extern int git_config_from_file(config_fn_t fn, const char *, void *); diff --combined sha1_file.c index 357f9ab7ff,f0ca6a1749..064a330408 --- a/sha1_file.c +++ b/sha1_file.c @@@ -11,6 -11,7 +11,7 @@@ #include "pack.h" #include "blob.h" #include "commit.h" + #include "run-command.h" #include "tag.h" #include "tree.h" #include "tree-walk.h" @@@ -31,6 -32,8 +32,6 @@@ static inline uintmax_t sz_fmt(size_t s const unsigned char null_sha1[20]; -static int git_open_noatime(const char *name, struct packed_git *p); - /* * This is meant to hold a *small* number of objects that you would * want read_sha1_file() to be able to return, but yet you do not want @@@ -225,7 -228,6 +226,7 @@@ struct alternate_object_database *alt_o static struct alternate_object_database **alt_odb_tail; static void read_info_alternates(const char * alternates, int depth); +static int git_open_noatime(const char *name); /* * Prepare alternate object database registry. @@@ -359,7 -361,7 +360,7 @@@ static void read_info_alternates(const int fd; sprintf(path, "%s/%s", relative_base, alt_file_name); - fd = git_open_noatime(path, NULL); + fd = git_open_noatime(path); if (fd < 0) return; if (fstat(fd, &st) || (st.st_size == 0)) { @@@ -474,7 -476,7 +475,7 @@@ static int check_packed_git_idx(const c struct pack_idx_header *hdr; size_t idx_size; uint32_t version, nr, i, *index; - int fd = git_open_noatime(path, p); + int fd = git_open_noatime(path); struct stat st; if (fd < 0) @@@ -756,7 -758,7 +757,7 @@@ static int open_packed_git_1(struct pac while (pack_max_fds <= pack_open_fds && unuse_one_window(NULL, -1)) ; /* nothing */ - p->pack_fd = git_open_noatime(p->pack_name, p); + p->pack_fd = git_open_noatime(p->pack_name); if (p->pack_fd < 0 || fstat(p->pack_fd, &st)) return -1; pack_open_fds++; @@@ -1144,7 -1146,7 +1145,7 @@@ int check_sha1_signature(const unsigne return hashcmp(sha1, real_sha1) ? -1 : 0; } -static int git_open_noatime(const char *name, struct packed_git *p) +static int git_open_noatime(const char *name) { static int sha1_file_open_flag = O_NOATIME; @@@ -1169,7 -1171,7 +1170,7 @@@ static int open_sha1_file(const unsigne char *name = sha1_file_name(sha1); struct alternate_object_database *alt; - fd = git_open_noatime(name, NULL); + fd = git_open_noatime(name); if (fd >= 0) return fd; @@@ -1178,7 -1180,7 +1179,7 @@@ for (alt = alt_odb_list; alt; alt = alt->next) { name = alt->name; fill_sha1_path(name, sha1); - fd = git_open_noatime(alt->base, NULL); + fd = git_open_noatime(alt->base); if (fd >= 0) return fd; } @@@ -1307,7 -1309,7 +1308,7 @@@ static void *unpack_sha1_rest(z_stream /* * The above condition must be (bytes <= size), not * (bytes < size). In other words, even though we - * expect no more output and set avail_out to zer0, + * expect no more output and set avail_out to zero, * the input zlib stream may have bytes that express * "this concludes the stream", and we *do* want to * eat that input. @@@ -2205,21 -2207,23 +2206,21 @@@ static void *read_object(const unsigne * deal with them should arrange to call read_object() and give error * messages themselves. */ -void *read_sha1_file_repl(const unsigned char *sha1, - enum object_type *type, - unsigned long *size, - const unsigned char **replacement) +void *read_sha1_file_extended(const unsigned char *sha1, + enum object_type *type, + unsigned long *size, + unsigned flag) { - const unsigned char *repl = lookup_replace_object(sha1); void *data; char *path; const struct packed_git *p; + const unsigned char *repl = (flag & READ_SHA1_FILE_REPLACE) + ? lookup_replace_object(sha1) : sha1; errno = 0; data = read_object(repl, type, size); - if (data) { - if (replacement) - *replacement = repl; + if (data) return data; - } if (errno && errno != ENOENT) die_errno("failed to read object %s", sha1_to_hex(sha1)); @@@ -2578,10 -2582,11 +2579,11 @@@ static void check_tag(const void *buf, } static int index_mem(unsigned char *sha1, void *buf, size_t size, - int write_object, enum object_type type, - const char *path, int format_check) + enum object_type type, + const char *path, unsigned flags) { int ret, re_allocated = 0; + int write_object = flags & HASH_WRITE_OBJECT; if (!type) type = OBJ_BLOB; @@@ -2597,7 -2602,7 +2599,7 @@@ re_allocated = 1; } } - if (format_check) { + if (flags & HASH_FORMAT_CHECK) { if (type == OBJ_TREE) check_tree(buf, size); if (type == OBJ_COMMIT) @@@ -2615,44 -2620,141 +2617,141 @@@ return ret; } + static int index_pipe(unsigned char *sha1, int fd, enum object_type type, + const char *path, unsigned flags) + { + struct strbuf sbuf = STRBUF_INIT; + int ret; + + if (strbuf_read(&sbuf, fd, 4096) >= 0) + ret = index_mem(sha1, sbuf.buf, sbuf.len, type, path, flags); + else + ret = -1; + strbuf_release(&sbuf); + return ret; + } + #define SMALL_FILE_SIZE (32*1024) - int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, - enum object_type type, const char *path, int format_check) + static int index_core(unsigned char *sha1, int fd, size_t size, + enum object_type type, const char *path, + unsigned flags) { int ret; - size_t size = xsize_t(st->st_size); - if (!S_ISREG(st->st_mode)) { - struct strbuf sbuf = STRBUF_INIT; - if (strbuf_read(&sbuf, fd, 4096) >= 0) - ret = index_mem(sha1, sbuf.buf, sbuf.len, write_object, - type, path, format_check); - else - ret = -1; - strbuf_release(&sbuf); - } else if (!size) { - ret = index_mem(sha1, NULL, size, write_object, type, path, - format_check); + if (!size) { + ret = index_mem(sha1, NULL, size, type, path, flags); } else if (size <= SMALL_FILE_SIZE) { char *buf = xmalloc(size); if (size == read_in_full(fd, buf, size)) - ret = index_mem(sha1, buf, size, write_object, type, - path, format_check); + ret = index_mem(sha1, buf, size, type, path, flags); else ret = error("short read %s", strerror(errno)); free(buf); } else { void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); - ret = index_mem(sha1, buf, size, write_object, type, path, - format_check); + ret = index_mem(sha1, buf, size, type, path, flags); munmap(buf, size); } + return ret; + } + + /* + * This creates one packfile per large blob, because the caller + * immediately wants the result sha1, and fast-import can report the + * object name via marks mechanism only by closing the created + * packfile. + * + * This also bypasses the usual "convert-to-git" dance, and that is on + * purpose. We could write a streaming version of the converting + * functions and insert that before feeding the data to fast-import + * (or equivalent in-core API described above), but the primary + * motivation for trying to stream from the working tree file and to + * avoid mmaping it in core is to deal with large binary blobs, and + * by definition they do _not_ want to get any conversion. + */ + static int index_stream(unsigned char *sha1, int fd, size_t size, + enum object_type type, const char *path, + unsigned flags) + { + struct child_process fast_import; + char export_marks[512]; + const char *argv[] = { "fast-import", "--quiet", export_marks, NULL }; + char tmpfile[512]; + char fast_import_cmd[512]; + char buf[512]; + int len, tmpfd; + + strcpy(tmpfile, git_path("hashstream_XXXXXX")); + tmpfd = git_mkstemp_mode(tmpfile, 0600); + if (tmpfd < 0) + die_errno("cannot create tempfile: %s", tmpfile); + if (close(tmpfd)) + die_errno("cannot close tempfile: %s", tmpfile); + sprintf(export_marks, "--export-marks=%s", tmpfile); + + memset(&fast_import, 0, sizeof(fast_import)); + fast_import.in = -1; + fast_import.argv = argv; + fast_import.git_cmd = 1; + if (start_command(&fast_import)) + die_errno("index-stream: git fast-import failed"); + + len = sprintf(fast_import_cmd, "blob\nmark :1\ndata %lu\n", + (unsigned long) size); + write_or_whine(fast_import.in, fast_import_cmd, len, + "index-stream: feeding fast-import"); + while (size) { + char buf[10240]; + size_t sz = size < sizeof(buf) ? size : sizeof(buf); + size_t actual; + + actual = read_in_full(fd, buf, sz); + if (actual < 0) + die_errno("index-stream: reading input"); + if (write_in_full(fast_import.in, buf, actual) != actual) + die_errno("index-stream: feeding fast-import"); + size -= actual; + } + if (close(fast_import.in)) + die_errno("index-stream: closing fast-import"); + if (finish_command(&fast_import)) + die_errno("index-stream: finishing fast-import"); + + tmpfd = open(tmpfile, O_RDONLY); + if (tmpfd < 0) + die_errno("index-stream: cannot open fast-import mark"); + len = read(tmpfd, buf, sizeof(buf)); + if (len < 0) + die_errno("index-stream: reading fast-import mark"); + if (close(tmpfd) < 0) + die_errno("index-stream: closing fast-import mark"); + if (unlink(tmpfile)) + die_errno("index-stream: unlinking fast-import mark"); + if (len != 44 || + memcmp(":1 ", buf, 3) || + get_sha1_hex(buf + 3, sha1)) + die_errno("index-stream: unexpected fast-import mark: <%s>", buf); + return 0; + } + + int index_fd(unsigned char *sha1, int fd, struct stat *st, + enum object_type type, const char *path, unsigned flags) + { + int ret; + size_t size = xsize_t(st->st_size); + + if (!S_ISREG(st->st_mode)) + ret = index_pipe(sha1, fd, type, path, flags); + else if (size <= big_file_threshold || type != OBJ_BLOB) + ret = index_core(sha1, fd, size, type, path, flags); + else + ret = index_stream(sha1, fd, size, type, path, flags); close(fd); return ret; } - int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object) + int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags) { int fd; struct strbuf sb = STRBUF_INIT; @@@ -2663,7 -2765,7 +2762,7 @@@ if (fd < 0) return error("open(\"%s\"): %s", path, strerror(errno)); - if (index_fd(sha1, fd, st, write_object, OBJ_BLOB, path, 0) < 0) + if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0) return error("%s: failed to insert into database", path); break; @@@ -2673,7 -2775,7 +2772,7 @@@ return error("readlink(\"%s\"): %s", path, errstr); } - if (!write_object) + if (!(flags & HASH_WRITE_OBJECT)) hash_sha1_file(sb.buf, sb.len, blob_type, sha1); else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1)) return error("%s: failed to insert into database",