From: Junio C Hamano Date: Thu, 11 Sep 2014 17:33:32 +0000 (-0700) Subject: Merge branch 'nd/large-blobs' X-Git-Tag: v2.2.0-rc0~136 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/bedd3b4b7b25df0b994abf444ee2136995dfeffa?ds=inline;hp=-c Merge branch 'nd/large-blobs' Teach a few codepaths to punt (instead of dying) when large blobs that would not fit in core are involved in the operation. * nd/large-blobs: diff: shortcut for diff'ing two binary SHA-1 objects diff --stat: mark any file larger than core.bigfilethreshold binary diff.c: allow to pass more flags to diff_populate_filespec sha1_file.c: do not die failing to malloc in unpack_compressed_entry wrapper.c: introduce gentle xmallocz that does not die() --- bedd3b4b7b25df0b994abf444ee2136995dfeffa diff --combined diff.c index e7d4d4200f,b85bcfbf52..d7a5c81bb8 --- a/diff.c +++ b/diff.c @@@ -376,7 -376,7 +376,7 @@@ static unsigned long diff_filespec_size { if (!DIFF_FILE_VALID(one)) return 0; - diff_populate_filespec(one, 1); + diff_populate_filespec(one, CHECK_SIZE_ONLY); return one->size; } @@@ -1910,11 -1910,11 +1910,11 @@@ static void show_dirstat(struct diff_op diff_free_filespec_data(p->one); diff_free_filespec_data(p->two); } else if (DIFF_FILE_VALID(p->one)) { - diff_populate_filespec(p->one, 1); + diff_populate_filespec(p->one, CHECK_SIZE_ONLY); copied = added = 0; diff_free_filespec_data(p->one); } else if (DIFF_FILE_VALID(p->two)) { - diff_populate_filespec(p->two, 1); + diff_populate_filespec(p->two, CHECK_SIZE_ONLY); copied = 0; added = p->two->size; diff_free_filespec_data(p->two); @@@ -2188,8 -2188,8 +2188,8 @@@ int diff_filespec_is_binary(struct diff one->is_binary = one->driver->binary; else { if (!one->data && DIFF_FILE_VALID(one)) - diff_populate_filespec(one, 0); - if (one->data) + diff_populate_filespec(one, CHECK_BINARY); + if (one->is_binary == -1 && one->data) one->is_binary = buffer_is_binary(one->data, one->size); if (one->is_binary == -1) @@@ -2324,6 -2324,19 +2324,19 @@@ static void builtin_diff(const char *na } else if (!DIFF_OPT_TST(o, TEXT) && ( (!textconv_one && diff_filespec_is_binary(one)) || (!textconv_two && diff_filespec_is_binary(two)) )) { + if (!one->data && !two->data && + S_ISREG(one->mode) && S_ISREG(two->mode) && + !DIFF_OPT_TST(o, BINARY)) { + if (!hashcmp(one->sha1, two->sha1)) { + if (must_show_header) + fprintf(o->file, "%s", header.buf); + goto free_ab_and_return; + } + fprintf(o->file, "%s", header.buf); + fprintf(o->file, "%sBinary files %s and %s differ\n", + line_prefix, lbl[0], lbl[1]); + goto free_ab_and_return; + } if (fill_mmfile(&mf1, one) < 0 || fill_mmfile(&mf2, two) < 0) die("unable to read files to diff"); /* Quite common confusing case */ @@@ -2668,8 -2681,9 +2681,9 @@@ static int diff_populate_gitlink(struc * grab the data for the blob (or file) for our own in-core comparison. * diff_filespec has data and size fields for this purpose. */ - int diff_populate_filespec(struct diff_filespec *s, int size_only) + int diff_populate_filespec(struct diff_filespec *s, unsigned int flags) { + int size_only = flags & CHECK_SIZE_ONLY; int err = 0; /* * demote FAIL to WARN to allow inspecting the situation @@@ -2724,6 -2738,11 +2738,11 @@@ } if (size_only) return 0; + if ((flags & CHECK_BINARY) && + s->size > big_file_threshold && s->is_binary == -1) { + s->is_binary = 1; + return 0; + } fd = open(s->path, O_RDONLY); if (fd < 0) goto err_empty; @@@ -2745,16 -2764,21 +2764,21 @@@ } else { enum object_type type; - if (size_only) { + if (size_only || (flags & CHECK_BINARY)) { type = sha1_object_info(s->sha1, &s->size); if (type < 0) die("unable to read %s", sha1_to_hex(s->sha1)); - } else { - s->data = read_sha1_file(s->sha1, &type, &s->size); - if (!s->data) - die("unable to read %s", sha1_to_hex(s->sha1)); - s->should_free = 1; + if (size_only) + return 0; + if (s->size > big_file_threshold && s->is_binary == -1) { + s->is_binary = 1; + return 0; + } } + s->data = read_sha1_file(s->sha1, &type, &s->size); + if (!s->data) + die("unable to read %s", sha1_to_hex(s->sha1)); + s->should_free = 1; } return 0; } @@@ -4688,8 -4712,8 +4712,8 @@@ static int diff_filespec_check_stat_unm !DIFF_FILE_VALID(p->two) || (p->one->sha1_valid && p->two->sha1_valid) || (p->one->mode != p->two->mode) || - diff_populate_filespec(p->one, 1) || - diff_populate_filespec(p->two, 1) || + diff_populate_filespec(p->one, CHECK_SIZE_ONLY) || + diff_populate_filespec(p->two, CHECK_SIZE_ONLY) || (p->one->size != p->two->size) || !diff_filespec_is_identical(p->one, p->two)) /* (2) */ p->skip_stat_unmatch_result = 1; @@@ -4931,7 -4955,7 +4955,7 @@@ static char *run_textconv(const char *p struct diff_tempfile *temp; const char *argv[3]; const char **arg = argv; - struct child_process child; + struct child_process child = CHILD_PROCESS_INIT; struct strbuf buf = STRBUF_INIT; int err = 0; @@@ -4940,6 -4964,7 +4964,6 @@@ *arg++ = temp->name; *arg = NULL; - memset(&child, 0, sizeof(child)); child.use_shell = 1; child.argv = argv; child.out = -1; diff --combined git-compat-util.h index d675c89603,8785fd30d9..4e7e3f8726 --- a/git-compat-util.h +++ b/git-compat-util.h @@@ -264,35 -264,19 +264,35 @@@ extern char *gitbasename(char *) #endif #ifndef has_dos_drive_prefix -#define has_dos_drive_prefix(path) 0 +static inline int git_has_dos_drive_prefix(const char *path) +{ + return 0; +} +#define has_dos_drive_prefix git_has_dos_drive_prefix #endif -#ifndef offset_1st_component -#define offset_1st_component(path) (is_dir_sep((path)[0])) +#ifndef is_dir_sep +static inline int git_is_dir_sep(int c) +{ + return c == '/'; +} +#define is_dir_sep git_is_dir_sep #endif -#ifndef is_dir_sep -#define is_dir_sep(c) ((c) == '/') +#ifndef offset_1st_component +static inline int git_offset_1st_component(const char *path) +{ + return is_dir_sep(path[0]); +} +#define offset_1st_component git_offset_1st_component #endif #ifndef find_last_dir_sep -#define find_last_dir_sep(path) strrchr(path, '/') +static inline char *git_find_last_dir_sep(const char *path) +{ + return strrchr(path, '/'); +} +#define find_last_dir_sep git_find_last_dir_sep #endif #if defined(__HP_cc) && (__HP_cc >= 61000) @@@ -609,6 -593,7 +609,7 @@@ extern try_to_free_t set_try_to_free_ro extern char *xstrdup(const char *str); extern void *xmalloc(size_t size); extern void *xmallocz(size_t size); + extern void *xmallocz_gently(size_t size); extern void *xmemdupz(const void *data, size_t len); extern char *xstrndup(const char *str, size_t len); extern void *xrealloc(void *ptr, size_t size); @@@ -623,7 -608,6 +624,7 @@@ extern int xmkstemp(char *template) extern int xmkstemp_mode(char *template, int mode); extern int odb_mkstemp(char *template, size_t limit, const char *pattern); extern int odb_pack_keep(char *name, size_t namesz, const unsigned char *sha1); +extern char *xgetcwd(void); static inline size_t xsize_t(off_t len) { diff --combined sha1_file.c index 95afd20910,8db73f0c53..c08c0cbea8 --- a/sha1_file.c +++ b/sha1_file.c @@@ -350,7 -350,7 +350,7 @@@ static void link_alt_odb_entries(const return; } - strbuf_addstr(&objdirbuf, absolute_path(get_object_directory())); + strbuf_add_absolute_path(&objdirbuf, get_object_directory()); normalize_path_copy(objdirbuf.buf, objdirbuf.buf); alt_copy = xmemdupz(alt, len); @@@ -1923,7 -1923,9 +1923,9 @@@ static void *unpack_compressed_entry(st git_zstream stream; unsigned char *buffer, *in; - buffer = xmallocz(size); + buffer = xmallocz_gently(size); + if (!buffer) + return NULL; memset(&stream, 0, sizeof(stream)); stream.next_out = buffer; stream.avail_out = size + 1; diff --combined wrapper.c index bd24cdabfb,dc9c8f4dd3..25074d71b6 --- a/wrapper.c +++ b/wrapper.c @@@ -9,16 -9,23 +9,23 @@@ static void do_nothing(size_t size static void (*try_to_free_routine)(size_t size) = do_nothing; - static void memory_limit_check(size_t size) + static int memory_limit_check(size_t size, int gentle) { static int limit = -1; if (limit == -1) { const char *env = getenv("GIT_ALLOC_LIMIT"); limit = env ? atoi(env) * 1024 : 0; } - if (limit && size > limit) - die("attempting to allocate %"PRIuMAX" over limit %d", - (intmax_t)size, limit); + if (limit && size > limit) { + if (gentle) { + error("attempting to allocate %"PRIuMAX" over limit %d", + (intmax_t)size, limit); + return -1; + } else + die("attempting to allocate %"PRIuMAX" over limit %d", + (intmax_t)size, limit); + } + return 0; } try_to_free_t set_try_to_free_routine(try_to_free_t routine) @@@ -42,11 -49,12 +49,12 @@@ char *xstrdup(const char *str return ret; } - void *xmalloc(size_t size) + static void *do_xmalloc(size_t size, int gentle) { void *ret; - memory_limit_check(size); + if (memory_limit_check(size, gentle)) + return NULL; ret = malloc(size); if (!ret && !size) ret = malloc(1); @@@ -55,9 -63,16 +63,16 @@@ ret = malloc(size); if (!ret && !size) ret = malloc(1); - if (!ret) - die("Out of memory, malloc failed (tried to allocate %lu bytes)", - (unsigned long)size); + if (!ret) { + if (!gentle) + die("Out of memory, malloc failed (tried to allocate %lu bytes)", + (unsigned long)size); + else { + error("Out of memory, malloc failed (tried to allocate %lu bytes)", + (unsigned long)size); + return NULL; + } + } } #ifdef XMALLOC_POISON memset(ret, 0xA5, size); @@@ -65,16 -80,37 +80,37 @@@ return ret; } - void *xmallocz(size_t size) + void *xmalloc(size_t size) + { + return do_xmalloc(size, 0); + } + + static void *do_xmallocz(size_t size, int gentle) { void *ret; - if (unsigned_add_overflows(size, 1)) - die("Data too large to fit into virtual memory space."); - ret = xmalloc(size + 1); - ((char*)ret)[size] = 0; + if (unsigned_add_overflows(size, 1)) { + if (gentle) { + error("Data too large to fit into virtual memory space."); + return NULL; + } else + die("Data too large to fit into virtual memory space."); + } + ret = do_xmalloc(size + 1, gentle); + if (ret) + ((char*)ret)[size] = 0; return ret; } + void *xmallocz(size_t size) + { + return do_xmallocz(size, 0); + } + + void *xmallocz_gently(size_t size) + { + return do_xmallocz(size, 1); + } + /* * xmemdupz() allocates (len + 1) bytes of memory, duplicates "len" bytes of * "data" to the allocated memory, zero terminates the allocated memory, @@@ -96,7 -132,7 +132,7 @@@ void *xrealloc(void *ptr, size_t size { void *ret; - memory_limit_check(size); + memory_limit_check(size, 0); ret = realloc(ptr, size); if (!ret && !size) ret = realloc(ptr, 1); @@@ -115,7 -151,7 +151,7 @@@ void *xcalloc(size_t nmemb, size_t size { void *ret; - memory_limit_check(size * nmemb); + memory_limit_check(size * nmemb, 0); ret = calloc(nmemb, size); if (!ret && (!nmemb || !size)) ret = calloc(1, 1); @@@ -493,11 -529,3 +529,11 @@@ struct passwd *xgetpwuid_self(void errno ? strerror(errno) : _("no such user")); return pw; } + +char *xgetcwd(void) +{ + struct strbuf sb = STRBUF_INIT; + if (strbuf_getcwd(&sb)) + die_errno(_("unable to get current working directory")); + return strbuf_detach(&sb, NULL); +}