Merge branch 'nd/large-blobs'
authorJunio C Hamano <gitster@pobox.com>
Thu, 11 Sep 2014 17:33:32 +0000 (10:33 -0700)
committerJunio C Hamano <gitster@pobox.com>
Thu, 11 Sep 2014 17:33:33 +0000 (10:33 -0700)
Teach a few codepaths to punt (instead of dying) when large blobs
that would not fit in core are involved in the operation.

* nd/large-blobs:
diff: shortcut for diff'ing two binary SHA-1 objects
diff --stat: mark any file larger than core.bigfilethreshold binary
diff.c: allow to pass more flags to diff_populate_filespec
sha1_file.c: do not die failing to malloc in unpack_compressed_entry
wrapper.c: introduce gentle xmallocz that does not die()

1  2 
diff.c
git-compat-util.h
sha1_file.c
wrapper.c
diff --combined diff.c
index e7d4d4200f593ca43a1eba88fbc43bc5348a3afc,b85bcfbf52e52dbd5521dde1078de9593d772f67..d7a5c81bb8545584ce5fe652dc42f2ee8bc1e2fd
--- 1/diff.c
--- 2/diff.c
+++ b/diff.c
@@@ -376,7 -376,7 +376,7 @@@ static unsigned long diff_filespec_size
  {
        if (!DIFF_FILE_VALID(one))
                return 0;
-       diff_populate_filespec(one, 1);
+       diff_populate_filespec(one, CHECK_SIZE_ONLY);
        return one->size;
  }
  
@@@ -1910,11 -1910,11 +1910,11 @@@ static void show_dirstat(struct diff_op
                        diff_free_filespec_data(p->one);
                        diff_free_filespec_data(p->two);
                } else if (DIFF_FILE_VALID(p->one)) {
-                       diff_populate_filespec(p->one, 1);
+                       diff_populate_filespec(p->one, CHECK_SIZE_ONLY);
                        copied = added = 0;
                        diff_free_filespec_data(p->one);
                } else if (DIFF_FILE_VALID(p->two)) {
-                       diff_populate_filespec(p->two, 1);
+                       diff_populate_filespec(p->two, CHECK_SIZE_ONLY);
                        copied = 0;
                        added = p->two->size;
                        diff_free_filespec_data(p->two);
@@@ -2188,8 -2188,8 +2188,8 @@@ int diff_filespec_is_binary(struct diff
                        one->is_binary = one->driver->binary;
                else {
                        if (!one->data && DIFF_FILE_VALID(one))
-                               diff_populate_filespec(one, 0);
-                       if (one->data)
+                               diff_populate_filespec(one, CHECK_BINARY);
+                       if (one->is_binary == -1 && one->data)
                                one->is_binary = buffer_is_binary(one->data,
                                                one->size);
                        if (one->is_binary == -1)
@@@ -2324,6 -2324,19 +2324,19 @@@ static void builtin_diff(const char *na
        } else if (!DIFF_OPT_TST(o, TEXT) &&
            ( (!textconv_one && diff_filespec_is_binary(one)) ||
              (!textconv_two && diff_filespec_is_binary(two)) )) {
+               if (!one->data && !two->data &&
+                   S_ISREG(one->mode) && S_ISREG(two->mode) &&
+                   !DIFF_OPT_TST(o, BINARY)) {
+                       if (!hashcmp(one->sha1, two->sha1)) {
+                               if (must_show_header)
+                                       fprintf(o->file, "%s", header.buf);
+                               goto free_ab_and_return;
+                       }
+                       fprintf(o->file, "%s", header.buf);
+                       fprintf(o->file, "%sBinary files %s and %s differ\n",
+                               line_prefix, lbl[0], lbl[1]);
+                       goto free_ab_and_return;
+               }
                if (fill_mmfile(&mf1, one) < 0 || fill_mmfile(&mf2, two) < 0)
                        die("unable to read files to diff");
                /* Quite common confusing case */
@@@ -2668,8 -2681,9 +2681,9 @@@ static int diff_populate_gitlink(struc
   * grab the data for the blob (or file) for our own in-core comparison.
   * diff_filespec has data and size fields for this purpose.
   */
- int diff_populate_filespec(struct diff_filespec *s, int size_only)
+ int diff_populate_filespec(struct diff_filespec *s, unsigned int flags)
  {
+       int size_only = flags & CHECK_SIZE_ONLY;
        int err = 0;
        /*
         * demote FAIL to WARN to allow inspecting the situation
                }
                if (size_only)
                        return 0;
+               if ((flags & CHECK_BINARY) &&
+                   s->size > big_file_threshold && s->is_binary == -1) {
+                       s->is_binary = 1;
+                       return 0;
+               }
                fd = open(s->path, O_RDONLY);
                if (fd < 0)
                        goto err_empty;
        }
        else {
                enum object_type type;
-               if (size_only) {
+               if (size_only || (flags & CHECK_BINARY)) {
                        type = sha1_object_info(s->sha1, &s->size);
                        if (type < 0)
                                die("unable to read %s", sha1_to_hex(s->sha1));
-               } else {
-                       s->data = read_sha1_file(s->sha1, &type, &s->size);
-                       if (!s->data)
-                               die("unable to read %s", sha1_to_hex(s->sha1));
-                       s->should_free = 1;
+                       if (size_only)
+                               return 0;
+                       if (s->size > big_file_threshold && s->is_binary == -1) {
+                               s->is_binary = 1;
+                               return 0;
+                       }
                }
+               s->data = read_sha1_file(s->sha1, &type, &s->size);
+               if (!s->data)
+                       die("unable to read %s", sha1_to_hex(s->sha1));
+               s->should_free = 1;
        }
        return 0;
  }
@@@ -4688,8 -4712,8 +4712,8 @@@ static int diff_filespec_check_stat_unm
            !DIFF_FILE_VALID(p->two) ||
            (p->one->sha1_valid && p->two->sha1_valid) ||
            (p->one->mode != p->two->mode) ||
-           diff_populate_filespec(p->one, 1) ||
-           diff_populate_filespec(p->two, 1) ||
+           diff_populate_filespec(p->one, CHECK_SIZE_ONLY) ||
+           diff_populate_filespec(p->two, CHECK_SIZE_ONLY) ||
            (p->one->size != p->two->size) ||
            !diff_filespec_is_identical(p->one, p->two)) /* (2) */
                p->skip_stat_unmatch_result = 1;
@@@ -4931,7 -4955,7 +4955,7 @@@ static char *run_textconv(const char *p
        struct diff_tempfile *temp;
        const char *argv[3];
        const char **arg = argv;
 -      struct child_process child;
 +      struct child_process child = CHILD_PROCESS_INIT;
        struct strbuf buf = STRBUF_INIT;
        int err = 0;
  
        *arg++ = temp->name;
        *arg = NULL;
  
 -      memset(&child, 0, sizeof(child));
        child.use_shell = 1;
        child.argv = argv;
        child.out = -1;
diff --combined git-compat-util.h
index d675c89603eaf6135354c120f06289bd90a3f3d3,8785fd30d9390417c216e991b7291ebf734329f2..4e7e3f8726a26a1f7dfbb889fe3531d01d4419b9
@@@ -264,35 -264,19 +264,35 @@@ extern char *gitbasename(char *)
  #endif
  
  #ifndef has_dos_drive_prefix
 -#define has_dos_drive_prefix(path) 0
 +static inline int git_has_dos_drive_prefix(const char *path)
 +{
 +      return 0;
 +}
 +#define has_dos_drive_prefix git_has_dos_drive_prefix
  #endif
  
 -#ifndef offset_1st_component
 -#define offset_1st_component(path) (is_dir_sep((path)[0]))
 +#ifndef is_dir_sep
 +static inline int git_is_dir_sep(int c)
 +{
 +      return c == '/';
 +}
 +#define is_dir_sep git_is_dir_sep
  #endif
  
 -#ifndef is_dir_sep
 -#define is_dir_sep(c) ((c) == '/')
 +#ifndef offset_1st_component
 +static inline int git_offset_1st_component(const char *path)
 +{
 +      return is_dir_sep(path[0]);
 +}
 +#define offset_1st_component git_offset_1st_component
  #endif
  
  #ifndef find_last_dir_sep
 -#define find_last_dir_sep(path) strrchr(path, '/')
 +static inline char *git_find_last_dir_sep(const char *path)
 +{
 +      return strrchr(path, '/');
 +}
 +#define find_last_dir_sep git_find_last_dir_sep
  #endif
  
  #if defined(__HP_cc) && (__HP_cc >= 61000)
@@@ -609,6 -593,7 +609,7 @@@ extern try_to_free_t set_try_to_free_ro
  extern char *xstrdup(const char *str);
  extern void *xmalloc(size_t size);
  extern void *xmallocz(size_t size);
+ extern void *xmallocz_gently(size_t size);
  extern void *xmemdupz(const void *data, size_t len);
  extern char *xstrndup(const char *str, size_t len);
  extern void *xrealloc(void *ptr, size_t size);
@@@ -623,7 -608,6 +624,7 @@@ extern int xmkstemp(char *template)
  extern int xmkstemp_mode(char *template, int mode);
  extern int odb_mkstemp(char *template, size_t limit, const char *pattern);
  extern int odb_pack_keep(char *name, size_t namesz, const unsigned char *sha1);
 +extern char *xgetcwd(void);
  
  static inline size_t xsize_t(off_t len)
  {
diff --combined sha1_file.c
index 95afd209107277da3154226dd08bd8040a9097b5,8db73f0c53f01bf39decb71d036610d2835cf7ac..c08c0cbea805b38104504b9b51266949affb6991
@@@ -350,7 -350,7 +350,7 @@@ static void link_alt_odb_entries(const 
                return;
        }
  
 -      strbuf_addstr(&objdirbuf, absolute_path(get_object_directory()));
 +      strbuf_add_absolute_path(&objdirbuf, get_object_directory());
        normalize_path_copy(objdirbuf.buf, objdirbuf.buf);
  
        alt_copy = xmemdupz(alt, len);
@@@ -1923,7 -1923,9 +1923,9 @@@ static void *unpack_compressed_entry(st
        git_zstream stream;
        unsigned char *buffer, *in;
  
-       buffer = xmallocz(size);
+       buffer = xmallocz_gently(size);
+       if (!buffer)
+               return NULL;
        memset(&stream, 0, sizeof(stream));
        stream.next_out = buffer;
        stream.avail_out = size + 1;
diff --combined wrapper.c
index bd24cdabfb818d4eddde4f08819f5a0166283a6a,dc9c8f4dd32566e9e310682987d212602a2ac16b..25074d71b6ce72066efc02abda74ddfb10f71d6c
+++ b/wrapper.c
@@@ -9,16 -9,23 +9,23 @@@ static void do_nothing(size_t size
  
  static void (*try_to_free_routine)(size_t size) = do_nothing;
  
- static void memory_limit_check(size_t size)
+ static int memory_limit_check(size_t size, int gentle)
  {
        static int limit = -1;
        if (limit == -1) {
                const char *env = getenv("GIT_ALLOC_LIMIT");
                limit = env ? atoi(env) * 1024 : 0;
        }
-       if (limit && size > limit)
-               die("attempting to allocate %"PRIuMAX" over limit %d",
-                   (intmax_t)size, limit);
+       if (limit && size > limit) {
+               if (gentle) {
+                       error("attempting to allocate %"PRIuMAX" over limit %d",
+                             (intmax_t)size, limit);
+                       return -1;
+               } else
+                       die("attempting to allocate %"PRIuMAX" over limit %d",
+                           (intmax_t)size, limit);
+       }
+       return 0;
  }
  
  try_to_free_t set_try_to_free_routine(try_to_free_t routine)
@@@ -42,11 -49,12 +49,12 @@@ char *xstrdup(const char *str
        return ret;
  }
  
void *xmalloc(size_t size)
static void *do_xmalloc(size_t size, int gentle)
  {
        void *ret;
  
-       memory_limit_check(size);
+       if (memory_limit_check(size, gentle))
+               return NULL;
        ret = malloc(size);
        if (!ret && !size)
                ret = malloc(1);
                ret = malloc(size);
                if (!ret && !size)
                        ret = malloc(1);
-               if (!ret)
-                       die("Out of memory, malloc failed (tried to allocate %lu bytes)",
-                           (unsigned long)size);
+               if (!ret) {
+                       if (!gentle)
+                               die("Out of memory, malloc failed (tried to allocate %lu bytes)",
+                                   (unsigned long)size);
+                       else {
+                               error("Out of memory, malloc failed (tried to allocate %lu bytes)",
+                                     (unsigned long)size);
+                               return NULL;
+                       }
+               }
        }
  #ifdef XMALLOC_POISON
        memset(ret, 0xA5, size);
        return ret;
  }
  
- void *xmallocz(size_t size)
+ void *xmalloc(size_t size)
+ {
+       return do_xmalloc(size, 0);
+ }
+ static void *do_xmallocz(size_t size, int gentle)
  {
        void *ret;
-       if (unsigned_add_overflows(size, 1))
-               die("Data too large to fit into virtual memory space.");
-       ret = xmalloc(size + 1);
-       ((char*)ret)[size] = 0;
+       if (unsigned_add_overflows(size, 1)) {
+               if (gentle) {
+                       error("Data too large to fit into virtual memory space.");
+                       return NULL;
+               } else
+                       die("Data too large to fit into virtual memory space.");
+       }
+       ret = do_xmalloc(size + 1, gentle);
+       if (ret)
+               ((char*)ret)[size] = 0;
        return ret;
  }
  
+ void *xmallocz(size_t size)
+ {
+       return do_xmallocz(size, 0);
+ }
+ void *xmallocz_gently(size_t size)
+ {
+       return do_xmallocz(size, 1);
+ }
  /*
   * xmemdupz() allocates (len + 1) bytes of memory, duplicates "len" bytes of
   * "data" to the allocated memory, zero terminates the allocated memory,
@@@ -96,7 -132,7 +132,7 @@@ void *xrealloc(void *ptr, size_t size
  {
        void *ret;
  
-       memory_limit_check(size);
+       memory_limit_check(size, 0);
        ret = realloc(ptr, size);
        if (!ret && !size)
                ret = realloc(ptr, 1);
@@@ -115,7 -151,7 +151,7 @@@ void *xcalloc(size_t nmemb, size_t size
  {
        void *ret;
  
-       memory_limit_check(size * nmemb);
+       memory_limit_check(size * nmemb, 0);
        ret = calloc(nmemb, size);
        if (!ret && (!nmemb || !size))
                ret = calloc(1, 1);
@@@ -493,11 -529,3 +529,11 @@@ struct passwd *xgetpwuid_self(void
                    errno ? strerror(errno) : _("no such user"));
        return pw;
  }
 +
 +char *xgetcwd(void)
 +{
 +      struct strbuf sb = STRBUF_INIT;
 +      if (strbuf_getcwd(&sb))
 +              die_errno(_("unable to get current working directory"));
 +      return strbuf_detach(&sb, NULL);
 +}