Merge branch 'nd/pack-ofs-4gb-limit' into maint
authorJunio C Hamano <gitster@pobox.com>
Mon, 8 Aug 2016 21:21:36 +0000 (14:21 -0700)
committerJunio C Hamano <gitster@pobox.com>
Mon, 8 Aug 2016 21:21:36 +0000 (14:21 -0700)
"git pack-objects" and "git index-pack" mostly operate with off_t
when talking about the offset of objects in a packfile, but there
were a handful of places that used "unsigned long" to hold that
value, leading to an unintended truncation.

* nd/pack-ofs-4gb-limit:
fsck: use streaming interface for large blobs in pack
pack-objects: do not truncate result in-pack object size on 32-bit systems
index-pack: correct "offset" type in unpack_entry_data()
index-pack: report correct bad object offsets even if they are large
index-pack: correct "len" type in unpack_data()
sha1_file.c: use type off_t* for object_info->disk_sizep
pack-objects: pass length to check_pack_crc() without truncation

1  2 
builtin/cat-file.c
builtin/fsck.c
builtin/pack-objects.c
cache.h
sha1_file.c
diff --combined builtin/cat-file.c
index 618103fdeeb7f7b35911ebd12d0d811cc4a5297a,13ed944d2d2b158713167dbf1822195445ca07cf..2dfe6265f7df6099b51645fa67dbeeb3f4f567a4
@@@ -131,7 -131,7 +131,7 @@@ struct expand_data 
        unsigned char sha1[20];
        enum object_type type;
        unsigned long size;
-       unsigned long disk_size;
+       off_t disk_size;
        const char *rest;
        unsigned char delta_base_sha1[20];
  
         * elements above, so you can retrieve the response from there.
         */
        struct object_info info;
 +
 +      /*
 +       * This flag will be true if the requested batch format and options
 +       * don't require us to call sha1_object_info, which can then be
 +       * optimized out.
 +       */
 +      unsigned skip_object_info : 1;
  };
  
  static int is_atom(const char *atom, const char *s, int slen)
@@@ -191,7 -184,7 +191,7 @@@ static void expand_atom(struct strbuf *
                if (data->mark_query)
                        data->info.disk_sizep = &data->disk_size;
                else
-                       strbuf_addf(sb, "%lu", data->disk_size);
+                       strbuf_addf(sb, "%"PRIuMAX, (uintmax_t)data->disk_size);
        } else if (is_atom("rest", atom, len)) {
                if (data->mark_query)
                        data->split_on_whitespace = 1;
@@@ -265,8 -258,7 +265,8 @@@ static void batch_object_write(const ch
  {
        struct strbuf buf = STRBUF_INIT;
  
 -      if (sha1_object_info_extended(data->sha1, &data->info, LOOKUP_REPLACE_OBJECT) < 0) {
 +      if (!data->skip_object_info &&
 +          sha1_object_info_extended(data->sha1, &data->info, LOOKUP_REPLACE_OBJECT) < 0) {
                printf("%s missing\n", obj_name ? obj_name : sha1_to_hex(data->sha1));
                fflush(stdout);
                return;
@@@ -377,13 -369,6 +377,13 @@@ static int batch_objects(struct batch_o
        strbuf_expand(&buf, opt->format, expand_format, &data);
        data.mark_query = 0;
  
 +      if (opt->all_objects) {
 +              struct object_info empty;
 +              memset(&empty, 0, sizeof(empty));
 +              if (!memcmp(&data.info, &empty, sizeof(empty)))
 +                      data.skip_object_info = 1;
 +      }
 +
        /*
         * If we are printing out the object, then always fill in the type,
         * since we will want to decide whether or not to stream.
@@@ -504,7 -489,6 +504,7 @@@ int cmd_cat_file(int argc, const char *
  
        git_config(git_cat_file_config, NULL);
  
 +      batch.buffer_output = -1;
        argc = parse_options(argc, argv, prefix, options, cat_file_usage, 0);
  
        if (opt) {
                usage_with_options(cat_file_usage, options);
        }
  
 +      if (batch.buffer_output < 0)
 +              batch.buffer_output = batch.all_objects;
 +
        if (batch.enabled)
                return batch_objects(&batch);
  
diff --combined builtin/fsck.c
index 3f27456883ede5efa2a57cfa9e2dcc889e952338,b08bc8be24ede7096f8454ac94411e0e67cac728..9923b1089898eaee82fa394dc758b5b2d62c2649
@@@ -356,6 -356,10 +356,10 @@@ static int fsck_sha1(const unsigned cha
  static int fsck_obj_buffer(const unsigned char *sha1, enum object_type type,
                           unsigned long size, void *buffer, int *eaten)
  {
+       /*
+        * Note, buffer may be NULL if type is OBJ_BLOB. See
+        * verify_packfile(), data_valid variable for details.
+        */
        struct object *obj;
        obj = parse_object_buffer(sha1, type, size, buffer, eaten);
        if (!obj) {
@@@ -493,12 -497,13 +497,12 @@@ static void fsck_object_dir(const char 
  
  static int fsck_head_link(void)
  {
 -      int flag;
        int null_is_error = 0;
  
        if (verbose)
                fprintf(stderr, "Checking HEAD link\n");
  
 -      head_points_at = resolve_ref_unsafe("HEAD", 0, head_oid.hash, &flag);
 +      head_points_at = resolve_ref_unsafe("HEAD", 0, head_oid.hash, NULL);
        if (!head_points_at) {
                errors_found |= ERROR_REFS;
                return error("Invalid HEAD");
diff --combined builtin/pack-objects.c
index 8f5e358e22b40ff2e162cc7037339ad357622650,ac7a3a589522324452db451b444daafc861952ae..f854ca42562dd062c1294455229e46b6206f7a0d
@@@ -341,15 -341,15 +341,15 @@@ static unsigned long write_no_reuse_obj
  }
  
  /* Return 0 if we will bust the pack-size limit */
- static unsigned long write_reuse_object(struct sha1file *f, struct object_entry *entry,
-                                       unsigned long limit, int usable_delta)
+ static off_t write_reuse_object(struct sha1file *f, struct object_entry *entry,
+                               unsigned long limit, int usable_delta)
  {
        struct packed_git *p = entry->in_pack;
        struct pack_window *w_curs = NULL;
        struct revindex_entry *revidx;
        off_t offset;
        enum object_type type = entry->type;
-       unsigned long datalen;
+       off_t datalen;
        unsigned char header[10], dheader[10];
        unsigned hdrlen;
  
  }
  
  /* Return 0 if we will bust the pack-size limit */
- static unsigned long write_object(struct sha1file *f,
-                                 struct object_entry *entry,
-                                 off_t write_offset)
+ static off_t write_object(struct sha1file *f,
+                         struct object_entry *entry,
+                         off_t write_offset)
  {
-       unsigned long limit, len;
+       unsigned long limit;
+       off_t len;
        int usable_delta, to_reuse;
  
        if (!pack_to_stdout)
@@@ -491,7 -492,7 +492,7 @@@ static enum write_one_status write_one(
                                       struct object_entry *e,
                                       off_t *offset)
  {
-       unsigned long size;
+       off_t size;
        int recursing;
  
        /*
@@@ -835,7 -836,8 +836,7 @@@ static void write_pack_file(void
                         * to preserve this property.
                         */
                        if (stat(pack_tmp_name, &st) < 0) {
 -                              warning("failed to stat %s: %s",
 -                                      pack_tmp_name, strerror(errno));
 +                              warning_errno("failed to stat %s", pack_tmp_name);
                        } else if (!last_mtime) {
                                last_mtime = st.st_mtime;
                        } else {
                                utb.actime = st.st_atime;
                                utb.modtime = --last_mtime;
                                if (utime(pack_tmp_name, &utb) < 0)
 -                                      warning("failed utime() on %s: %s",
 -                                              pack_tmp_name, strerror(errno));
 +                                      warning_errno("failed utime() on %s", pack_tmp_name);
                        }
  
                        strbuf_addf(&tmpname, "%s-", base_name);
@@@ -1191,7 -1194,7 +1192,7 @@@ static void add_pbase_object(struct tre
                if (cmp < 0)
                        return;
                if (name[cmplen] != '/') {
 -                      add_object_entry(entry.sha1,
 +                      add_object_entry(entry.oid->hash,
                                         object_type(entry.mode),
                                         fullname, 1);
                        return;
                        const char *down = name+cmplen+1;
                        int downlen = name_cmp_len(down);
  
 -                      tree = pbase_tree_get(entry.sha1);
 +                      tree = pbase_tree_get(entry.oid->hash);
                        if (!tree)
                                return;
                        init_tree_desc(&sub, tree->tree_data, tree->tree_size);
diff --combined cache.h
index c73becbf2dc8ae939116617e36c76afa60857cef,ea64b51846f9f974c4462ab5ec38ebd6c941cb20..a4465cb6a2ab3da4b8b73d45cc6500275b8abecf
+++ b/cache.h
@@@ -367,8 -367,8 +367,8 @@@ extern void free_name_hash(struct index
  #define rename_cache_entry_at(pos, new_name) rename_index_entry_at(&the_index, (pos), (new_name))
  #define remove_cache_entry_at(pos) remove_index_entry_at(&the_index, (pos))
  #define remove_file_from_cache(path) remove_file_from_index(&the_index, (path))
 -#define add_to_cache(path, st, flags) add_to_index(&the_index, (path), (st), (flags))
 -#define add_file_to_cache(path, flags) add_file_to_index(&the_index, (path), (flags))
 +#define add_to_cache(path, st, flags) add_to_index(&the_index, (path), (st), (flags), 0)
 +#define add_file_to_cache(path, flags) add_file_to_index(&the_index, (path), (flags), 0)
  #define refresh_cache(flags) refresh_index(&the_index, (flags), NULL, NULL, NULL)
  #define ce_match_stat(ce, st, options) ie_match_stat(&the_index, (ce), (st), (options))
  #define ce_modified(ce, st, options) ie_modified(&the_index, (ce), (st), (options))
@@@ -581,8 -581,8 +581,8 @@@ extern int remove_file_from_index(struc
  #define ADD_CACHE_IGNORE_ERRORS       4
  #define ADD_CACHE_IGNORE_REMOVAL 8
  #define ADD_CACHE_INTENT 16
 -extern int add_to_index(struct index_state *, const char *path, struct stat *, int flags);
 -extern int add_file_to_index(struct index_state *, const char *path, int flags);
 +extern int add_to_index(struct index_state *, const char *path, struct stat *, int flags, int force_mode);
 +extern int add_file_to_index(struct index_state *, const char *path, int flags, int force_mode);
  extern struct cache_entry *make_cache_entry(unsigned int mode, const unsigned char *sha1, const char *path, int stage, unsigned int refresh_options);
  extern int ce_same_name(const struct cache_entry *a, const struct cache_entry *b);
  extern void set_object_name_for_intent_to_add_entry(struct cache_entry *ce);
@@@ -654,7 -654,6 +654,7 @@@ extern int warn_on_object_refname_ambig
  extern const char *apply_default_whitespace;
  extern const char *apply_default_ignorewhitespace;
  extern const char *git_attributes_file;
 +extern const char *git_hooks_path;
  extern int zlib_compression_level;
  extern int core_compression_level;
  extern int core_compression_seen;
@@@ -808,14 -807,11 +808,14 @@@ extern void check_repository_format(voi
   */
  extern const char *mkpath(const char *fmt, ...) __attribute__((format (printf, 1, 2)));
  extern const char *git_path(const char *fmt, ...) __attribute__((format (printf, 1, 2)));
 +extern const char *git_common_path(const char *fmt, ...) __attribute__((format (printf, 1, 2)));
  
  extern char *mksnpath(char *buf, size_t n, const char *fmt, ...)
        __attribute__((format (printf, 3, 4)));
  extern void strbuf_git_path(struct strbuf *sb, const char *fmt, ...)
        __attribute__((format (printf, 2, 3)));
 +extern void strbuf_git_common_path(struct strbuf *sb, const char *fmt, ...)
 +      __attribute__((format (printf, 2, 3)));
  extern char *git_path_buf(struct strbuf *buf, const char *fmt, ...)
        __attribute__((format (printf, 2, 3)));
  extern void strbuf_git_path_submodule(struct strbuf *sb, const char *path,
@@@ -1166,8 -1162,6 +1166,8 @@@ extern int get_sha1_blob(const char *st
  extern void maybe_die_on_misspelt_object_name(const char *name, const char *prefix);
  extern int get_sha1_with_context(const char *str, unsigned flags, unsigned char *sha1, struct object_context *orc);
  
 +extern int get_oid(const char *str, struct object_id *oid);
 +
  typedef int each_abbrev_fn(const unsigned char *sha1, void *);
  extern int for_each_abbrev(const char *prefix, each_abbrev_fn, void *);
  
@@@ -1508,7 -1502,7 +1508,7 @@@ struct object_info 
        /* Request */
        enum object_type *typep;
        unsigned long *sizep;
-       unsigned long *disk_sizep;
+       off_t *disk_sizep;
        unsigned char *delta_base_sha1;
        struct strbuf *typename;
  
@@@ -1772,14 -1766,14 +1772,14 @@@ void packet_trace_identity(const char *
   * return 0 if success, 1 - if addition of a file failed and
   * ADD_FILES_IGNORE_ERRORS was specified in flags
   */
 -int add_files_to_cache(const char *prefix, const struct pathspec *pathspec, int flags);
 +int add_files_to_cache(const char *prefix, const struct pathspec *pathspec, int flags, int force_mode);
  
  /* diff.c */
  extern int diff_auto_refresh_index;
  
  /* match-trees.c */
 -void shift_tree(const unsigned char *, const unsigned char *, unsigned char *, int);
 -void shift_tree_by(const unsigned char *, const unsigned char *, unsigned char *, const char *);
 +void shift_tree(const struct object_id *, const struct object_id *, struct object_id *, int);
 +void shift_tree_by(const struct object_id *, const struct object_id *, struct object_id *, const char *);
  
  /*
   * whitespace rules.
diff --combined sha1_file.c
index d5e11217f523018008b3c4861d5d70068de14c72,cd9b560e70bb6222d3977d16d9c7f1da101fd571..cb571ac6e8ed0657e39b346b41961caa8cc825be
@@@ -301,7 -301,7 +301,7 @@@ static int link_alt_odb_entry(const cha
                        return -1;
                }
        }
 -      if (!strcmp_icase(ent->base, normalized_objdir)) {
 +      if (!fspathcmp(ent->base, normalized_objdir)) {
                free(ent);
                return -1;
        }
@@@ -1107,8 -1107,9 +1107,8 @@@ unsigned char *use_pack(struct packed_g
                                PROT_READ, MAP_PRIVATE,
                                p->pack_fd, win->offset);
                        if (win->base == MAP_FAILED)
 -                              die("packfile %s cannot be mapped: %s",
 -                                      p->pack_name,
 -                                      strerror(errno));
 +                              die_errno("packfile %s cannot be mapped",
 +                                        p->pack_name);
                        if (!win->offset && win->len == p->pack_size
                                && !p->do_not_close)
                                close_pack_fd(p);
@@@ -1278,8 -1279,8 +1278,8 @@@ static void prepare_packed_git_one(cha
        dir = opendir(path.buf);
        if (!dir) {
                if (errno != ENOENT)
 -                      error("unable to open object pack directory: %s: %s",
 -                            path.buf, strerror(errno));
 +                      error_errno("unable to open object pack directory: %s",
 +                                  path.buf);
                strbuf_release(&path);
                return;
        }
@@@ -2281,7 -2282,7 +2281,7 @@@ void *unpack_entry(struct packed_git *p
  
                if (do_check_packed_object_crc && p->index_version > 1) {
                        struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
-                       unsigned long len = revidx[1].offset - obj_offset;
+                       off_t len = revidx[1].offset - obj_offset;
                        if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
                                const unsigned char *sha1 =
                                        nth_packed_object_sha1(p, revidx->nr);
@@@ -2983,7 -2984,7 +2983,7 @@@ int finalize_object_file(const char *tm
        unlink_or_warn(tmpfile);
        if (ret) {
                if (ret != EEXIST) {
 -                      return error("unable to write sha1 filename %s: %s", filename, strerror(ret));
 +                      return error_errno("unable to write sha1 filename %s", filename);
                }
                /* FIXME!!! Collision check here ? */
        }
@@@ -2997,7 -2998,7 +2997,7 @@@ out
  static int write_buffer(int fd, const void *buf, size_t len)
  {
        if (write_in_full(fd, buf, len) < 0)
 -              return error("file write error (%s)", strerror(errno));
 +              return error_errno("file write error");
        return 0;
  }
  
@@@ -3080,7 -3081,7 +3080,7 @@@ static int write_loose_object(const uns
                if (errno == EACCES)
                        return error("insufficient permission for adding an object to repository database %s", get_object_directory());
                else
 -                      return error("unable to create temporary file: %s", strerror(errno));
 +                      return error_errno("unable to create temporary file");
        }
  
        /* Set it up */
                utb.actime = mtime;
                utb.modtime = mtime;
                if (utime(tmp_file.buf, &utb) < 0)
 -                      warning("failed utime() on %s: %s",
 -                              tmp_file.buf, strerror(errno));
 +                      warning_errno("failed utime() on %s", tmp_file.buf);
        }
  
        return finalize_object_file(tmp_file.buf, filename);
@@@ -3358,7 -3360,7 +3358,7 @@@ static int index_core(unsigned char *sh
                if (size == read_in_full(fd, buf, size))
                        ret = index_mem(sha1, buf, size, type, path, flags);
                else
 -                      ret = error("short read %s", strerror(errno));
 +                      ret = error_errno("short read");
                free(buf);
        } else {
                void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
@@@ -3423,14 -3425,18 +3423,14 @@@ int index_path(unsigned char *sha1, con
        case S_IFREG:
                fd = open(path, O_RDONLY);
                if (fd < 0)
 -                      return error("open(\"%s\"): %s", path,
 -                                   strerror(errno));
 +                      return error_errno("open(\"%s\")", path);
                if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0)
                        return error("%s: failed to insert into database",
                                     path);
                break;
        case S_IFLNK:
 -              if (strbuf_readlink(&sb, path, st->st_size)) {
 -                      char *errstr = strerror(errno);
 -                      return error("readlink(\"%s\"): %s", path,
 -                                   errstr);
 -              }
 +              if (strbuf_readlink(&sb, path, st->st_size))
 +                      return error_errno("readlink(\"%s\")", path);
                if (!(flags & HASH_WRITE_OBJECT))
                        hash_sha1_file(sb.buf, sb.len, blob_type, sha1);
                else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1))
@@@ -3486,7 -3492,7 +3486,7 @@@ static int for_each_file_in_obj_subdir(
        if (!dir) {
                if (errno == ENOENT)
                        return 0;
 -              return error("unable to open %s: %s", path->buf, strerror(errno));
 +              return error_errno("unable to open %s", path->buf);
        }
  
        while ((de = readdir(dir))) {