From: Junio C Hamano Date: Mon, 8 Aug 2016 21:21:36 +0000 (-0700) Subject: Merge branch 'nd/pack-ofs-4gb-limit' into maint X-Git-Tag: v2.9.3~39 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/aa9136a87e3ce299fe966da4686a571d6a528311?ds=inline;hp=-c Merge branch 'nd/pack-ofs-4gb-limit' into maint "git pack-objects" and "git index-pack" mostly operate with off_t when talking about the offset of objects in a packfile, but there were a handful of places that used "unsigned long" to hold that value, leading to an unintended truncation. * nd/pack-ofs-4gb-limit: fsck: use streaming interface for large blobs in pack pack-objects: do not truncate result in-pack object size on 32-bit systems index-pack: correct "offset" type in unpack_entry_data() index-pack: report correct bad object offsets even if they are large index-pack: correct "len" type in unpack_data() sha1_file.c: use type off_t* for object_info->disk_sizep pack-objects: pass length to check_pack_crc() without truncation --- aa9136a87e3ce299fe966da4686a571d6a528311 diff --combined builtin/cat-file.c index 618103fdee,13ed944d2d..2dfe6265f7 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@@ -131,7 -131,7 +131,7 @@@ struct expand_data unsigned char sha1[20]; enum object_type type; unsigned long size; - unsigned long disk_size; + off_t disk_size; const char *rest; unsigned char delta_base_sha1[20]; @@@ -154,13 -154,6 +154,13 @@@ * elements above, so you can retrieve the response from there. */ struct object_info info; + + /* + * This flag will be true if the requested batch format and options + * don't require us to call sha1_object_info, which can then be + * optimized out. + */ + unsigned skip_object_info : 1; }; static int is_atom(const char *atom, const char *s, int slen) @@@ -191,7 -184,7 +191,7 @@@ static void expand_atom(struct strbuf * if (data->mark_query) data->info.disk_sizep = &data->disk_size; else - strbuf_addf(sb, "%lu", data->disk_size); + strbuf_addf(sb, "%"PRIuMAX, (uintmax_t)data->disk_size); } else if (is_atom("rest", atom, len)) { if (data->mark_query) data->split_on_whitespace = 1; @@@ -265,8 -258,7 +265,8 @@@ static void batch_object_write(const ch { struct strbuf buf = STRBUF_INIT; - if (sha1_object_info_extended(data->sha1, &data->info, LOOKUP_REPLACE_OBJECT) < 0) { + if (!data->skip_object_info && + sha1_object_info_extended(data->sha1, &data->info, LOOKUP_REPLACE_OBJECT) < 0) { printf("%s missing\n", obj_name ? obj_name : sha1_to_hex(data->sha1)); fflush(stdout); return; @@@ -377,13 -369,6 +377,13 @@@ static int batch_objects(struct batch_o strbuf_expand(&buf, opt->format, expand_format, &data); data.mark_query = 0; + if (opt->all_objects) { + struct object_info empty; + memset(&empty, 0, sizeof(empty)); + if (!memcmp(&data.info, &empty, sizeof(empty))) + data.skip_object_info = 1; + } + /* * If we are printing out the object, then always fill in the type, * since we will want to decide whether or not to stream. @@@ -504,7 -489,6 +504,7 @@@ int cmd_cat_file(int argc, const char * git_config(git_cat_file_config, NULL); + batch.buffer_output = -1; argc = parse_options(argc, argv, prefix, options, cat_file_usage, 0); if (opt) { @@@ -528,9 -512,6 +528,9 @@@ usage_with_options(cat_file_usage, options); } + if (batch.buffer_output < 0) + batch.buffer_output = batch.all_objects; + if (batch.enabled) return batch_objects(&batch); diff --combined builtin/fsck.c index 3f27456883,b08bc8be24..9923b10898 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@@ -356,6 -356,10 +356,10 @@@ static int fsck_sha1(const unsigned cha static int fsck_obj_buffer(const unsigned char *sha1, enum object_type type, unsigned long size, void *buffer, int *eaten) { + /* + * Note, buffer may be NULL if type is OBJ_BLOB. See + * verify_packfile(), data_valid variable for details. + */ struct object *obj; obj = parse_object_buffer(sha1, type, size, buffer, eaten); if (!obj) { @@@ -493,12 -497,13 +497,12 @@@ static void fsck_object_dir(const char static int fsck_head_link(void) { - int flag; int null_is_error = 0; if (verbose) fprintf(stderr, "Checking HEAD link\n"); - head_points_at = resolve_ref_unsafe("HEAD", 0, head_oid.hash, &flag); + head_points_at = resolve_ref_unsafe("HEAD", 0, head_oid.hash, NULL); if (!head_points_at) { errors_found |= ERROR_REFS; return error("Invalid HEAD"); diff --combined builtin/pack-objects.c index 8f5e358e22,ac7a3a5895..f854ca4256 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@@ -341,15 -341,15 +341,15 @@@ static unsigned long write_no_reuse_obj } /* Return 0 if we will bust the pack-size limit */ - static unsigned long write_reuse_object(struct sha1file *f, struct object_entry *entry, - unsigned long limit, int usable_delta) + static off_t write_reuse_object(struct sha1file *f, struct object_entry *entry, + unsigned long limit, int usable_delta) { struct packed_git *p = entry->in_pack; struct pack_window *w_curs = NULL; struct revindex_entry *revidx; off_t offset; enum object_type type = entry->type; - unsigned long datalen; + off_t datalen; unsigned char header[10], dheader[10]; unsigned hdrlen; @@@ -415,11 -415,12 +415,12 @@@ } /* Return 0 if we will bust the pack-size limit */ - static unsigned long write_object(struct sha1file *f, - struct object_entry *entry, - off_t write_offset) + static off_t write_object(struct sha1file *f, + struct object_entry *entry, + off_t write_offset) { - unsigned long limit, len; + unsigned long limit; + off_t len; int usable_delta, to_reuse; if (!pack_to_stdout) @@@ -491,7 -492,7 +492,7 @@@ static enum write_one_status write_one( struct object_entry *e, off_t *offset) { - unsigned long size; + off_t size; int recursing; /* @@@ -835,7 -836,8 +836,7 @@@ static void write_pack_file(void * to preserve this property. */ if (stat(pack_tmp_name, &st) < 0) { - warning("failed to stat %s: %s", - pack_tmp_name, strerror(errno)); + warning_errno("failed to stat %s", pack_tmp_name); } else if (!last_mtime) { last_mtime = st.st_mtime; } else { @@@ -843,7 -845,8 +844,7 @@@ utb.actime = st.st_atime; utb.modtime = --last_mtime; if (utime(pack_tmp_name, &utb) < 0) - warning("failed utime() on %s: %s", - pack_tmp_name, strerror(errno)); + warning_errno("failed utime() on %s", pack_tmp_name); } strbuf_addf(&tmpname, "%s-", base_name); @@@ -1191,7 -1194,7 +1192,7 @@@ static void add_pbase_object(struct tre if (cmp < 0) return; if (name[cmplen] != '/') { - add_object_entry(entry.sha1, + add_object_entry(entry.oid->hash, object_type(entry.mode), fullname, 1); return; @@@ -1202,7 -1205,7 +1203,7 @@@ const char *down = name+cmplen+1; int downlen = name_cmp_len(down); - tree = pbase_tree_get(entry.sha1); + tree = pbase_tree_get(entry.oid->hash); if (!tree) return; init_tree_desc(&sub, tree->tree_data, tree->tree_size); diff --combined cache.h index c73becbf2d,ea64b51846..a4465cb6a2 --- a/cache.h +++ b/cache.h @@@ -367,8 -367,8 +367,8 @@@ extern void free_name_hash(struct index #define rename_cache_entry_at(pos, new_name) rename_index_entry_at(&the_index, (pos), (new_name)) #define remove_cache_entry_at(pos) remove_index_entry_at(&the_index, (pos)) #define remove_file_from_cache(path) remove_file_from_index(&the_index, (path)) -#define add_to_cache(path, st, flags) add_to_index(&the_index, (path), (st), (flags)) -#define add_file_to_cache(path, flags) add_file_to_index(&the_index, (path), (flags)) +#define add_to_cache(path, st, flags) add_to_index(&the_index, (path), (st), (flags), 0) +#define add_file_to_cache(path, flags) add_file_to_index(&the_index, (path), (flags), 0) #define refresh_cache(flags) refresh_index(&the_index, (flags), NULL, NULL, NULL) #define ce_match_stat(ce, st, options) ie_match_stat(&the_index, (ce), (st), (options)) #define ce_modified(ce, st, options) ie_modified(&the_index, (ce), (st), (options)) @@@ -581,8 -581,8 +581,8 @@@ extern int remove_file_from_index(struc #define ADD_CACHE_IGNORE_ERRORS 4 #define ADD_CACHE_IGNORE_REMOVAL 8 #define ADD_CACHE_INTENT 16 -extern int add_to_index(struct index_state *, const char *path, struct stat *, int flags); -extern int add_file_to_index(struct index_state *, const char *path, int flags); +extern int add_to_index(struct index_state *, const char *path, struct stat *, int flags, int force_mode); +extern int add_file_to_index(struct index_state *, const char *path, int flags, int force_mode); extern struct cache_entry *make_cache_entry(unsigned int mode, const unsigned char *sha1, const char *path, int stage, unsigned int refresh_options); extern int ce_same_name(const struct cache_entry *a, const struct cache_entry *b); extern void set_object_name_for_intent_to_add_entry(struct cache_entry *ce); @@@ -654,7 -654,6 +654,7 @@@ extern int warn_on_object_refname_ambig extern const char *apply_default_whitespace; extern const char *apply_default_ignorewhitespace; extern const char *git_attributes_file; +extern const char *git_hooks_path; extern int zlib_compression_level; extern int core_compression_level; extern int core_compression_seen; @@@ -808,14 -807,11 +808,14 @@@ extern void check_repository_format(voi */ extern const char *mkpath(const char *fmt, ...) __attribute__((format (printf, 1, 2))); extern const char *git_path(const char *fmt, ...) __attribute__((format (printf, 1, 2))); +extern const char *git_common_path(const char *fmt, ...) __attribute__((format (printf, 1, 2))); extern char *mksnpath(char *buf, size_t n, const char *fmt, ...) __attribute__((format (printf, 3, 4))); extern void strbuf_git_path(struct strbuf *sb, const char *fmt, ...) __attribute__((format (printf, 2, 3))); +extern void strbuf_git_common_path(struct strbuf *sb, const char *fmt, ...) + __attribute__((format (printf, 2, 3))); extern char *git_path_buf(struct strbuf *buf, const char *fmt, ...) __attribute__((format (printf, 2, 3))); extern void strbuf_git_path_submodule(struct strbuf *sb, const char *path, @@@ -1166,8 -1162,6 +1166,8 @@@ extern int get_sha1_blob(const char *st extern void maybe_die_on_misspelt_object_name(const char *name, const char *prefix); extern int get_sha1_with_context(const char *str, unsigned flags, unsigned char *sha1, struct object_context *orc); +extern int get_oid(const char *str, struct object_id *oid); + typedef int each_abbrev_fn(const unsigned char *sha1, void *); extern int for_each_abbrev(const char *prefix, each_abbrev_fn, void *); @@@ -1508,7 -1502,7 +1508,7 @@@ struct object_info /* Request */ enum object_type *typep; unsigned long *sizep; - unsigned long *disk_sizep; + off_t *disk_sizep; unsigned char *delta_base_sha1; struct strbuf *typename; @@@ -1772,14 -1766,14 +1772,14 @@@ void packet_trace_identity(const char * * return 0 if success, 1 - if addition of a file failed and * ADD_FILES_IGNORE_ERRORS was specified in flags */ -int add_files_to_cache(const char *prefix, const struct pathspec *pathspec, int flags); +int add_files_to_cache(const char *prefix, const struct pathspec *pathspec, int flags, int force_mode); /* diff.c */ extern int diff_auto_refresh_index; /* match-trees.c */ -void shift_tree(const unsigned char *, const unsigned char *, unsigned char *, int); -void shift_tree_by(const unsigned char *, const unsigned char *, unsigned char *, const char *); +void shift_tree(const struct object_id *, const struct object_id *, struct object_id *, int); +void shift_tree_by(const struct object_id *, const struct object_id *, struct object_id *, const char *); /* * whitespace rules. diff --combined sha1_file.c index d5e11217f5,cd9b560e70..cb571ac6e8 --- a/sha1_file.c +++ b/sha1_file.c @@@ -301,7 -301,7 +301,7 @@@ static int link_alt_odb_entry(const cha return -1; } } - if (!strcmp_icase(ent->base, normalized_objdir)) { + if (!fspathcmp(ent->base, normalized_objdir)) { free(ent); return -1; } @@@ -1107,8 -1107,9 +1107,8 @@@ unsigned char *use_pack(struct packed_g PROT_READ, MAP_PRIVATE, p->pack_fd, win->offset); if (win->base == MAP_FAILED) - die("packfile %s cannot be mapped: %s", - p->pack_name, - strerror(errno)); + die_errno("packfile %s cannot be mapped", + p->pack_name); if (!win->offset && win->len == p->pack_size && !p->do_not_close) close_pack_fd(p); @@@ -1278,8 -1279,8 +1278,8 @@@ static void prepare_packed_git_one(cha dir = opendir(path.buf); if (!dir) { if (errno != ENOENT) - error("unable to open object pack directory: %s: %s", - path.buf, strerror(errno)); + error_errno("unable to open object pack directory: %s", + path.buf); strbuf_release(&path); return; } @@@ -2281,7 -2282,7 +2281,7 @@@ void *unpack_entry(struct packed_git *p if (do_check_packed_object_crc && p->index_version > 1) { struct revindex_entry *revidx = find_pack_revindex(p, obj_offset); - unsigned long len = revidx[1].offset - obj_offset; + off_t len = revidx[1].offset - obj_offset; if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) { const unsigned char *sha1 = nth_packed_object_sha1(p, revidx->nr); @@@ -2983,7 -2984,7 +2983,7 @@@ int finalize_object_file(const char *tm unlink_or_warn(tmpfile); if (ret) { if (ret != EEXIST) { - return error("unable to write sha1 filename %s: %s", filename, strerror(ret)); + return error_errno("unable to write sha1 filename %s", filename); } /* FIXME!!! Collision check here ? */ } @@@ -2997,7 -2998,7 +2997,7 @@@ out static int write_buffer(int fd, const void *buf, size_t len) { if (write_in_full(fd, buf, len) < 0) - return error("file write error (%s)", strerror(errno)); + return error_errno("file write error"); return 0; } @@@ -3080,7 -3081,7 +3080,7 @@@ static int write_loose_object(const uns if (errno == EACCES) return error("insufficient permission for adding an object to repository database %s", get_object_directory()); else - return error("unable to create temporary file: %s", strerror(errno)); + return error_errno("unable to create temporary file"); } /* Set it up */ @@@ -3125,7 -3126,8 +3125,7 @@@ utb.actime = mtime; utb.modtime = mtime; if (utime(tmp_file.buf, &utb) < 0) - warning("failed utime() on %s: %s", - tmp_file.buf, strerror(errno)); + warning_errno("failed utime() on %s", tmp_file.buf); } return finalize_object_file(tmp_file.buf, filename); @@@ -3358,7 -3360,7 +3358,7 @@@ static int index_core(unsigned char *sh if (size == read_in_full(fd, buf, size)) ret = index_mem(sha1, buf, size, type, path, flags); else - ret = error("short read %s", strerror(errno)); + ret = error_errno("short read"); free(buf); } else { void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); @@@ -3423,14 -3425,18 +3423,14 @@@ int index_path(unsigned char *sha1, con case S_IFREG: fd = open(path, O_RDONLY); if (fd < 0) - return error("open(\"%s\"): %s", path, - strerror(errno)); + return error_errno("open(\"%s\")", path); if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0) return error("%s: failed to insert into database", path); break; case S_IFLNK: - if (strbuf_readlink(&sb, path, st->st_size)) { - char *errstr = strerror(errno); - return error("readlink(\"%s\"): %s", path, - errstr); - } + if (strbuf_readlink(&sb, path, st->st_size)) + return error_errno("readlink(\"%s\")", path); if (!(flags & HASH_WRITE_OBJECT)) hash_sha1_file(sb.buf, sb.len, blob_type, sha1); else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1)) @@@ -3486,7 -3492,7 +3486,7 @@@ static int for_each_file_in_obj_subdir( if (!dir) { if (errno == ENOENT) return 0; - return error("unable to open %s: %s", path->buf, strerror(errno)); + return error_errno("unable to open %s", path->buf); } while ((de = readdir(dir))) {