From: Junio C Hamano Date: Thu, 11 Dec 2008 08:36:31 +0000 (-0800) Subject: Merge branch 'maint' X-Git-Tag: v1.6.1-rc3~6 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/de0db422782ddaf7754ac5b03fdc6dc5de1a9ae4?ds=inline;hp=-c Merge branch 'maint' * maint: fsck: reduce stack footprint make sure packs to be replaced are closed beforehand --- de0db422782ddaf7754ac5b03fdc6dc5de1a9ae4 diff --combined builtin-fsck.c index afded5e68d,30971ce0ad..297b2c41c6 --- a/builtin-fsck.c +++ b/builtin-fsck.c @@@ -64,11 -64,11 +64,11 @@@ static int fsck_error_func(struct objec return (type == FSCK_WARN) ? 0 : 1; } + static struct object_array pending; + static int mark_object(struct object *obj, int type, void *data) { - struct tree *tree = NULL; struct object *parent = data; - int result; if (!obj) { printf("broken link from %7s %s\n", @@@ -96,6 -96,20 +96,20 @@@ return 1; } + add_object_array(obj, (void *) parent, &pending); + return 0; + } + + static void mark_object_reachable(struct object *obj) + { + mark_object(obj, OBJ_ANY, 0); + } + + static int traverse_one_object(struct object *obj, struct object *parent) + { + int result; + struct tree *tree = NULL; + if (obj->type == OBJ_TREE) { obj->parsed = 0; tree = (struct tree *)obj; @@@ -107,15 -121,22 +121,22 @@@ free(tree->buffer); tree->buffer = NULL; } - if (result < 0) - result = 1; - return result; } - static void mark_object_reachable(struct object *obj) + static int traverse_reachable(void) { - mark_object(obj, OBJ_ANY, 0); + int result = 0; + while (pending.nr) { + struct object_array_entry *entry; + struct object *obj, *parent; + + entry = pending.objects + --pending.nr; + obj = entry->item; + parent = (struct object *) entry->name; + result |= traverse_one_object(obj, parent); + } + return !!result; } static int mark_used(struct object *obj, int type, void *data) @@@ -201,16 -222,12 +222,16 @@@ static void check_unreachable_object(st char *buf = read_sha1_file(obj->sha1, &type, &size); if (buf) { - fwrite(buf, size, 1, f); + if (fwrite(buf, size, 1, f) != 1) + die("Could not write %s: %s", + filename, strerror(errno)); free(buf); } } else fprintf(f, "%s\n", sha1_to_hex(obj->sha1)); - fclose(f); + if (fclose(f)) + die("Could not finish %s: %s", + filename, strerror(errno)); } return; } @@@ -237,6 -254,9 +258,9 @@@ static void check_connectivity(void { int i, max; + /* Traverse the pending reachable objects */ + traverse_reachable(); + /* Look up all the requirements, warn about missing objects.. */ max = get_max_object_index(); if (verbose) diff --combined builtin-pack-objects.c index 67eefa2932,fb5e14d56e..cedef52fd3 --- a/builtin-pack-objects.c +++ b/builtin-pack-objects.c @@@ -286,7 -286,6 +286,7 @@@ static unsigned long write_object(struc */ if (!to_reuse) { + no_reuse: if (!usable_delta) { buf = read_sha1_file(entry->idx.sha1, &type, &size); if (!buf) @@@ -368,60 -367,46 +368,60 @@@ struct revindex_entry *revidx; off_t offset; - if (entry->delta) { + if (entry->delta) type = (allow_ofs_delta && entry->delta->idx.offset) ? OBJ_OFS_DELTA : OBJ_REF_DELTA; - reused_delta++; - } hdrlen = encode_header(type, entry->size, header); + offset = entry->in_pack_offset; revidx = find_pack_revindex(p, offset); datalen = revidx[1].offset - offset; if (!pack_to_stdout && p->index_version > 1 && - check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) - die("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1)); + check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) { + error("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1)); + unuse_pack(&w_curs); + goto no_reuse; + } + offset += entry->in_pack_header_size; datalen -= entry->in_pack_header_size; + if (!pack_to_stdout && p->index_version == 1 && + check_pack_inflate(p, &w_curs, offset, datalen, entry->size)) { + error("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1)); + unuse_pack(&w_curs); + goto no_reuse; + } + if (type == OBJ_OFS_DELTA) { off_t ofs = entry->idx.offset - entry->delta->idx.offset; unsigned pos = sizeof(dheader) - 1; dheader[pos] = ofs & 127; while (ofs >>= 7) dheader[--pos] = 128 | (--ofs & 127); - if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) + if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) { + unuse_pack(&w_curs); return 0; + } sha1write(f, header, hdrlen); sha1write(f, dheader + pos, sizeof(dheader) - pos); hdrlen += sizeof(dheader) - pos; + reused_delta++; } else if (type == OBJ_REF_DELTA) { - if (limit && hdrlen + 20 + datalen + 20 >= limit) + if (limit && hdrlen + 20 + datalen + 20 >= limit) { + unuse_pack(&w_curs); return 0; + } sha1write(f, header, hdrlen); sha1write(f, entry->delta->idx.sha1, 20); hdrlen += 20; + reused_delta++; } else { - if (limit && hdrlen + datalen + 20 >= limit) + if (limit && hdrlen + datalen + 20 >= limit) { + unuse_pack(&w_curs); return 0; + } sha1write(f, header, hdrlen); } - - if (!pack_to_stdout && p->index_version == 1 && - check_pack_inflate(p, &w_curs, offset, datalen, entry->size)) - die("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1)); copy_pack_data(f, p, &w_curs, offset, datalen); unuse_pack(&w_curs); reused++; @@@ -535,6 -520,7 +535,7 @@@ static void write_pack_file(void snprintf(tmpname, sizeof(tmpname), "%s-%s.pack", base_name, sha1_to_hex(sha1)); + free_pack_by_name(tmpname); if (adjust_perm(pack_tmp_name, mode)) die("unable to make temporary pack file readable: %s", strerror(errno)); @@@ -1031,11 -1017,9 +1032,11 @@@ static void check_object(struct object_ * We want in_pack_type even if we do not reuse delta * since non-delta representations could still be reused. */ - used = unpack_object_header_gently(buf, avail, + used = unpack_object_header_buffer(buf, avail, &entry->in_pack_type, &entry->size); + if (used == 0) + goto give_up; /* * Determine if this is a delta and if so whether we can @@@ -1047,8 -1031,6 +1048,8 @@@ /* Not a delta hence we've already got all we need. */ entry->type = entry->in_pack_type; entry->in_pack_header_size = used; + if (entry->type < OBJ_COMMIT || entry->type > OBJ_BLOB) + goto give_up; unuse_pack(&w_curs); return; case OBJ_REF_DELTA: @@@ -1065,25 -1047,19 +1066,25 @@@ ofs = c & 127; while (c & 128) { ofs += 1; - if (!ofs || MSB(ofs, 7)) - die("delta base offset overflow in pack for %s", - sha1_to_hex(entry->idx.sha1)); + if (!ofs || MSB(ofs, 7)) { + error("delta base offset overflow in pack for %s", + sha1_to_hex(entry->idx.sha1)); + goto give_up; + } c = buf[used_0++]; ofs = (ofs << 7) + (c & 127); } - if (ofs >= entry->in_pack_offset) - die("delta base offset out of bound for %s", - sha1_to_hex(entry->idx.sha1)); ofs = entry->in_pack_offset - ofs; + if (ofs <= 0 || ofs >= entry->in_pack_offset) { + error("delta base offset out of bound for %s", + sha1_to_hex(entry->idx.sha1)); + goto give_up; + } if (reuse_delta && !entry->preferred_base) { struct revindex_entry *revidx; revidx = find_pack_revindex(p, ofs); + if (!revidx) + goto give_up; base_ref = nth_packed_object_sha1(p, revidx->nr); } entry->in_pack_header_size = used + used_0; @@@ -1103,7 -1079,6 +1104,7 @@@ */ entry->type = entry->in_pack_type; entry->delta = base_entry; + entry->delta_size = entry->size; entry->delta_sibling = base_entry->delta_child; base_entry->delta_child = entry; unuse_pack(&w_curs); @@@ -1118,8 -1093,6 +1119,8 @@@ */ entry->size = get_size_from_delta(p, &w_curs, entry->in_pack_offset + entry->in_pack_header_size); + if (entry->size == 0) + goto give_up; unuse_pack(&w_curs); return; } @@@ -1129,7 -1102,6 +1130,7 @@@ * with sha1_object_info() to find about the object type * at this point... */ + give_up: unuse_pack(&w_curs); } @@@ -1412,10 -1384,12 +1413,10 @@@ static void find_deltas(struct object_e int window, int depth, unsigned *processed) { uint32_t i, idx = 0, count = 0; - unsigned int array_size = window * sizeof(struct unpacked); struct unpacked *array; unsigned long mem_usage = 0; - array = xmalloc(array_size); - memset(array, 0, array_size); + array = xcalloc(window, sizeof(struct unpacked)); for (;;) { struct object_entry *entry; @@@ -1741,16 -1715,6 +1742,16 @@@ static void prepare_pack(int window, in get_object_details(); + /* + * If we're locally repacking then we need to be doubly careful + * from now on in order to make sure no stealth corruption gets + * propagated to the new pack. Clients receiving streamed packs + * should validate everything they get anyway so no need to incur + * the additional cost here in that case. + */ + if (!pack_to_stdout) + do_check_packed_object_crc = 1; + if (!nr_objects || !window || !depth) return; @@@ -1777,14 -1741,6 +1778,14 @@@ if (entry->type < 0) die("unable to get type of object %s", sha1_to_hex(entry->idx.sha1)); + } else { + if (entry->type < 0) { + /* + * This object is not found, but we + * don't have to include it anyway. + */ + continue; + } } delta_list[n++] = entry; diff --combined cache.h index f15b3fc906,42f2f2754b..231c06d772 --- a/cache.h +++ b/cache.h @@@ -6,14 -6,8 +6,14 @@@ #include "hash.h" #include SHA1_HEADER -#include +#ifndef git_SHA_CTX +#define git_SHA_CTX SHA_CTX +#define git_SHA1_Init SHA1_Init +#define git_SHA1_Update SHA1_Update +#define git_SHA1_Final SHA1_Final +#endif +#include #if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200 #define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11) #endif @@@ -115,26 -109,6 +115,26 @@@ struct ondisk_cache_entry char name[FLEX_ARRAY]; /* more */ }; +/* + * This struct is used when CE_EXTENDED bit is 1 + * The struct must match ondisk_cache_entry exactly from + * ctime till flags + */ +struct ondisk_cache_entry_extended { + struct cache_time ctime; + struct cache_time mtime; + unsigned int dev; + unsigned int ino; + unsigned int mode; + unsigned int uid; + unsigned int gid; + unsigned int size; + unsigned char sha1[20]; + unsigned short flags; + unsigned short flags2; + char name[FLEX_ARRAY]; /* more */ +}; + struct cache_entry { unsigned int ce_ctime; unsigned int ce_mtime; @@@ -152,19 -126,10 +152,19 @@@ #define CE_NAMEMASK (0x0fff) #define CE_STAGEMASK (0x3000) +#define CE_EXTENDED (0x4000) #define CE_VALID (0x8000) #define CE_STAGESHIFT 12 -/* In-memory only */ +/* + * Range 0xFFFF0000 in ce_flags is divided into + * two parts: in-memory flags and on-disk ones. + * Flags in CE_EXTENDED_FLAGS will get saved on-disk + * if you want to save a new flag, add it in + * CE_EXTENDED_FLAGS + * + * In-memory only flags + */ #define CE_UPDATE (0x10000) #define CE_REMOVE (0x20000) #define CE_UPTODATE (0x40000) @@@ -173,25 -138,6 +173,25 @@@ #define CE_HASHED (0x100000) #define CE_UNHASHED (0x200000) +/* + * Extended on-disk flags + */ +#define CE_INTENT_TO_ADD 0x20000000 +/* CE_EXTENDED2 is for future extension */ +#define CE_EXTENDED2 0x80000000 + +#define CE_EXTENDED_FLAGS (CE_INTENT_TO_ADD) + +/* + * Safeguard to avoid saving wrong flags: + * - CE_EXTENDED2 won't get saved until its semantic is known + * - Bits in 0x0000FFFF have been saved in ce_flags already + * - Bits in 0x003F0000 are currently in-memory flags + */ +#if CE_EXTENDED_FLAGS & 0x803FFFFF +#error "CE_EXTENDED_FLAGS out of range" +#endif + /* * Copy the sha1 and stat state of a cache entry from one to * another. But we never change the name, or the hash state! @@@ -224,9 -170,7 +224,9 @@@ static inline size_t ce_namelen(const s } #define ce_size(ce) cache_entry_size(ce_namelen(ce)) -#define ondisk_ce_size(ce) ondisk_cache_entry_size(ce_namelen(ce)) +#define ondisk_ce_size(ce) (((ce)->ce_flags & CE_EXTENDED) ? \ + ondisk_cache_entry_extended_size(ce_namelen(ce)) : \ + ondisk_cache_entry_size(ce_namelen(ce))) #define ce_stage(ce) ((CE_STAGEMASK & (ce)->ce_flags) >> CE_STAGESHIFT) #define ce_uptodate(ce) ((ce)->ce_flags & CE_UPTODATE) #define ce_mark_uptodate(ce) ((ce)->ce_flags |= CE_UPTODATE) @@@ -269,10 -213,8 +269,10 @@@ static inline int ce_to_dtype(const str (S_ISREG(mode) ? (S_IFREG | ce_permissions(mode)) : \ S_ISLNK(mode) ? S_IFLNK : S_ISDIR(mode) ? S_IFDIR : S_IFGITLINK) -#define cache_entry_size(len) ((offsetof(struct cache_entry,name) + (len) + 8) & ~7) -#define ondisk_cache_entry_size(len) ((offsetof(struct ondisk_cache_entry,name) + (len) + 8) & ~7) +#define flexible_size(STRUCT,len) ((offsetof(struct STRUCT,name) + (len) + 8) & ~7) +#define cache_entry_size(len) flexible_size(cache_entry,len) +#define ondisk_cache_entry_size(len) flexible_size(ondisk_cache_entry,len) +#define ondisk_cache_entry_extended_size(len) flexible_size(ondisk_cache_entry_extended,len) struct index_state { struct cache_entry **cache; @@@ -313,7 -255,6 +313,7 @@@ static inline void remove_name_hash(str #define read_cache() read_index(&the_index) #define read_cache_from(path) read_index_from(&the_index, (path)) +#define read_cache_preload(pathspec) read_index_preload(&the_index, (pathspec)) #define is_cache_unborn() is_index_unborn(&the_index) #define read_cache_unmerged() read_index_unmerged(&the_index) #define write_cache(newfd, cache, entries) write_index(&the_index, (newfd)) @@@ -373,7 -314,6 +373,7 @@@ extern int is_bare_repository(void) extern int is_inside_git_dir(void); extern char *git_work_tree_cfg; extern int is_inside_work_tree(void); +extern int have_git_dir(void); extern const char *get_git_dir(void); extern char *get_object_directory(void); extern char *get_index_file(void); @@@ -420,7 -360,6 +420,7 @@@ extern int init_db(const char *template /* Initialize and use the cache information */ extern int read_index(struct index_state *); +extern int read_index_preload(struct index_state *, const char **pathspec); extern int read_index_from(struct index_state *, const char *path); extern int is_index_unborn(struct index_state *); extern int read_index_unmerged(struct index_state *); @@@ -434,7 -373,6 +434,7 @@@ extern int index_name_pos(const struct #define ADD_CACHE_OK_TO_REPLACE 2 /* Ok to replace file/directory */ #define ADD_CACHE_SKIP_DFCHECK 4 /* Ok to skip DF conflict checks */ #define ADD_CACHE_JUST_APPEND 8 /* Append only; tree.c::read_tree() */ +#define ADD_CACHE_NEW_ONLY 16 /* Do not replace existing ones */ extern int add_index_entry(struct index_state *, struct cache_entry *ce, int option); extern struct cache_entry *refresh_cache_entry(struct cache_entry *ce, int really); extern void rename_index_entry_at(struct index_state *, int pos, const char *new_name); @@@ -443,8 -381,6 +443,8 @@@ extern int remove_file_from_index(struc #define ADD_CACHE_VERBOSE 1 #define ADD_CACHE_PRETEND 2 #define ADD_CACHE_IGNORE_ERRORS 4 +#define ADD_CACHE_IGNORE_REMOVAL 8 +#define ADD_CACHE_INTENT 16 extern int add_to_index(struct index_state *, const char *path, struct stat *, int flags); extern int add_file_to_index(struct index_state *, const char *path, int flags); extern struct cache_entry *make_cache_entry(unsigned int mode, const unsigned char *sha1, const char *path, int stage, int refresh); @@@ -460,6 -396,7 +460,6 @@@ extern int ie_modified(const struct ind extern int ce_path_match(const struct cache_entry *ce, const char **pathspec); extern int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, enum object_type type, const char *path); -extern int index_pipe(unsigned char *sha1, int fd, const char *type, int write_object); extern int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object); extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st); @@@ -511,7 -448,6 +511,7 @@@ extern size_t packed_git_limit extern size_t delta_base_cache_limit; extern int auto_crlf; extern int fsync_object_files; +extern int core_preload_index; enum safe_crlf { SAFE_CRLF_FALSE = 0, @@@ -522,7 -458,6 +522,7 @@@ extern enum safe_crlf safe_crlf; enum branch_track { + BRANCH_TRACK_UNSPECIFIED = -1, BRANCH_TRACK_NEVER = 0, BRANCH_TRACK_REMOTE, BRANCH_TRACK_ALWAYS, @@@ -582,13 -517,6 +582,13 @@@ static inline void hashclr(unsigned cha { memset(hash, 0, 20); } +extern int is_empty_blob_sha1(const unsigned char *sha1); + +#define EMPTY_TREE_SHA1_HEX \ + "4b825dc642cb6eb9a060e54bf8d69288fbee4904" +#define EMPTY_TREE_SHA1_BIN \ + "\x4b\x82\x5d\xc6\x42\xcb\x6e\xb9\xa0\x60" \ + "\xe5\x4b\xf8\xd6\x92\x88\xfb\xee\x49\x04" int git_mkstemp(char *path, size_t n, const char *template); @@@ -616,7 -544,6 +616,7 @@@ static inline int is_absolute_path(cons { return path[0] == '/' || has_dos_drive_prefix(path); } +int is_directory(const char *); const char *make_absolute_path(const char *path); const char *make_nonrelative_path(const char *path); const char *make_relative_path(const char *abs, const char *base); @@@ -634,9 -561,6 +634,9 @@@ extern int force_object_loose(const uns /* just like read_sha1_file(), but non fatal in presence of bad objects */ extern void *read_object(const unsigned char *sha1, enum object_type *type, unsigned long *size); +/* global flag to enable extra checks when accessing packed objects */ +extern int do_check_packed_object_crc; + extern int check_sha1_signature(const unsigned char *sha1, void *buf, unsigned long size, const char *type); extern int move_temp_to_file(const char *tmpfile, const char *filename); @@@ -728,8 -652,6 +728,8 @@@ extern struct alternate_object_databas } *alt_odb_list; extern void prepare_alt_odb(void); extern void add_to_alternates_file(const char *reference); +typedef int alt_odb_fn(struct alternate_object_database *, void *); +extern void foreach_alt_odb(alt_odb_fn, void*); struct pack_window { struct pack_window *next; @@@ -799,11 -721,7 +799,11 @@@ extern struct child_process *git_connec extern int finish_connect(struct child_process *conn); extern int path_match(const char *path, int nr, char **match); extern int get_ack(int fd, unsigned char *result_sha1); -extern struct ref **get_remote_heads(int in, struct ref **list, int nr_match, char **match, unsigned int flags); +struct extra_have_objects { + int nr, alloc; + unsigned char (*array)[20]; +}; +extern struct ref **get_remote_heads(int in, struct ref **list, int nr_match, char **match, unsigned int flags, struct extra_have_objects *); extern int server_supports(const char *feature); extern struct packed_git *parse_pack_index(unsigned char *sha1); @@@ -820,12 -738,13 +820,13 @@@ extern int open_pack_index(struct packe extern unsigned char* use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *); extern void close_pack_windows(struct packed_git *); extern void unuse_pack(struct pack_window **); + extern void free_pack_by_name(const char *); extern struct packed_git *add_packed_git(const char *, int, int); extern const unsigned char *nth_packed_object_sha1(struct packed_git *, uint32_t); extern off_t nth_packed_object_offset(const struct packed_git *, uint32_t); extern off_t find_pack_entry_one(const unsigned char *, struct packed_git *); extern void *unpack_entry(struct packed_git *, off_t, enum object_type *, unsigned long *); -extern unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep); +extern unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep); extern unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t); extern const char *packed_object_info_detail(struct packed_git *, off_t, unsigned long *, unsigned long *, unsigned int *, unsigned char *); extern int matches_pack_name(struct packed_git *p, const char *name); @@@ -837,6 -756,7 +838,6 @@@ typedef int (*config_fn_t)(const char * extern int git_default_config(const char *, const char *, void *); extern int git_config_from_file(config_fn_t fn, const char *, void *); extern int git_config(config_fn_t fn, void *); -extern int git_parse_long(const char *, long *); extern int git_parse_ulong(const char *, unsigned long *); extern int git_config_int(const char *, const char *); extern unsigned long git_config_ulong(const char *, const char *); diff --combined sha1_file.c index 6c0e251e02,88035a0cd1..0e021c5eca --- a/sha1_file.c +++ b/sha1_file.c @@@ -99,11 -99,7 +99,11 @@@ int safe_create_leading_directories(cha pos = strchr(pos, '/'); if (!pos) break; - *pos = 0; + while (*++pos == '/') + ; + if (!*pos) + break; + *--pos = '\0'; if (!stat(path, &st)) { /* path exists */ if (!S_ISDIR(st.st_mode)) { @@@ -254,6 -250,7 +254,6 @@@ static void read_info_alternates(const */ static int link_alt_odb_entry(const char * entry, int len, const char * relative_base, int depth) { - struct stat st; const char *objdir = get_object_directory(); struct alternate_object_database *ent; struct alternate_object_database *alt; @@@ -284,7 -281,7 +284,7 @@@ ent->base[pfxlen] = ent->base[entlen-1] = 0; /* Detect cases where alternate disappeared */ - if (stat(ent->base, &st) || !S_ISDIR(st.st_mode)) { + if (!is_directory(ent->base)) { error("object directory %s does not exist; " "check .git/objects/info/alternates.", ent->base); @@@ -397,16 -394,6 +397,16 @@@ void add_to_alternates_file(const char link_alt_odb_entries(alt, alt + strlen(alt), '\n', NULL, 0); } +void foreach_alt_odb(alt_odb_fn fn, void *cb) +{ + struct alternate_object_database *ent; + + prepare_alt_odb(); + for (ent = alt_odb_list; ent; ent = ent->next) + if (fn(ent, cb)) + return; +} + void prepare_alt_odb(void) { const char *alt; @@@ -672,6 -659,37 +672,37 @@@ void unuse_pack(struct pack_window **w_ } } + /* + * This is used by git-repack in case a newly created pack happens to + * contain the same set of objects as an existing one. In that case + * the resulting file might be different even if its name would be the + * same. It is best to close any reference to the old pack before it is + * replaced on disk. Of course no index pointers nor windows for given pack + * must subsist at this point. If ever objects from this pack are requested + * again, the new version of the pack will be reinitialized through + * reprepare_packed_git(). + */ + void free_pack_by_name(const char *pack_name) + { + struct packed_git *p, **pp = &packed_git; + + while (*pp) { + p = *pp; + if (strcmp(pack_name, p->pack_name) == 0) { + close_pack_windows(p); + if (p->pack_fd != -1) + close(p->pack_fd); + if (p->index_data) + munmap((void *)p->index_data, p->index_size); + free(p->bad_object_sha1); + *pp = p->next; + free(p); + return; + } + pp = &p->next; + } + } + /* * Do not call this directly as this leaks p->pack_fd on error return; * call open_packed_git() instead. @@@ -1122,8 -1140,7 +1153,8 @@@ static int legacy_loose_object(unsigne return 0; } -unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep) +unsigned long unpack_object_header_buffer(const unsigned char *buf, + unsigned long len, enum object_type *type, unsigned long *sizep) { unsigned shift; unsigned char c; @@@ -1135,10 -1152,10 +1166,10 @@@ size = c & 15; shift = 4; while (c & 0x80) { - if (len <= used) - return 0; - if (sizeof(long) * 8 <= shift) + if (len <= used || sizeof(long) * 8 <= shift) { + error("bad object header"); return 0; + } c = buf[used++]; size += (c & 0x7f) << shift; shift += 7; @@@ -1177,7 -1194,7 +1208,7 @@@ static int unpack_sha1_header(z_stream * really worth it and we don't write it any longer. But we * can still read it. */ - used = unpack_object_header_gently(map, mapsize, &type, &size); + used = unpack_object_header_buffer(map, mapsize, &type, &size); if (!used || !valid_loose_object_type[type]) return -1; map += used; @@@ -1326,10 -1343,8 +1357,10 @@@ unsigned long get_size_from_delta(struc } while ((st == Z_OK || st == Z_BUF_ERROR) && stream.total_out < sizeof(delta_head)); inflateEnd(&stream); - if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head)) - die("delta data unpack-initial failed"); + if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head)) { + error("delta data unpack-initial failed"); + return 0; + } /* Examine the initial part of the delta to figure out * the result size. @@@ -1370,7 -1385,7 +1401,7 @@@ static off_t get_delta_base(struct pack base_offset = (base_offset << 7) + (c & 127); } base_offset = delta_obj_offset - base_offset; - if (base_offset >= delta_obj_offset) + if (base_offset <= 0 || base_offset >= delta_obj_offset) return 0; /* out of bound */ *curpos += used; } else if (type == OBJ_REF_DELTA) { @@@ -1396,32 -1411,15 +1427,32 @@@ static int packed_delta_info(struct pac off_t base_offset; base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset); + if (!base_offset) + return OBJ_BAD; type = packed_object_info(p, base_offset, NULL); + if (type <= OBJ_NONE) { + struct revindex_entry *revidx; + const unsigned char *base_sha1; + revidx = find_pack_revindex(p, base_offset); + if (!revidx) + return OBJ_BAD; + base_sha1 = nth_packed_object_sha1(p, revidx->nr); + mark_bad_packed_object(p, base_sha1); + type = sha1_object_info(base_sha1, NULL); + if (type <= OBJ_NONE) + return OBJ_BAD; + } /* We choose to only get the type of the base object and * ignore potentially corrupt pack file that expects the delta * based on a base with a wrong size. This saves tons of * inflate() calls. */ - if (sizep) + if (sizep) { *sizep = get_size_from_delta(p, w_curs, curpos); + if (*sizep == 0) + type = OBJ_BAD; + } return type; } @@@ -1443,11 -1441,10 +1474,11 @@@ static int unpack_object_header(struct * insane, so we know won't exceed what we have been given. */ base = use_pack(p, w_curs, *curpos, &left); - used = unpack_object_header_gently(base, left, &type, sizep); - if (!used) - die("object offset outside of pack file"); - *curpos += used; + used = unpack_object_header_buffer(base, left, &type, sizep); + if (!used) { + type = OBJ_BAD; + } else + *curpos += used; return type; } @@@ -1531,9 -1528,8 +1562,9 @@@ static int packed_object_info(struct pa *sizep = size; break; default: - die("pack %s contains unknown object type %d", - p->pack_name, type); + error("unknown object type %i at offset %"PRIuMAX" in %s", + type, (uintmax_t)obj_offset, p->pack_name); + type = OBJ_BAD; } unuse_pack(&w_curs); return type; @@@ -1605,9 -1601,11 +1636,9 @@@ static void *cache_or_unpack_entry(stru struct delta_base_cache_entry *ent = delta_base_cache + hash; ret = ent->data; - if (ret && ent->p == p && ent->base_offset == base_offset) - goto found_cache_entry; - return unpack_entry(p, base_offset, type, base_size); + if (!ret || ent->p != p || ent->base_offset != base_offset) + return unpack_entry(p, base_offset, type, base_size); -found_cache_entry: if (!keep_cache) { ent->data = NULL; ent->lru.next->prev = ent->lru.prev; @@@ -1697,12 -1695,9 +1728,12 @@@ static void *unpack_delta_entry(struct * This is costly but should happen only in the presence * of a corrupted pack, and is better than failing outright. */ - struct revindex_entry *revidx = find_pack_revindex(p, base_offset); - const unsigned char *base_sha1 = - nth_packed_object_sha1(p, revidx->nr); + struct revindex_entry *revidx; + const unsigned char *base_sha1; + revidx = find_pack_revindex(p, base_offset); + if (!revidx) + return NULL; + base_sha1 = nth_packed_object_sha1(p, revidx->nr); error("failed to read delta base object %s" " at offset %"PRIuMAX" from %s", sha1_to_hex(base_sha1), (uintmax_t)base_offset, @@@ -1731,8 -1726,6 +1762,8 @@@ return result; } +int do_check_packed_object_crc; + void *unpack_entry(struct packed_git *p, off_t obj_offset, enum object_type *type, unsigned long *sizep) { @@@ -1740,20 -1733,6 +1771,20 @@@ off_t curpos = obj_offset; void *data; + if (do_check_packed_object_crc && p->index_version > 1) { + struct revindex_entry *revidx = find_pack_revindex(p, obj_offset); + unsigned long len = revidx[1].offset - obj_offset; + if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) { + const unsigned char *sha1 = + nth_packed_object_sha1(p, revidx->nr); + error("bad packed object CRC for %s", + sha1_to_hex(sha1)); + mark_bad_packed_object(p, sha1); + unuse_pack(&w_curs); + return NULL; + } + } + *type = unpack_object_header(p, &w_curs, &curpos, sizep); switch (*type) { case OBJ_OFS_DELTA: @@@ -2007,14 -1986,7 +2038,14 @@@ int sha1_object_info(const unsigned cha if (!find_pack_entry(sha1, &e, NULL)) return status; } - return packed_object_info(e.p, e.offset, sizep); + + status = packed_object_info(e.p, e.offset, sizep); + if (status < 0) { + mark_bad_packed_object(e.p, sha1); + status = sha1_object_info(sha1, sizep); + } + + return status; } static void *read_packed_sha1(const unsigned char *sha1, @@@ -2056,7 -2028,9 +2087,7 @@@ static struct cached_object static int cached_object_nr, cached_object_alloc; static struct cached_object empty_tree = { - /* empty tree sha1: 4b825dc642cb6eb9a060e54bf8d69288fbee4904 */ - "\x4b\x82\x5d\xc6\x42\xcb\x6e\xb9\xa0\x60" - "\xe5\x4b\xf8\xd6\x92\x88\xfb\xee\x49\x04", + EMPTY_TREE_SHA1_BIN, OBJ_TREE, "", 0 @@@ -2188,16 -2162,16 +2219,16 @@@ static void write_sha1_file_prepare(con const char *type, unsigned char *sha1, char *hdr, int *hdrlen) { - SHA_CTX c; + git_SHA_CTX c; /* Generate the header */ *hdrlen = sprintf(hdr, "%s %lu", type, len)+1; /* Sha1.. */ - SHA1_Init(&c); - SHA1_Update(&c, hdr, *hdrlen); - SHA1_Update(&c, buf, len); - SHA1_Final(sha1, &c); + git_SHA1_Init(&c); + git_SHA1_Update(&c, hdr, *hdrlen); + git_SHA1_Update(&c, buf, len); + git_SHA1_Final(sha1, &c); } /* @@@ -2436,21 -2410,51 +2467,21 @@@ int has_sha1_file(const unsigned char * return has_loose_object(sha1); } -int index_pipe(unsigned char *sha1, int fd, const char *type, int write_object) +static int index_mem(unsigned char *sha1, void *buf, size_t size, + int write_object, enum object_type type, const char *path) { - struct strbuf buf; - int ret; - - strbuf_init(&buf, 0); - if (strbuf_read(&buf, fd, 4096) < 0) { - strbuf_release(&buf); - return -1; - } - - if (!type) - type = blob_type; - if (write_object) - ret = write_sha1_file(buf.buf, buf.len, type, sha1); - else - ret = hash_sha1_file(buf.buf, buf.len, type, sha1); - strbuf_release(&buf); - - return ret; -} - -int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, - enum object_type type, const char *path) -{ - size_t size = xsize_t(st->st_size); - void *buf = NULL; int ret, re_allocated = 0; - if (size) - buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); - if (!type) type = OBJ_BLOB; /* * Convert blobs to git internal format */ - if ((type == OBJ_BLOB) && S_ISREG(st->st_mode)) { - struct strbuf nbuf; - strbuf_init(&nbuf, 0); + if ((type == OBJ_BLOB) && path) { + struct strbuf nbuf = STRBUF_INIT; if (convert_to_git(path, buf, size, &nbuf, write_object ? safe_crlf : 0)) { - munmap(buf, size); buf = strbuf_detach(&nbuf, &size); re_allocated = 1; } @@@ -2460,32 -2464,12 +2491,32 @@@ ret = write_sha1_file(buf, size, typename(type), sha1); else ret = hash_sha1_file(buf, size, typename(type), sha1); - if (re_allocated) { + if (re_allocated) free(buf); - return ret; - } - if (size) + return ret; +} + +int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, + enum object_type type, const char *path) +{ + int ret; + size_t size = xsize_t(st->st_size); + + if (!S_ISREG(st->st_mode)) { + struct strbuf sbuf = STRBUF_INIT; + if (strbuf_read(&sbuf, fd, 4096) >= 0) + ret = index_mem(sha1, sbuf.buf, sbuf.len, write_object, + type, path); + else + ret = -1; + strbuf_release(&sbuf); + } else if (size) { + void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); + ret = index_mem(sha1, buf, size, write_object, type, path); munmap(buf, size); + } else + ret = index_mem(sha1, NULL, size, write_object, type, path); + close(fd); return ret; }