From: Junio C Hamano Date: Wed, 20 Sep 2017 02:33:06 +0000 (+0900) Subject: Merge branch 'jk/info-alternates-fix-2.11' into jk/info-alternates-fix X-Git-Tag: v2.14.3~1^2~1 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/0db625f5d69c9d1fb6ace137fc755887aef3049e?hp=-c Merge branch 'jk/info-alternates-fix-2.11' into jk/info-alternates-fix * jk/info-alternates-fix-2.11: read_info_alternates: read contents into strbuf --- 0db625f5d69c9d1fb6ace137fc755887aef3049e diff --combined sha1_file.c index 4fa4b185f3,bfebf78707..a913865df8 --- a/sha1_file.c +++ b/sha1_file.c @@@ -7,7 -7,6 +7,7 @@@ * creation etc. */ #include "cache.h" +#include "config.h" #include "string-list.h" #include "lockfile.h" #include "delta.h" @@@ -29,6 -28,14 +29,6 @@@ #include "mergesort.h" #include "quote.h" -#ifndef O_NOATIME -#if defined(__linux__) && (defined(__i386__) || defined(__PPC__)) -#define O_NOATIME 01000000 -#else -#define O_NOATIME 0 -#endif -#endif - #define SZ_FMT PRIuMAX static inline uintmax_t sz_fmt(size_t s) { return s; } @@@ -130,10 -137,8 +130,10 @@@ enum scld_error safe_create_leading_dir *slash = '\0'; if (!stat(path, &st)) { /* path exists */ - if (!S_ISDIR(st.st_mode)) + if (!S_ISDIR(st.st_mode)) { + errno = ENOTDIR; ret = SCLD_EXISTS; + } } else if (mkdir(path, 0777)) { if (errno == EEXIST && !stat(path, &st) && S_ISDIR(st.st_mode)) @@@ -161,85 -166,13 +161,85 @@@ enum scld_error safe_create_leading_directories_const(const char *path) { + int save_errno; /* path points to cache entries, so xstrdup before messing with it */ char *buf = xstrdup(path); enum scld_error result = safe_create_leading_directories(buf); + + save_errno = errno; free(buf); + errno = save_errno; return result; } +int raceproof_create_file(const char *path, create_file_fn fn, void *cb) +{ + /* + * The number of times we will try to remove empty directories + * in the way of path. This is only 1 because if another + * process is racily creating directories that conflict with + * us, we don't want to fight against them. + */ + int remove_directories_remaining = 1; + + /* + * The number of times that we will try to create the + * directories containing path. We are willing to attempt this + * more than once, because another process could be trying to + * clean up empty directories at the same time as we are + * trying to create them. + */ + int create_directories_remaining = 3; + + /* A scratch copy of path, filled lazily if we need it: */ + struct strbuf path_copy = STRBUF_INIT; + + int ret, save_errno; + + /* Sanity check: */ + assert(*path); + +retry_fn: + ret = fn(path, cb); + save_errno = errno; + if (!ret) + goto out; + + if (errno == EISDIR && remove_directories_remaining-- > 0) { + /* + * A directory is in the way. Maybe it is empty; try + * to remove it: + */ + if (!path_copy.len) + strbuf_addstr(&path_copy, path); + + if (!remove_dir_recursively(&path_copy, REMOVE_DIR_EMPTY_ONLY)) + goto retry_fn; + } else if (errno == ENOENT && create_directories_remaining-- > 0) { + /* + * Maybe the containing directory didn't exist, or + * maybe it was just deleted by a process that is + * racing with us to clean up empty directories. Try + * to create it: + */ + enum scld_error scld_result; + + if (!path_copy.len) + strbuf_addstr(&path_copy, path); + + do { + scld_result = safe_create_leading_directories(path_copy.buf); + if (scld_result == SCLD_OK) + goto retry_fn; + } while (scld_result == SCLD_VANISHED && create_directories_remaining-- > 0); + } + +out: + strbuf_release(&path_copy); + errno = save_errno; + return ret; +} + static void fill_sha1_path(struct strbuf *buf, const unsigned char *sha1) { int i; @@@ -278,26 -211,31 +278,26 @@@ static const char *alt_sha1_path(struc return buf->buf; } -/* - * Return the name of the pack or index file with the specified sha1 - * in its filename. *base and *name are scratch space that must be - * provided by the caller. which should be "pack" or "idx". - */ -static char *sha1_get_pack_name(const unsigned char *sha1, - struct strbuf *buf, - const char *which) + char *odb_pack_name(struct strbuf *buf, + const unsigned char *sha1, + const char *ext) { strbuf_reset(buf); strbuf_addf(buf, "%s/pack/pack-%s.%s", get_object_directory(), - sha1_to_hex(sha1), which); + sha1_to_hex(sha1), ext); return buf->buf; } char *sha1_pack_name(const unsigned char *sha1) { static struct strbuf buf = STRBUF_INIT; - return sha1_get_pack_name(sha1, &buf, "pack"); + return odb_pack_name(&buf, sha1, "pack"); } char *sha1_pack_index_name(const unsigned char *sha1) { static struct strbuf buf = STRBUF_INIT; - return sha1_get_pack_name(sha1, &buf, "idx"); + return odb_pack_name(&buf, sha1, "idx"); } struct alternate_object_database *alt_odb_list; @@@ -347,7 -285,6 +347,7 @@@ static int alt_odb_usable(struct strbu * SHA1, an extra slash for the first level indirection, and the * terminating NUL. */ +static void read_info_alternates(const char * relative_base, int depth); static int link_alt_odb_entry(const char *entry, const char *relative_base, int depth, const char *normalized_objdir) { @@@ -355,7 -292,7 +355,7 @@@ struct strbuf pathbuf = STRBUF_INIT; if (!is_absolute_path(entry) && relative_base) { - strbuf_addstr(&pathbuf, real_path(relative_base)); + strbuf_realpath(&pathbuf, relative_base, 1); strbuf_addch(&pathbuf, '/'); } strbuf_addstr(&pathbuf, entry); @@@ -422,7 -359,7 +422,7 @@@ static const char *parse_alt_odb_entry( return end; } - static void link_alt_odb_entries(const char *alt, int len, int sep, + static void link_alt_odb_entries(const char *alt, int sep, const char *relative_base, int depth) { struct strbuf objdirbuf = STRBUF_INIT; @@@ -449,30 -386,20 +449,20 @@@ strbuf_release(&objdirbuf); } -void read_info_alternates(const char * relative_base, int depth) +static void read_info_alternates(const char * relative_base, int depth) { - char *map; - size_t mapsz; - struct stat st; char *path; - int fd; + struct strbuf buf = STRBUF_INIT; path = xstrfmt("%s/info/alternates", relative_base); - fd = git_open(path); - free(path); - if (fd < 0) - return; - if (fstat(fd, &st) || (st.st_size == 0)) { - close(fd); + if (strbuf_read_file(&buf, path, 1024) < 0) { + free(path); return; } - mapsz = xsize_t(st.st_size); - map = xmmap(NULL, mapsz, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); - link_alt_odb_entries(map, mapsz, '\n', relative_base, depth); - - munmap(map, mapsz); + link_alt_odb_entries(buf.buf, '\n', relative_base, depth); + strbuf_release(&buf); + free(path); } struct alternate_object_database *alloc_alt_odb(const char *dir) @@@ -527,7 -454,7 +517,7 @@@ void add_to_alternates_file(const char if (commit_lock_file(lock)) die_errno("unable to move new alternates file into place"); if (alt_odb_tail) - link_alt_odb_entries(reference, strlen(reference), '\n', NULL, 0); + link_alt_odb_entries(reference, '\n', NULL, 0); } free(alts); } @@@ -540,7 -467,7 +530,7 @@@ void add_to_alternates_memory(const cha */ prepare_alt_odb(); - link_alt_odb_entries(reference, strlen(reference), '\n', NULL, 0); + link_alt_odb_entries(reference, '\n', NULL, 0); } /* @@@ -612,7 -539,8 +602,7 @@@ char *compute_alternate_path(const cha out: if (seen_error) { - free(ref_git); - ref_git = NULL; + FREE_AND_NULL(ref_git); } return ref_git; @@@ -643,7 -571,7 +633,7 @@@ void prepare_alt_odb(void if (!alt) alt = ""; alt_odb_tail = &alt_odb_list; - link_alt_odb_entries(alt, strlen(alt), PATH_SEP, NULL, 0); + link_alt_odb_entries(alt, PATH_SEP, NULL, 0); read_info_alternates(get_object_directory(), 0); } @@@ -663,7 -591,7 +653,7 @@@ static int freshen_file(const char *fn * either does not exist on disk, or has a stale mtime and may be subject to * pruning). */ -static int check_and_freshen_file(const char *fn, int freshen) +int check_and_freshen_file(const char *fn, int freshen) { if (access(fn, F_OK)) return 0; @@@ -1607,7 -1535,7 +1597,7 @@@ static void mark_bad_packed_object(stru if (!hashcmp(sha1, p->bad_object_sha1 + GIT_SHA1_RAWSZ * i)) return; p->bad_object_sha1 = xrealloc(p->bad_object_sha1, - st_mult(GIT_SHA1_RAWSZ, + st_mult(GIT_MAX_RAWSZ, st_add(p->num_bad_objects, 1))); hashcpy(p->bad_object_sha1 + GIT_SHA1_RAWSZ * p->num_bad_objects, sha1); p->num_bad_objects++; @@@ -1673,81 -1601,66 +1663,81 @@@ int check_sha1_signature(const unsigne return hashcmp(sha1, real_sha1) ? -1 : 0; } -int git_open(const char *name) +int git_open_cloexec(const char *name, int flags) { - static int sha1_file_open_flag = O_NOATIME | O_CLOEXEC; - - for (;;) { - int fd; - - errno = 0; - fd = open(name, O_RDONLY | sha1_file_open_flag); - if (fd >= 0) - return fd; + int fd; + static int o_cloexec = O_CLOEXEC; + fd = open(name, flags | o_cloexec); + if ((o_cloexec & O_CLOEXEC) && fd < 0 && errno == EINVAL) { /* Try again w/o O_CLOEXEC: the kernel might not support it */ - if ((sha1_file_open_flag & O_CLOEXEC) && errno == EINVAL) { - sha1_file_open_flag &= ~O_CLOEXEC; - continue; - } + o_cloexec &= ~O_CLOEXEC; + fd = open(name, flags | o_cloexec); + } - /* Might the failure be due to O_NOATIME? */ - if (errno != ENOENT && (sha1_file_open_flag & O_NOATIME)) { - sha1_file_open_flag &= ~O_NOATIME; - continue; +#if defined(F_GETFD) && defined(F_SETFD) && defined(FD_CLOEXEC) + { + static int fd_cloexec = FD_CLOEXEC; + + if (!o_cloexec && 0 <= fd && fd_cloexec) { + /* Opened w/o O_CLOEXEC? try with fcntl(2) to add it */ + int flags = fcntl(fd, F_GETFD); + if (fcntl(fd, F_SETFD, flags | fd_cloexec)) + fd_cloexec = 0; } - return -1; } +#endif + return fd; } -static int stat_sha1_file(const unsigned char *sha1, struct stat *st) +/* + * Find "sha1" as a loose object in the local repository or in an alternate. + * Returns 0 on success, negative on failure. + * + * The "path" out-parameter will give the path of the object we found (if any). + * Note that it may point to static storage and is only valid until another + * call to sha1_file_name(), etc. + */ +static int stat_sha1_file(const unsigned char *sha1, struct stat *st, + const char **path) { struct alternate_object_database *alt; - if (!lstat(sha1_file_name(sha1), st)) + *path = sha1_file_name(sha1); + if (!lstat(*path, st)) return 0; prepare_alt_odb(); errno = ENOENT; for (alt = alt_odb_list; alt; alt = alt->next) { - const char *path = alt_sha1_path(alt, sha1); - if (!lstat(path, st)) + *path = alt_sha1_path(alt, sha1); + if (!lstat(*path, st)) return 0; } return -1; } -static int open_sha1_file(const unsigned char *sha1) +/* + * Like stat_sha1_file(), but actually open the object and return the + * descriptor. See the caveats on the "path" parameter above. + */ +static int open_sha1_file(const unsigned char *sha1, const char **path) { int fd; struct alternate_object_database *alt; int most_interesting_errno; - fd = git_open(sha1_file_name(sha1)); + *path = sha1_file_name(sha1); + fd = git_open(*path); if (fd >= 0) return fd; most_interesting_errno = errno; prepare_alt_odb(); for (alt = alt_odb_list; alt; alt = alt->next) { - const char *path = alt_sha1_path(alt, sha1); - fd = git_open(path); + *path = alt_sha1_path(alt, sha1); + fd = git_open(*path); if (fd >= 0) return fd; if (most_interesting_errno == ENOENT) @@@ -1757,21 -1670,12 +1747,21 @@@ return -1; } -void *map_sha1_file(const unsigned char *sha1, unsigned long *size) +/* + * Map the loose object at "path" if it is not NULL, or the path found by + * searching for a loose object named "sha1". + */ +static void *map_sha1_file_1(const char *path, + const unsigned char *sha1, + unsigned long *size) { void *map; int fd; - fd = open_sha1_file(sha1); + if (path) + fd = git_open(path); + else + fd = open_sha1_file(sha1, &path); map = NULL; if (fd >= 0) { struct stat st; @@@ -1780,7 -1684,7 +1770,7 @@@ *size = xsize_t(st.st_size); if (!*size) { /* mmap() is forbidden on empty files */ - error("object file %s is empty", sha1_file_name(sha1)); + error("object file %s is empty", path); return NULL; } map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0); @@@ -1790,11 -1694,6 +1780,11 @@@ return map; } +void *map_sha1_file(const unsigned char *sha1, unsigned long *size) +{ + return map_sha1_file_1(NULL, sha1, size); +} + unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep) { @@@ -1965,7 -1864,7 +1955,7 @@@ static int parse_sha1_header_extended(c * we're obtaining the type using '--allow-unknown-type' * option. */ - if ((flags & LOOKUP_UNKNOWN_OBJECT) && (type < 0)) + if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE) && (type < 0)) type = 0; else if (type < 0) die("invalid object type"); @@@ -2003,7 -1902,20 +1993,7 @@@ int parse_sha1_header(const char *hdr, struct object_info oi = OBJECT_INFO_INIT; oi.sizep = sizep; - return parse_sha1_header_extended(hdr, &oi, LOOKUP_REPLACE_OBJECT); -} - -static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1) -{ - int ret; - git_zstream stream; - char hdr[8192]; - - ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)); - if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0) - return NULL; - - return unpack_sha1_rest(&stream, hdr, *size, sha1); + return parse_sha1_header_extended(hdr, &oi, 0); } unsigned long get_size_from_delta(struct packed_git *p, @@@ -2227,6 -2139,107 +2217,6 @@@ unwind goto out; } -int packed_object_info(struct packed_git *p, off_t obj_offset, - struct object_info *oi) -{ - struct pack_window *w_curs = NULL; - unsigned long size; - off_t curpos = obj_offset; - enum object_type type; - - /* - * We always get the representation type, but only convert it to - * a "real" type later if the caller is interested. - */ - type = unpack_object_header(p, &w_curs, &curpos, &size); - - if (oi->sizep) { - if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) { - off_t tmp_pos = curpos; - off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos, - type, obj_offset); - if (!base_offset) { - type = OBJ_BAD; - goto out; - } - *oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos); - if (*oi->sizep == 0) { - type = OBJ_BAD; - goto out; - } - } else { - *oi->sizep = size; - } - } - - if (oi->disk_sizep) { - struct revindex_entry *revidx = find_pack_revindex(p, obj_offset); - *oi->disk_sizep = revidx[1].offset - obj_offset; - } - - if (oi->typep) { - *oi->typep = packed_to_object_type(p, obj_offset, type, &w_curs, curpos); - if (*oi->typep < 0) { - type = OBJ_BAD; - goto out; - } - } - - if (oi->delta_base_sha1) { - if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) { - const unsigned char *base; - - base = get_delta_base_sha1(p, &w_curs, curpos, - type, obj_offset); - if (!base) { - type = OBJ_BAD; - goto out; - } - - hashcpy(oi->delta_base_sha1, base); - } else - hashclr(oi->delta_base_sha1); - } - -out: - unuse_pack(&w_curs); - return type; -} - -static void *unpack_compressed_entry(struct packed_git *p, - struct pack_window **w_curs, - off_t curpos, - unsigned long size) -{ - int st; - git_zstream stream; - unsigned char *buffer, *in; - - buffer = xmallocz_gently(size); - if (!buffer) - return NULL; - memset(&stream, 0, sizeof(stream)); - stream.next_out = buffer; - stream.avail_out = size + 1; - - git_inflate_init(&stream); - do { - in = use_pack(p, w_curs, curpos, &stream.avail_in); - stream.next_in = in; - st = git_inflate(&stream, Z_FINISH); - if (!stream.avail_out) - break; /* the payload is larger than it should be */ - curpos += stream.next_in - in; - } while (st == Z_OK || st == Z_BUF_ERROR); - git_inflate_end(&stream); - if ((st != Z_STREAM_END) || stream.total_out != size) { - free(buffer); - return NULL; - } - - return buffer; -} - static struct hashmap delta_base_cache; static size_t delta_base_cached; @@@ -2276,8 -2289,7 +2266,8 @@@ static int delta_base_cache_key_eq(cons return a->p == b->p && a->base_offset == b->base_offset; } -static int delta_base_cache_hash_cmp(const void *va, const void *vb, +static int delta_base_cache_hash_cmp(const void *unused_cmp_data, + const void *va, const void *vb, const void *vkey) { const struct delta_base_cache_entry *a = va, *b = vb; @@@ -2315,10 -2327,8 +2305,10 @@@ static void *cache_or_unpack_entry(stru if (!ent) return unpack_entry(p, base_offset, type, base_size); - *type = ent->type; - *base_size = ent->size; + if (type) + *type = ent->type; + if (base_size) + *base_size = ent->size; return xmemdupz(ent->data, ent->size); } @@@ -2330,10 -2340,11 +2320,10 @@@ static inline void release_delta_base_c void clear_delta_base_cache(void) { - struct hashmap_iter iter; - struct delta_base_cache_entry *entry; - for (entry = hashmap_iter_first(&delta_base_cache, &iter); - entry; - entry = hashmap_iter_next(&iter)) { + struct list_head *lru, *tmp; + list_for_each_safe(lru, tmp, &delta_base_cache_lru) { + struct delta_base_cache_entry *entry = + list_entry(lru, struct delta_base_cache_entry, lru); release_delta_base_cache(entry); } } @@@ -2362,128 -2373,11 +2352,128 @@@ static void add_delta_base_cache(struc list_add_tail(&ent->lru, &delta_base_cache_lru); if (!delta_base_cache.cmpfn) - hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, 0); + hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, NULL, 0); hashmap_entry_init(ent, pack_entry_hash(p, base_offset)); hashmap_add(&delta_base_cache, ent); } +int packed_object_info(struct packed_git *p, off_t obj_offset, + struct object_info *oi) +{ + struct pack_window *w_curs = NULL; + unsigned long size; + off_t curpos = obj_offset; + enum object_type type; + + /* + * We always get the representation type, but only convert it to + * a "real" type later if the caller is interested. + */ + if (oi->contentp) { + *oi->contentp = cache_or_unpack_entry(p, obj_offset, oi->sizep, + &type); + if (!*oi->contentp) + type = OBJ_BAD; + } else { + type = unpack_object_header(p, &w_curs, &curpos, &size); + } + + if (!oi->contentp && oi->sizep) { + if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) { + off_t tmp_pos = curpos; + off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos, + type, obj_offset); + if (!base_offset) { + type = OBJ_BAD; + goto out; + } + *oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos); + if (*oi->sizep == 0) { + type = OBJ_BAD; + goto out; + } + } else { + *oi->sizep = size; + } + } + + if (oi->disk_sizep) { + struct revindex_entry *revidx = find_pack_revindex(p, obj_offset); + *oi->disk_sizep = revidx[1].offset - obj_offset; + } + + if (oi->typep || oi->typename) { + enum object_type ptot; + ptot = packed_to_object_type(p, obj_offset, type, &w_curs, + curpos); + if (oi->typep) + *oi->typep = ptot; + if (oi->typename) { + const char *tn = typename(ptot); + if (tn) + strbuf_addstr(oi->typename, tn); + } + if (ptot < 0) { + type = OBJ_BAD; + goto out; + } + } + + if (oi->delta_base_sha1) { + if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) { + const unsigned char *base; + + base = get_delta_base_sha1(p, &w_curs, curpos, + type, obj_offset); + if (!base) { + type = OBJ_BAD; + goto out; + } + + hashcpy(oi->delta_base_sha1, base); + } else + hashclr(oi->delta_base_sha1); + } + +out: + unuse_pack(&w_curs); + return type; +} + +static void *unpack_compressed_entry(struct packed_git *p, + struct pack_window **w_curs, + off_t curpos, + unsigned long size) +{ + int st; + git_zstream stream; + unsigned char *buffer, *in; + + buffer = xmallocz_gently(size); + if (!buffer) + return NULL; + memset(&stream, 0, sizeof(stream)); + stream.next_out = buffer; + stream.avail_out = size + 1; + + git_inflate_init(&stream); + do { + in = use_pack(p, w_curs, curpos, &stream.avail_in); + stream.next_in = in; + st = git_inflate(&stream, Z_FINISH); + if (!stream.avail_out) + break; /* the payload is larger than it should be */ + curpos += stream.next_in - in; + } while (st == Z_OK || st == Z_BUF_ERROR); + git_inflate_end(&stream); + if ((st != Z_STREAM_END) || stream.total_out != size) { + free(buffer); + return NULL; + } + + return buffer; +} + static void *read_object(const unsigned char *sha1, enum object_type *type, unsigned long *size); @@@ -2543,8 -2437,8 +2533,8 @@@ void *unpack_entry(struct packed_git *p error("bad packed object CRC for %s", sha1_to_hex(sha1)); mark_bad_packed_object(p, sha1); - unuse_pack(&w_curs); - return NULL; + data = NULL; + goto out; } } @@@ -2608,7 -2502,6 +2598,7 @@@ while (delta_stack_nr) { void *delta_data; void *base = data; + void *external_base = NULL; unsigned long delta_size, base_size = size; int i; @@@ -2635,7 -2528,6 +2625,7 @@@ p->pack_name); mark_bad_packed_object(p, base_sha1); base = read_object(base_sha1, &type, &base_size); + external_base = base; } } @@@ -2654,7 -2546,6 +2644,7 @@@ "at offset %"PRIuMAX" from %s", (uintmax_t)curpos, p->pack_name); data = NULL; + free(external_base); continue; } @@@ -2674,15 -2565,11 +2664,15 @@@ error("failed to apply delta"); free(delta_data); + free(external_base); } - *final_type = type; - *final_size = size; + if (final_type) + *final_type = type; + if (final_size) + *final_size = size; +out: unuse_pack(&w_curs); if (delta_stack != small_delta_stack) @@@ -2711,17 -2598,6 +2701,17 @@@ const unsigned char *nth_packed_object_ } } +const struct object_id *nth_packed_object_oid(struct object_id *oid, + struct packed_git *p, + uint32_t n) +{ + const unsigned char *hash = nth_packed_object_sha1(p, n); + if (!hash) + return NULL; + hashcpy(oid->hash, hash); + return oid; +} + void check_pack_index_ptr(const struct packed_git *p, const void *vptr) { const unsigned char *ptr = vptr; @@@ -2801,7 -2677,7 +2791,7 @@@ off_t find_pack_entry_one(const unsigne return nth_packed_object_offset(p, pos); } - do { + while (lo < hi) { unsigned mi = (lo + hi) / 2; int cmp = hashcmp(index + mi * stride, sha1); @@@ -2814,7 -2690,7 +2804,7 @@@ hi = mi; else lo = mi+1; - } while (lo < hi); + } return 0; } @@@ -2915,7 -2791,6 +2905,7 @@@ static int sha1_loose_object_info(cons git_zstream stream; char hdr[32]; struct strbuf hdrbuf = STRBUF_INIT; + unsigned long size_scratch; if (oi->delta_base_sha1) hashclr(oi->delta_base_sha1); @@@ -2928,10 -2803,9 +2918,10 @@@ * return value implicitly indicates whether the * object even exists. */ - if (!oi->typep && !oi->typename && !oi->sizep) { + if (!oi->typep && !oi->typename && !oi->sizep && !oi->contentp) { + const char *path; struct stat st; - if (stat_sha1_file(sha1, &st) < 0) + if (stat_sha1_file(sha1, &st, &path) < 0) return -1; if (oi->disk_sizep) *oi->disk_sizep = st.st_size; @@@ -2941,13 -2815,9 +2931,13 @@@ map = map_sha1_file(sha1, &mapsize); if (!map) return -1; + + if (!oi->sizep) + oi->sizep = &size_scratch; + if (oi->disk_sizep) *oi->disk_sizep = mapsize; - if ((flags & LOOKUP_UNKNOWN_OBJECT)) { + if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE)) { if (unpack_sha1_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0) status = error("unable to unpack %s header with --allow-unknown-type", sha1_to_hex(sha1)); @@@ -2962,52 -2832,36 +2952,52 @@@ sha1_to_hex(sha1)); } else if ((status = parse_sha1_header_extended(hdr, oi, flags)) < 0) status = error("unable to parse %s header", sha1_to_hex(sha1)); - git_inflate_end(&stream); + + if (status >= 0 && oi->contentp) + *oi->contentp = unpack_sha1_rest(&stream, hdr, + *oi->sizep, sha1); + else + git_inflate_end(&stream); + munmap(map, mapsize); if (status && oi->typep) *oi->typep = status; + if (oi->sizep == &size_scratch) + oi->sizep = NULL; strbuf_release(&hdrbuf); - return 0; + return (status < 0) ? status : 0; } int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags) { - struct cached_object *co; + static struct object_info blank_oi = OBJECT_INFO_INIT; struct pack_entry e; int rtype; - enum object_type real_type; - const unsigned char *real = lookup_replace_object_extended(sha1, flags); - - co = find_cached_object(real); - if (co) { - if (oi->typep) - *(oi->typep) = co->type; - if (oi->sizep) - *(oi->sizep) = co->size; - if (oi->disk_sizep) - *(oi->disk_sizep) = 0; - if (oi->delta_base_sha1) - hashclr(oi->delta_base_sha1); - if (oi->typename) - strbuf_addstr(oi->typename, typename(co->type)); - oi->whence = OI_CACHED; - return 0; + const unsigned char *real = (flags & OBJECT_INFO_LOOKUP_REPLACE) ? + lookup_replace_object(sha1) : + sha1; + + if (!oi) + oi = &blank_oi; + + if (!(flags & OBJECT_INFO_SKIP_CACHED)) { + struct cached_object *co = find_cached_object(real); + if (co) { + if (oi->typep) + *(oi->typep) = co->type; + if (oi->sizep) + *(oi->sizep) = co->size; + if (oi->disk_sizep) + *(oi->disk_sizep) = 0; + if (oi->delta_base_sha1) + hashclr(oi->delta_base_sha1); + if (oi->typename) + strbuf_addstr(oi->typename, typename(co->type)); + if (oi->contentp) + *oi->contentp = xmemdupz(co->buf, co->size); + oi->whence = OI_CACHED; + return 0; + } } if (!find_pack_entry(real, &e)) { @@@ -3018,25 -2872,23 +3008,25 @@@ } /* Not a loose object; someone else may have just packed it. */ - reprepare_packed_git(); - if (!find_pack_entry(real, &e)) + if (flags & OBJECT_INFO_QUICK) { return -1; + } else { + reprepare_packed_git(); + if (!find_pack_entry(real, &e)) + return -1; + } } - /* - * packed_object_info() does not follow the delta chain to - * find out the real type, unless it is given oi->typep. - */ - if (oi->typename && !oi->typep) - oi->typep = &real_type; + if (oi == &blank_oi) + /* + * We know that the caller doesn't actually need the + * information below, so return early. + */ + return 0; rtype = packed_object_info(e.p, e.offset, oi); if (rtype < 0) { mark_bad_packed_object(e.p, real); - if (oi->typep == &real_type) - oi->typep = NULL; return sha1_object_info_extended(real, oi, 0); } else if (in_delta_base_cache(e.p, e.offset)) { oi->whence = OI_DBCACHED; @@@ -3047,6 -2899,10 +3037,6 @@@ oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA || rtype == OBJ_OFS_DELTA); } - if (oi->typename) - strbuf_addstr(oi->typename, typename(*oi->typep)); - if (oi->typep == &real_type) - oi->typep = NULL; return 0; } @@@ -3059,8 -2915,7 +3049,8 @@@ int sha1_object_info(const unsigned cha oi.typep = &type; oi.sizep = sizep; - if (sha1_object_info_extended(sha1, &oi, LOOKUP_REPLACE_OBJECT) < 0) + if (sha1_object_info_extended(sha1, &oi, + OBJECT_INFO_LOOKUP_REPLACE) < 0) return -1; return type; } @@@ -3110,15 -2965,28 +3100,15 @@@ int pretend_sha1_file(void *buf, unsign static void *read_object(const unsigned char *sha1, enum object_type *type, unsigned long *size) { - unsigned long mapsize; - void *map, *buf; - struct cached_object *co; - - co = find_cached_object(sha1); - if (co) { - *type = co->type; - *size = co->size; - return xmemdupz(co->buf, co->size); - } + struct object_info oi = OBJECT_INFO_INIT; + void *content; + oi.typep = type; + oi.sizep = size; + oi.contentp = &content; - buf = read_packed_sha1(sha1, type, size); - if (buf) - return buf; - map = map_sha1_file(sha1, &mapsize); - if (map) { - buf = unpack_sha1_file(map, mapsize, type, size, sha1); - munmap(map, mapsize); - return buf; - } - reprepare_packed_git(); - return read_packed_sha1(sha1, type, size); + if (sha1_object_info_extended(sha1, &oi, 0) < 0) + return NULL; + return content; } /* @@@ -3129,14 -2997,11 +3119,14 @@@ void *read_sha1_file_extended(const unsigned char *sha1, enum object_type *type, unsigned long *size, - unsigned flag) + int lookup_replace) { void *data; const struct packed_git *p; - const unsigned char *repl = lookup_replace_object_extended(sha1, flag); + const char *path; + struct stat st; + const unsigned char *repl = lookup_replace ? lookup_replace_object(sha1) + : sha1; errno = 0; data = read_object(repl, type, size); @@@ -3151,9 -3016,12 +3141,9 @@@ die("replacement %s not found for %s", sha1_to_hex(repl), sha1_to_hex(sha1)); - if (has_loose_object(repl)) { - const char *path = sha1_file_name(sha1); - + if (!stat_sha1_file(repl, &st, &path)) die("loose object %s (stored in %s) is corrupt", sha1_to_hex(repl), path); - } if ((p = has_packed_and_bad(repl)) != NULL) die("packed object %s (stored in %s) is corrupt", @@@ -3497,10 -3365,16 +3487,10 @@@ int has_sha1_pack(const unsigned char * int has_sha1_file_with_flags(const unsigned char *sha1, int flags) { - struct pack_entry e; - - if (find_pack_entry(sha1, &e)) - return 1; - if (has_loose_object(sha1)) - return 1; - if (flags & HAS_SHA1_QUICK) + if (!startup_info->have_repository) return 0; - reprepare_packed_git(); - return find_pack_entry(sha1, &e); + return sha1_object_info_extended(sha1, NULL, + flags | OBJECT_INFO_SKIP_CACHED) >= 0; } int has_object_file(const struct object_id *oid) @@@ -3556,7 -3430,7 +3546,7 @@@ static int index_mem(unsigned char *sha */ if ((type == OBJ_BLOB) && path) { struct strbuf nbuf = STRBUF_INIT; - if (convert_to_git(path, buf, size, &nbuf, + if (convert_to_git(&the_index, path, buf, size, &nbuf, write_object ? safe_crlf : SAFE_CRLF_FALSE)) { buf = strbuf_detach(&nbuf, &size); re_allocated = 1; @@@ -3590,7 -3464,7 +3580,7 @@@ static int index_stream_convert_blob(un assert(path); assert(would_convert_to_git_filter_fd(path)); - convert_to_git_filter_fd(path, fd, &sbuf, + convert_to_git_filter_fd(&the_index, path, fd, &sbuf, write_object ? safe_crlf : SAFE_CRLF_FALSE); if (write_object) @@@ -3678,7 -3552,7 +3668,7 @@@ int index_fd(unsigned char *sha1, int f else if (!S_ISREG(st->st_mode)) ret = index_pipe(sha1, fd, type, path, flags); else if (st->st_size <= big_file_threshold || type != OBJ_BLOB || - (path && would_convert_to_git(path))) + (path && would_convert_to_git(&the_index, path))) ret = index_core(sha1, fd, xsize_t(st->st_size), type, path, flags); else @@@ -3745,32 -3619,22 +3735,32 @@@ void assert_sha1_type(const unsigned ch typename(expect)); } -static int for_each_file_in_obj_subdir(int subdir_nr, - struct strbuf *path, - each_loose_object_fn obj_cb, - each_loose_cruft_fn cruft_cb, - each_loose_subdir_fn subdir_cb, - void *data) +int for_each_file_in_obj_subdir(unsigned int subdir_nr, + struct strbuf *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data) { - size_t baselen = path->len; - DIR *dir = opendir(path->buf); + size_t origlen, baselen; + DIR *dir; struct dirent *de; int r = 0; + if (subdir_nr > 0xff) + BUG("invalid loose object subdirectory: %x", subdir_nr); + + origlen = path->len; + strbuf_complete(path, '/'); + strbuf_addf(path, "%02x", subdir_nr); + baselen = path->len; + + dir = opendir(path->buf); if (!dir) { - if (errno == ENOENT) - return 0; - return error_errno("unable to open %s", path->buf); + if (errno != ENOENT) + r = error_errno("unable to open %s", path->buf); + strbuf_setlen(path, origlen); + return r; } while ((de = readdir(dir))) { @@@ -3780,15 -3644,15 +3770,15 @@@ strbuf_setlen(path, baselen); strbuf_addf(path, "/%s", de->d_name); - if (strlen(de->d_name) == 38) { - char hex[41]; - unsigned char sha1[20]; + if (strlen(de->d_name) == GIT_SHA1_HEXSZ - 2) { + char hex[GIT_MAX_HEXSZ+1]; + struct object_id oid; - snprintf(hex, sizeof(hex), "%02x%s", - subdir_nr, de->d_name); - if (!get_sha1_hex(hex, sha1)) { + xsnprintf(hex, sizeof(hex), "%02x%s", + subdir_nr, de->d_name); + if (!get_oid_hex(hex, &oid)) { if (obj_cb) { - r = obj_cb(sha1, path->buf, data); + r = obj_cb(&oid, path->buf, data); if (r) break; } @@@ -3808,8 -3672,6 +3798,8 @@@ if (!r && subdir_cb) r = subdir_cb(subdir_nr, path->buf, data); + strbuf_setlen(path, origlen); + return r; } @@@ -3819,12 -3681,15 +3809,12 @@@ int for_each_loose_file_in_objdir_buf(s each_loose_subdir_fn subdir_cb, void *data) { - size_t baselen = path->len; int r = 0; int i; for (i = 0; i < 256; i++) { - strbuf_addf(path, "/%02x", i); r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb, subdir_cb, data); - strbuf_setlen(path, baselen); if (r) break; } @@@ -3893,13 -3758,13 +3883,13 @@@ static int for_each_object_in_pack(stru int r = 0; for (i = 0; i < p->num_objects; i++) { - const unsigned char *sha1 = nth_packed_object_sha1(p, i); + struct object_id oid; - if (!sha1) + if (!nth_packed_object_oid(&oid, p, i)) return error("unable to get sha1 of object %u in %s", i, p->pack_name); - r = cb(sha1, p, i, data); + r = cb(&oid, p, i, data); if (r) break; } @@@ -3926,119 -3791,3 +3916,119 @@@ int for_each_packed_object(each_packed_ } return r ? r : pack_errors; } + +static int check_stream_sha1(git_zstream *stream, + const char *hdr, + unsigned long size, + const char *path, + const unsigned char *expected_sha1) +{ + git_SHA_CTX c; + unsigned char real_sha1[GIT_MAX_RAWSZ]; + unsigned char buf[4096]; + unsigned long total_read; + int status = Z_OK; + + git_SHA1_Init(&c); + git_SHA1_Update(&c, hdr, stream->total_out); + + /* + * We already read some bytes into hdr, but the ones up to the NUL + * do not count against the object's content size. + */ + total_read = stream->total_out - strlen(hdr) - 1; + + /* + * This size comparison must be "<=" to read the final zlib packets; + * see the comment in unpack_sha1_rest for details. + */ + while (total_read <= size && + (status == Z_OK || status == Z_BUF_ERROR)) { + stream->next_out = buf; + stream->avail_out = sizeof(buf); + if (size - total_read < stream->avail_out) + stream->avail_out = size - total_read; + status = git_inflate(stream, Z_FINISH); + git_SHA1_Update(&c, buf, stream->next_out - buf); + total_read += stream->next_out - buf; + } + git_inflate_end(stream); + + if (status != Z_STREAM_END) { + error("corrupt loose object '%s'", sha1_to_hex(expected_sha1)); + return -1; + } + if (stream->avail_in) { + error("garbage at end of loose object '%s'", + sha1_to_hex(expected_sha1)); + return -1; + } + + git_SHA1_Final(real_sha1, &c); + if (hashcmp(expected_sha1, real_sha1)) { + error("sha1 mismatch for %s (expected %s)", path, + sha1_to_hex(expected_sha1)); + return -1; + } + + return 0; +} + +int read_loose_object(const char *path, + const unsigned char *expected_sha1, + enum object_type *type, + unsigned long *size, + void **contents) +{ + int ret = -1; + void *map = NULL; + unsigned long mapsize; + git_zstream stream; + char hdr[32]; + + *contents = NULL; + + map = map_sha1_file_1(path, NULL, &mapsize); + if (!map) { + error_errno("unable to mmap %s", path); + goto out; + } + + if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) { + error("unable to unpack header of %s", path); + goto out; + } + + *type = parse_sha1_header(hdr, size); + if (*type < 0) { + error("unable to parse header of %s", path); + git_inflate_end(&stream); + goto out; + } + + if (*type == OBJ_BLOB) { + if (check_stream_sha1(&stream, hdr, *size, path, expected_sha1) < 0) + goto out; + } else { + *contents = unpack_sha1_rest(&stream, hdr, *size, expected_sha1); + if (!*contents) { + error("unable to unpack contents of %s", path); + git_inflate_end(&stream); + goto out; + } + if (check_sha1_signature(expected_sha1, *contents, + *size, typename(*type))) { + error("sha1 mismatch for %s (expected %s)", path, + sha1_to_hex(expected_sha1)); + free(*contents); + goto out; + } + } + + ret = 0; /* everything checks out */ + +out: + if (map) + munmap(map, mapsize); + return ret; +}