From: Junio C Hamano Date: Fri, 4 Mar 2016 21:45:46 +0000 (-0800) Subject: Merge branch 'jk/pack-idx-corruption-safety' X-Git-Tag: v2.8.0-rc1~7 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/090de6b289ff2d9fc1c82ef85069bd6cba296d63?hp=-c Merge branch 'jk/pack-idx-corruption-safety' The code to read the pack data using the offsets stored in the pack idx file has been made more carefully check the validity of the data in the idx. * jk/pack-idx-corruption-safety: sha1_file.c: mark strings for translation use_pack: handle signed off_t overflow nth_packed_object_offset: bounds-check extended offset t5313: test bounds-checks of corrupted/malicious pack/idx files --- 090de6b289ff2d9fc1c82ef85069bd6cba296d63 diff --combined builtin/index-pack.c index 193908a619,98bdbb5e6c..45245199ae --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@@ -18,14 -18,16 +18,14 @@@ static const char index_pack_usage[] struct object_entry { struct pack_idx_entry idx; unsigned long size; - unsigned int hdr_size; - enum object_type type; - enum object_type real_type; - unsigned delta_depth; - int base_object_no; + unsigned char hdr_size; + signed char type; + signed char real_type; }; -union delta_base { - unsigned char sha1[20]; - off_t offset; +struct object_stat { + unsigned delta_depth; + int base_object_no; }; struct base_data { @@@ -47,35 -49,31 +47,35 @@@ struct thread_local int pack_fd; }; -/* - * Even if sizeof(union delta_base) == 24 on 64-bit archs, we really want - * to memcmp() only the first 20 bytes. - */ -#define UNION_BASE_SZ 20 - #define FLAG_LINK (1u<<20) #define FLAG_CHECKED (1u<<21) -struct delta_entry { - union delta_base base; +struct ofs_delta_entry { + off_t offset; + int obj_no; +}; + +struct ref_delta_entry { + unsigned char sha1[20]; int obj_no; }; static struct object_entry *objects; -static struct delta_entry *deltas; +static struct object_stat *obj_stat; +static struct ofs_delta_entry *ofs_deltas; +static struct ref_delta_entry *ref_deltas; static struct thread_local nothread_data; static int nr_objects; -static int nr_deltas; +static int nr_ofs_deltas; +static int nr_ref_deltas; +static int ref_deltas_alloc; static int nr_resolved_deltas; static int nr_threads; static int from_stdin; static int strict; static int do_fsck_object; +static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT; static int verbose; static int show_stat; static int check_self_contained_and_connected; @@@ -193,13 -191,13 +193,13 @@@ static void cleanup_thread(void #endif -static int mark_link(struct object *obj, int type, void *data) +static int mark_link(struct object *obj, int type, void *data, struct fsck_options *options) { if (!obj) return -1; if (type != OBJ_ANY && obj->type != type) - die(_("object type mismatch at %s"), sha1_to_hex(obj->sha1)); + die(_("object type mismatch at %s"), oid_to_hex(&obj->oid)); obj->flags |= FLAG_LINK; return 0; @@@ -217,13 -215,13 +217,13 @@@ static unsigned check_object(struct obj if (!(obj->flags & FLAG_CHECKED)) { unsigned long size; - int type = sha1_object_info(obj->sha1, &size); + int type = sha1_object_info(obj->oid.hash, &size); if (type <= 0) die(_("did not receive expected object %s"), - sha1_to_hex(obj->sha1)); + oid_to_hex(&obj->oid)); if (type != obj->type) die(_("object %s: expected type %s, found %s"), - sha1_to_hex(obj->sha1), + oid_to_hex(&obj->oid), typename(obj->type), typename(type)); obj->flags |= FLAG_CHECKED; return 1; @@@ -441,7 -439,7 +441,7 @@@ static void *unpack_entry_data(unsigne int hdrlen; if (!is_delta_type(type)) { - hdrlen = sprintf(hdr, "%s %lu", typename(type), size) + 1; + hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(type), size) + 1; git_SHA1_Init(&c); git_SHA1_Update(&c, hdr, hdrlen); } else @@@ -478,8 -476,7 +478,8 @@@ } static void *unpack_raw_entry(struct object_entry *obj, - union delta_base *delta_base, + off_t *ofs_offset, + unsigned char *ref_sha1, unsigned char *sha1) { unsigned char *p; @@@ -508,10 -505,11 +508,10 @@@ switch (obj->type) { case OBJ_REF_DELTA: - hashcpy(delta_base->sha1, fill(20)); + hashcpy(ref_sha1, fill(20)); use(20); break; case OBJ_OFS_DELTA: - memset(delta_base, 0, sizeof(*delta_base)); p = fill(1); c = *p; use(1); @@@ -525,8 -523,8 +525,8 @@@ use(1); base_offset = (base_offset << 7) + (c & 127); } - delta_base->offset = obj->idx.offset - base_offset; - if (delta_base->offset <= 0 || delta_base->offset >= obj->idx.offset) + *ofs_offset = obj->idx.offset - base_offset; + if (*ofs_offset <= 0 || *ofs_offset >= obj->idx.offset) bad_object(obj->idx.offset, _("delta base offset is out of bound")); break; case OBJ_COMMIT: @@@ -610,110 -608,55 +610,110 @@@ static void *get_data_from_pack(struct return unpack_data(obj, NULL, NULL); } -static int compare_delta_bases(const union delta_base *base1, - const union delta_base *base2, - enum object_type type1, - enum object_type type2) +static int compare_ofs_delta_bases(off_t offset1, off_t offset2, + enum object_type type1, + enum object_type type2) +{ + int cmp = type1 - type2; + if (cmp) + return cmp; + return offset1 < offset2 ? -1 : + offset1 > offset2 ? 1 : + 0; +} + +static int find_ofs_delta(const off_t offset, enum object_type type) +{ + int first = 0, last = nr_ofs_deltas; + + while (first < last) { + int next = (first + last) / 2; + struct ofs_delta_entry *delta = &ofs_deltas[next]; + int cmp; + + cmp = compare_ofs_delta_bases(offset, delta->offset, + type, objects[delta->obj_no].type); + if (!cmp) + return next; + if (cmp < 0) { + last = next; + continue; + } + first = next+1; + } + return -first-1; +} + +static void find_ofs_delta_children(off_t offset, + int *first_index, int *last_index, + enum object_type type) +{ + int first = find_ofs_delta(offset, type); + int last = first; + int end = nr_ofs_deltas - 1; + + if (first < 0) { + *first_index = 0; + *last_index = -1; + return; + } + while (first > 0 && ofs_deltas[first - 1].offset == offset) + --first; + while (last < end && ofs_deltas[last + 1].offset == offset) + ++last; + *first_index = first; + *last_index = last; +} + +static int compare_ref_delta_bases(const unsigned char *sha1, + const unsigned char *sha2, + enum object_type type1, + enum object_type type2) { int cmp = type1 - type2; if (cmp) return cmp; - return memcmp(base1, base2, UNION_BASE_SZ); + return hashcmp(sha1, sha2); } -static int find_delta(const union delta_base *base, enum object_type type) +static int find_ref_delta(const unsigned char *sha1, enum object_type type) { - int first = 0, last = nr_deltas; - - while (first < last) { - int next = (first + last) / 2; - struct delta_entry *delta = &deltas[next]; - int cmp; - - cmp = compare_delta_bases(base, &delta->base, - type, objects[delta->obj_no].type); - if (!cmp) - return next; - if (cmp < 0) { - last = next; - continue; - } - first = next+1; - } - return -first-1; + int first = 0, last = nr_ref_deltas; + + while (first < last) { + int next = (first + last) / 2; + struct ref_delta_entry *delta = &ref_deltas[next]; + int cmp; + + cmp = compare_ref_delta_bases(sha1, delta->sha1, + type, objects[delta->obj_no].type); + if (!cmp) + return next; + if (cmp < 0) { + last = next; + continue; + } + first = next+1; + } + return -first-1; } -static void find_delta_children(const union delta_base *base, - int *first_index, int *last_index, - enum object_type type) +static void find_ref_delta_children(const unsigned char *sha1, + int *first_index, int *last_index, + enum object_type type) { - int first = find_delta(base, type); + int first = find_ref_delta(sha1, type); int last = first; - int end = nr_deltas - 1; + int end = nr_ref_deltas - 1; if (first < 0) { *first_index = 0; *last_index = -1; return; } - while (first > 0 && !memcmp(&deltas[first - 1].base, base, UNION_BASE_SZ)) + while (first > 0 && !hashcmp(ref_deltas[first - 1].sha1, sha1)) --first; - while (last < end && !memcmp(&deltas[last + 1].base, base, UNION_BASE_SZ)) + while (last < end && !hashcmp(ref_deltas[last + 1].sha1, sha1)) ++last; *first_index = first; *last_index = last; @@@ -839,10 -782,11 +839,10 @@@ static void sha1_object(const void *dat if (!obj) die(_("invalid %s"), typename(type)); if (do_fsck_object && - fsck_object(obj, buf, size, 1, - fsck_error_function)) + fsck_object(obj, buf, size, &fsck_options)) die(_("Error in object")); - if (fsck_walk(obj, mark_link, NULL)) - die(_("Not all child objects of %s are reachable"), sha1_to_hex(obj->sha1)); + if (fsck_walk(obj, NULL, &fsck_options)) + die(_("Not all child objects of %s are reachable"), oid_to_hex(&obj->oid)); if (obj->type == OBJ_TREE) { struct tree *item = (struct tree *) obj; @@@ -929,15 -873,13 +929,15 @@@ static void resolve_delta(struct object void *base_data, *delta_data; if (show_stat) { - delta_obj->delta_depth = base->obj->delta_depth + 1; + int i = delta_obj - objects; + int j = base->obj - objects; + obj_stat[i].delta_depth = obj_stat[j].delta_depth + 1; deepest_delta_lock(); - if (deepest_delta < delta_obj->delta_depth) - deepest_delta = delta_obj->delta_depth; + if (deepest_delta < obj_stat[i].delta_depth) + deepest_delta = obj_stat[i].delta_depth; deepest_delta_unlock(); + obj_stat[i].base_object_no = j; } - delta_obj->base_object_no = base->obj - objects; delta_data = get_data_from_pack(delta_obj); base_data = get_base_data(base); result->obj = delta_obj; @@@ -960,7 -902,7 +960,7 @@@ * "want"; if so, swap in "set" and return true. Otherwise, leave it untouched * and return false. */ -static int compare_and_swap_type(enum object_type *type, +static int compare_and_swap_type(signed char *type, enum object_type want, enum object_type set) { @@@ -979,13 -921,16 +979,13 @@@ static struct base_data *find_unresolve struct base_data *prev_base) { if (base->ref_last == -1 && base->ofs_last == -1) { - union delta_base base_spec; + find_ref_delta_children(base->obj->idx.sha1, + &base->ref_first, &base->ref_last, + OBJ_REF_DELTA); - hashcpy(base_spec.sha1, base->obj->idx.sha1); - find_delta_children(&base_spec, - &base->ref_first, &base->ref_last, OBJ_REF_DELTA); - - memset(&base_spec, 0, sizeof(base_spec)); - base_spec.offset = base->obj->idx.offset; - find_delta_children(&base_spec, - &base->ofs_first, &base->ofs_last, OBJ_OFS_DELTA); + find_ofs_delta_children(base->obj->idx.offset, + &base->ofs_first, &base->ofs_last, + OBJ_OFS_DELTA); if (base->ref_last == -1 && base->ofs_last == -1) { free(base->data); @@@ -996,7 -941,7 +996,7 @@@ } if (base->ref_first <= base->ref_last) { - struct object_entry *child = objects + deltas[base->ref_first].obj_no; + struct object_entry *child = objects + ref_deltas[base->ref_first].obj_no; struct base_data *result = alloc_base_data(); if (!compare_and_swap_type(&child->real_type, OBJ_REF_DELTA, @@@ -1012,7 -957,7 +1012,7 @@@ } if (base->ofs_first <= base->ofs_last) { - struct object_entry *child = objects + deltas[base->ofs_first].obj_no; + struct object_entry *child = objects + ofs_deltas[base->ofs_first].obj_no; struct base_data *result = alloc_base_data(); assert(child->real_type == OBJ_OFS_DELTA); @@@ -1048,22 -993,15 +1048,22 @@@ static void find_unresolved_deltas(stru } } -static int compare_delta_entry(const void *a, const void *b) +static int compare_ofs_delta_entry(const void *a, const void *b) +{ + const struct ofs_delta_entry *delta_a = a; + const struct ofs_delta_entry *delta_b = b; + + return delta_a->offset < delta_b->offset ? -1 : + delta_a->offset > delta_b->offset ? 1 : + 0; +} + +static int compare_ref_delta_entry(const void *a, const void *b) { - const struct delta_entry *delta_a = a; - const struct delta_entry *delta_b = b; + const struct ref_delta_entry *delta_a = a; + const struct ref_delta_entry *delta_b = b; - /* group by type (ref vs ofs) and then by value (sha-1 or offset) */ - return compare_delta_bases(&delta_a->base, &delta_b->base, - objects[delta_a->obj_no].type, - objects[delta_b->obj_no].type); + return hashcmp(delta_a->sha1, delta_b->sha1); } static void resolve_base(struct object_entry *obj) @@@ -1109,8 -1047,7 +1109,8 @@@ static void *threaded_second_pass(void static void parse_pack_objects(unsigned char *sha1) { int i, nr_delays = 0; - struct delta_entry *delta = deltas; + struct ofs_delta_entry *ofs_delta = ofs_deltas; + unsigned char ref_delta_sha1[20]; struct stat st; if (verbose) @@@ -1119,18 -1056,12 +1119,18 @@@ nr_objects); for (i = 0; i < nr_objects; i++) { struct object_entry *obj = &objects[i]; - void *data = unpack_raw_entry(obj, &delta->base, obj->idx.sha1); + void *data = unpack_raw_entry(obj, &ofs_delta->offset, + ref_delta_sha1, obj->idx.sha1); obj->real_type = obj->type; - if (is_delta_type(obj->type)) { - nr_deltas++; - delta->obj_no = i; - delta++; + if (obj->type == OBJ_OFS_DELTA) { + nr_ofs_deltas++; + ofs_delta->obj_no = i; + ofs_delta++; + } else if (obj->type == OBJ_REF_DELTA) { + ALLOC_GROW(ref_deltas, nr_ref_deltas + 1, ref_deltas_alloc); + hashcpy(ref_deltas[nr_ref_deltas].sha1, ref_delta_sha1); + ref_deltas[nr_ref_deltas].obj_no = i; + nr_ref_deltas++; } else if (!data) { /* large blobs, check later */ obj->real_type = OBJ_BAD; @@@ -1181,18 -1112,15 +1181,18 @@@ static void resolve_deltas(void { int i; - if (!nr_deltas) + if (!nr_ofs_deltas && !nr_ref_deltas) return; /* Sort deltas by base SHA1/offset for fast searching */ - qsort(deltas, nr_deltas, sizeof(struct delta_entry), - compare_delta_entry); + qsort(ofs_deltas, nr_ofs_deltas, sizeof(struct ofs_delta_entry), + compare_ofs_delta_entry); + qsort(ref_deltas, nr_ref_deltas, sizeof(struct ref_delta_entry), + compare_ref_delta_entry); if (verbose) - progress = start_progress(_("Resolving deltas"), nr_deltas); + progress = start_progress(_("Resolving deltas"), + nr_ref_deltas + nr_ofs_deltas); #ifndef NO_PTHREADS nr_dispatched = 0; @@@ -1227,10 -1155,10 +1227,10 @@@ * - append objects to convert thin pack to full pack if required * - write the final 20-byte SHA-1 */ -static void fix_unresolved_deltas(struct sha1file *f, int nr_unresolved); +static void fix_unresolved_deltas(struct sha1file *f); static void conclude_pack(int fix_thin_pack, const char *curr_pack, unsigned char *pack_sha1) { - if (nr_deltas == nr_resolved_deltas) { + if (nr_ref_deltas + nr_ofs_deltas == nr_resolved_deltas) { stop_progress(&progress); /* Flush remaining pack final 20-byte SHA1. */ flush(); @@@ -1241,7 -1169,7 +1241,7 @@@ struct sha1file *f; unsigned char read_sha1[20], tail_sha1[20]; struct strbuf msg = STRBUF_INIT; - int nr_unresolved = nr_deltas - nr_resolved_deltas; + int nr_unresolved = nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas; int nr_objects_initial = nr_objects; if (nr_unresolved <= 0) die(_("confusion beyond insanity")); @@@ -1249,7 -1177,7 +1249,7 @@@ memset(objects + nr_objects + 1, 0, nr_unresolved * sizeof(*objects)); f = sha1fd(output_fd, curr_pack); - fix_unresolved_deltas(f, nr_unresolved); + fix_unresolved_deltas(f); strbuf_addf(&msg, _("completed with %d local objects"), nr_objects - nr_objects_initial); stop_progress_msg(&progress, msg.buf); @@@ -1263,11 -1191,11 +1263,11 @@@ die(_("Unexpected tail checksum for %s " "(disk corruption?)"), curr_pack); } - if (nr_deltas != nr_resolved_deltas) + if (nr_ofs_deltas + nr_ref_deltas != nr_resolved_deltas) die(Q_("pack has %d unresolved delta", "pack has %d unresolved deltas", - nr_deltas - nr_resolved_deltas), - nr_deltas - nr_resolved_deltas); + nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas), + nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas); } static int write_compressed(struct sha1file *f, void *in, unsigned int size) @@@ -1326,15 -1254,15 +1326,15 @@@ static struct object_entry *append_obj_ static int delta_pos_compare(const void *_a, const void *_b) { - struct delta_entry *a = *(struct delta_entry **)_a; - struct delta_entry *b = *(struct delta_entry **)_b; + struct ref_delta_entry *a = *(struct ref_delta_entry **)_a; + struct ref_delta_entry *b = *(struct ref_delta_entry **)_b; return a->obj_no - b->obj_no; } -static void fix_unresolved_deltas(struct sha1file *f, int nr_unresolved) +static void fix_unresolved_deltas(struct sha1file *f) { - struct delta_entry **sorted_by_pos; - int i, n = 0; + struct ref_delta_entry **sorted_by_pos; + int i; /* * Since many unresolved deltas may well be themselves base objects @@@ -1346,26 -1274,29 +1346,26 @@@ * before deltas depending on them, a good heuristic is to start * resolving deltas in the same order as their position in the pack. */ - sorted_by_pos = xmalloc(nr_unresolved * sizeof(*sorted_by_pos)); - for (i = 0; i < nr_deltas; i++) { - if (objects[deltas[i].obj_no].real_type != OBJ_REF_DELTA) - continue; - sorted_by_pos[n++] = &deltas[i]; - } - qsort(sorted_by_pos, n, sizeof(*sorted_by_pos), delta_pos_compare); + ALLOC_ARRAY(sorted_by_pos, nr_ref_deltas); + for (i = 0; i < nr_ref_deltas; i++) + sorted_by_pos[i] = &ref_deltas[i]; + qsort(sorted_by_pos, nr_ref_deltas, sizeof(*sorted_by_pos), delta_pos_compare); - for (i = 0; i < n; i++) { - struct delta_entry *d = sorted_by_pos[i]; + for (i = 0; i < nr_ref_deltas; i++) { + struct ref_delta_entry *d = sorted_by_pos[i]; enum object_type type; struct base_data *base_obj = alloc_base_data(); if (objects[d->obj_no].real_type != OBJ_REF_DELTA) continue; - base_obj->data = read_sha1_file(d->base.sha1, &type, &base_obj->size); + base_obj->data = read_sha1_file(d->sha1, &type, &base_obj->size); if (!base_obj->data) continue; - if (check_sha1_signature(d->base.sha1, base_obj->data, + if (check_sha1_signature(d->sha1, base_obj->data, base_obj->size, typename(type))) - die(_("local object %s is corrupt"), sha1_to_hex(d->base.sha1)); - base_obj->obj = append_obj_to_pack(f, d->base.sha1, + die(_("local object %s is corrupt"), sha1_to_hex(d->sha1)); + base_obj->obj = append_obj_to_pack(f, d->sha1, base_obj->data, base_obj->size, type); find_unresolved_deltas(base_obj); display_progress(progress, nr_resolved_deltas); @@@ -1421,7 -1352,7 +1421,7 @@@ static void final(const char *final_pac get_object_directory(), sha1_to_hex(sha1)); final_pack_name = name; } - if (move_temp_to_file(curr_pack_name, final_pack_name)) + if (finalize_object_file(curr_pack_name, final_pack_name)) die(_("cannot store pack file")); } else if (from_stdin) chmod(final_pack_name, 0444); @@@ -1432,7 -1363,7 +1432,7 @@@ get_object_directory(), sha1_to_hex(sha1)); final_index_name = name; } - if (move_temp_to_file(curr_index_name, final_index_name)) + if (finalize_object_file(curr_index_name, final_index_name)) die(_("cannot store index file")); } else chmod(final_index_name, 0444); @@@ -1514,6 -1445,7 +1514,7 @@@ static void read_v2_anomalous_offsets(s if (!(off & 0x80000000)) continue; off = off & 0x7fffffff; + check_pack_index_ptr(p, &idx2[off * 2]); if (idx2[off * 2]) continue; /* @@@ -1557,7 -1489,7 +1558,7 @@@ static void read_idx_option(struct pack static void show_pack_info(int stat_only) { - int i, baseobjects = nr_objects - nr_deltas; + int i, baseobjects = nr_objects - nr_ref_deltas - nr_ofs_deltas; unsigned long *chain_histogram = NULL; if (deepest_delta) @@@ -1567,7 -1499,7 +1568,7 @@@ struct object_entry *obj = &objects[i]; if (is_delta_type(obj->type)) - chain_histogram[obj->delta_depth - 1]++; + chain_histogram[obj_stat[i].delta_depth - 1]++; if (stat_only) continue; printf("%s %-6s %lu %lu %"PRIuMAX, @@@ -1576,8 -1508,8 +1577,8 @@@ (unsigned long)(obj[1].idx.offset - obj->idx.offset), (uintmax_t)obj->idx.offset); if (is_delta_type(obj->type)) { - struct object_entry *bobj = &objects[obj->base_object_no]; - printf(" %u %s", obj->delta_depth, sha1_to_hex(bobj->idx.sha1)); + struct object_entry *bobj = &objects[obj_stat[i].base_object_no]; + printf(" %u %s", obj_stat[i].delta_depth, sha1_to_hex(bobj->idx.sha1)); } putchar('\n'); } @@@ -1615,7 -1547,6 +1616,7 @@@ int cmd_index_pack(int argc, const cha usage(index_pack_usage); check_replace_refs = 0; + fsck_options.walk = mark_link; reset_pack_idx_option(&opts); git_config(git_index_pack_config, &opts); @@@ -1633,10 -1564,6 +1634,10 @@@ } else if (!strcmp(arg, "--strict")) { strict = 1; do_fsck_object = 1; + } else if (skip_prefix(arg, "--strict=", &arg)) { + strict = 1; + do_fsck_object = 1; + fsck_set_msg_types(&fsck_options, arg); } else if (!strcmp(arg, "--check-self-contained-and-connected")) { strict = 1; check_self_contained_and_connected = 1; @@@ -1744,22 -1671,19 +1745,22 @@@ curr_pack = open_pack_file(pack_name); parse_pack_header(); - objects = xcalloc(nr_objects + 1, sizeof(struct object_entry)); - deltas = xcalloc(nr_objects, sizeof(struct delta_entry)); + objects = xcalloc(st_add(nr_objects, 1), sizeof(struct object_entry)); + if (show_stat) + obj_stat = xcalloc(st_add(nr_objects, 1), sizeof(struct object_stat)); + ofs_deltas = xcalloc(nr_objects, sizeof(struct ofs_delta_entry)); parse_pack_objects(pack_sha1); resolve_deltas(); conclude_pack(fix_thin_pack, curr_pack, pack_sha1); - free(deltas); + free(ofs_deltas); + free(ref_deltas); if (strict) foreign_nr = check_objects(); if (show_stat) show_pack_info(stat_only); - idx_objects = xmalloc((nr_objects) * sizeof(struct pack_idx_entry *)); + ALLOC_ARRAY(idx_objects, nr_objects); for (i = 0; i < nr_objects; i++) idx_objects[i] = &objects[i].idx; curr_index = write_idx_file(index_name, idx_objects, nr_objects, &opts, pack_sha1); diff --combined cache.h index d7ff46ec4a,6c9aaa1ae6..b829410f6d --- a/cache.h +++ b/cache.h @@@ -9,32 -9,13 +9,32 @@@ #include "convert.h" #include "trace.h" #include "string-list.h" +#include "pack-revindex.h" #include SHA1_HEADER -#ifndef git_SHA_CTX -#define git_SHA_CTX SHA_CTX -#define git_SHA1_Init SHA1_Init -#define git_SHA1_Update SHA1_Update -#define git_SHA1_Final SHA1_Final +#ifndef platform_SHA_CTX +/* + * platform's underlying implementation of SHA-1; could be OpenSSL, + * blk_SHA, Apple CommonCrypto, etc... Note that including + * SHA1_HEADER may have already defined platform_SHA_CTX for our + * own implementations like block-sha1 and ppc-sha1, so we list + * the default for OpenSSL compatible SHA-1 implementations here. + */ +#define platform_SHA_CTX SHA_CTX +#define platform_SHA1_Init SHA1_Init +#define platform_SHA1_Update SHA1_Update +#define platform_SHA1_Final SHA1_Final +#endif + +#define git_SHA_CTX platform_SHA_CTX +#define git_SHA1_Init platform_SHA1_Init +#define git_SHA1_Update platform_SHA1_Update +#define git_SHA1_Final platform_SHA1_Final + +#ifdef SHA1_MAX_BLOCK_SIZE +#include "compat/sha1-chunked.h" +#undef git_SHA1_Update +#define git_SHA1_Update git_SHA1_Update_Chunked #endif #include @@@ -62,14 -43,6 +62,14 @@@ int git_deflate_end_gently(git_zstream int git_deflate(git_zstream *, int flush); unsigned long git_deflate_bound(git_zstream *, unsigned long); +/* The length in bytes and in hex digits of an object name (SHA-1 value). */ +#define GIT_SHA1_RAWSZ 20 +#define GIT_SHA1_HEXSZ (2 * GIT_SHA1_RAWSZ) + +struct object_id { + unsigned char hash[GIT_SHA1_RAWSZ]; +}; + #if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT) #define DTYPE(de) ((de)->d_type) #else @@@ -215,7 -188,7 +215,7 @@@ struct cache_entry #define CE_INTENT_TO_ADD (1 << 29) #define CE_SKIP_WORKTREE (1 << 30) /* CE_EXTENDED2 is for future extension */ -#define CE_EXTENDED2 (1 << 31) +#define CE_EXTENDED2 (1U << 31) #define CE_EXTENDED_FLAGS (CE_INTENT_TO_ADD | CE_SKIP_WORKTREE) @@@ -229,9 -202,7 +229,9 @@@ #error "CE_EXTENDED_FLAGS out of range" #endif +/* Forward structure decls */ struct pathspec; +struct child_process; /* * Copy the sha1 and stat state of a cache entry from one to @@@ -262,7 -233,6 +262,7 @@@ static inline unsigned create_ce_flags( #define ce_uptodate(ce) ((ce)->ce_flags & CE_UPTODATE) #define ce_skip_worktree(ce) ((ce)->ce_flags & CE_SKIP_WORKTREE) #define ce_mark_uptodate(ce) ((ce)->ce_flags |= CE_UPTODATE) +#define ce_intent_to_add(ce) ((ce)->ce_flags & CE_INTENT_TO_ADD) #define ce_permissions(mode) (((mode) & 0100) ? 0755 : 0644) static inline unsigned int create_ce_mode(unsigned int mode) @@@ -319,11 -289,8 +319,11 @@@ static inline unsigned int canon_mode(u #define RESOLVE_UNDO_CHANGED (1 << 4) #define CACHE_TREE_CHANGED (1 << 5) #define SPLIT_INDEX_ORDERED (1 << 6) +#define UNTRACKED_CHANGED (1 << 7) struct split_index; +struct untracked_cache; + struct index_state { struct cache_entry **cache; unsigned int version; @@@ -337,7 -304,6 +337,7 @@@ struct hashmap name_hash; struct hashmap dir_hash; unsigned char sha1[20]; + struct untracked_cache *untracked; }; extern struct index_state the_index; @@@ -404,7 -370,6 +404,7 @@@ static inline enum object_type object_t /* Double-check local_repo_env below if you add to this list. */ #define GIT_DIR_ENVIRONMENT "GIT_DIR" +#define GIT_COMMON_DIR_ENVIRONMENT "GIT_COMMON_DIR" #define GIT_NAMESPACE_ENVIRONMENT "GIT_NAMESPACE" #define GIT_WORK_TREE_ENVIRONMENT "GIT_WORK_TREE" #define GIT_PREFIX_ENVIRONMENT "GIT_PREFIX" @@@ -419,7 -384,6 +419,7 @@@ #define EXEC_PATH_ENVIRONMENT "GIT_EXEC_PATH" #define CEILING_DIRECTORIES_ENVIRONMENT "GIT_CEILING_DIRECTORIES" #define NO_REPLACE_OBJECTS_ENVIRONMENT "GIT_NO_REPLACE_OBJECTS" +#define GIT_REPLACE_REF_BASE_ENVIRONMENT "GIT_REPLACE_REF_BASE" #define GITATTRIBUTES_FILE ".gitattributes" #define INFOATTRIBUTES_FILE "info/attributes" #define ATTRIBUTE_MACRO_PREFIX "[attr]" @@@ -459,46 -423,15 +459,46 @@@ extern int is_inside_git_dir(void) extern char *git_work_tree_cfg; extern int is_inside_work_tree(void); extern const char *get_git_dir(void); -extern int is_git_directory(const char *path); +extern const char *get_git_common_dir(void); extern char *get_object_directory(void); extern char *get_index_file(void); extern char *get_graft_file(void); extern int set_git_dir(const char *path); +extern int get_common_dir_noenv(struct strbuf *sb, const char *gitdir); +extern int get_common_dir(struct strbuf *sb, const char *gitdir); extern const char *get_git_namespace(void); extern const char *strip_namespace(const char *namespaced_ref); extern const char *get_git_work_tree(void); -extern const char *read_gitfile(const char *path); + +/* + * Return true if the given path is a git directory; note that this _just_ + * looks at the directory itself. If you want to know whether "foo/.git" + * is a repository, you must feed that path, not just "foo". + */ +extern int is_git_directory(const char *path); + +/* + * Return 1 if the given path is the root of a git repository or + * submodule, else 0. Will not return 1 for bare repositories with the + * exception of creating a bare repository in "foo/.git" and calling + * is_git_repository("foo"). + * + * If we run into read errors, we err on the side of saying "yes, it is", + * as we usually consider sub-repos precious, and would prefer to err on the + * side of not disrupting or deleting them. + */ +extern int is_nonbare_repository_dir(struct strbuf *path); + +#define READ_GITFILE_ERR_STAT_FAILED 1 +#define READ_GITFILE_ERR_NOT_A_FILE 2 +#define READ_GITFILE_ERR_OPEN_FAILED 3 +#define READ_GITFILE_ERR_READ_FAILED 4 +#define READ_GITFILE_ERR_INVALID_FORMAT 5 +#define READ_GITFILE_ERR_NO_PATH 6 +#define READ_GITFILE_ERR_NOT_A_REPO 7 +#define READ_GITFILE_ERR_TOO_LARGE 8 +extern const char *read_gitfile_gently(const char *path, int *return_error_code); +#define read_gitfile(path) read_gitfile_gently((path), NULL) extern const char *resolve_gitdir(const char *suspect); extern void set_git_work_tree(const char *tree); @@@ -561,8 -494,7 +561,8 @@@ extern int write_locked_index(struct in extern int discard_index(struct index_state *); extern int unmerged_index(const struct index_state *); extern int verify_path(const char *path); -extern struct cache_entry *index_dir_exists(struct index_state *istate, const char *name, int namelen); +extern int index_dir_exists(struct index_state *istate, const char *name, int namelen); +extern void adjust_dirname_case(struct index_state *istate, char *name); extern struct cache_entry *index_file_exists(struct index_state *istate, const char *name, int namelen, int igncase); extern int index_name_pos(const struct index_state *, const char *name, int namelen); #define ADD_CACHE_OK_TO_ADD 1 /* Ok to add */ @@@ -620,8 -552,6 +620,8 @@@ extern void fill_stat_data(struct stat_ * INODE_CHANGED, and DATA_CHANGED. */ extern int match_stat_data(const struct stat_data *sd, struct stat *st); +extern int match_stat_data_racy(const struct index_state *istate, + const struct stat_data *sd, struct stat *st); extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st); @@@ -638,6 -568,8 +638,6 @@@ extern void update_index_if_able(struc extern int hold_locked_index(struct lock_file *, int); extern void set_alternate_index_output(const char *); -extern int delete_ref(const char *, const unsigned char *sha1, unsigned int flags); - /* Environment bits from configuration mechanism */ extern int trust_executable_bit; extern int trust_ctime; @@@ -673,7 -605,6 +673,7 @@@ extern unsigned long pack_size_limit_cf * been sought but there were none. */ extern int check_replace_refs; +extern char *git_replace_ref_base; extern int fsync_object_files; extern int core_preload_index; @@@ -681,7 -612,6 +681,7 @@@ extern int core_apply_sparse_checkout extern int precomposed_unicode; extern int protect_hfs; extern int protect_ntfs; +extern int git_db_env, git_index_env, git_graft_env, git_common_dir_env; /* * Include broken refs in all ref iterations, which will @@@ -738,15 -668,8 +738,15 @@@ extern char *notes_ref_name extern int grafts_replace_parents; +/* + * GIT_REPO_VERSION is the version we write by default. The + * _READ variant is the highest number we know how to + * handle. + */ #define GIT_REPO_VERSION 0 +#define GIT_REPO_VERSION_READ 1 extern int repository_format_version; +extern int repository_format_precious_objects; extern int check_repository_format(void); #define MTIME_CHANGED 0x0001 @@@ -757,61 -680,21 +757,61 @@@ #define DATA_CHANGED 0x0020 #define TYPE_CHANGED 0x0040 +/* + * Return a statically allocated filename, either generically (mkpath), in + * the repository directory (git_path), or in a submodule's repository + * directory (git_path_submodule). In all cases, note that the result + * may be overwritten by another call to _any_ of the functions. Consider + * using the safer "dup" or "strbuf" formats below (in some cases, the + * unsafe versions have already been removed). + */ +extern const char *mkpath(const char *fmt, ...) __attribute__((format (printf, 1, 2))); +extern const char *git_path(const char *fmt, ...) __attribute__((format (printf, 1, 2))); + extern char *mksnpath(char *buf, size_t n, const char *fmt, ...) __attribute__((format (printf, 3, 4))); -extern char *git_snpath(char *buf, size_t n, const char *fmt, ...) +extern void strbuf_git_path(struct strbuf *sb, const char *fmt, ...) + __attribute__((format (printf, 2, 3))); +extern char *git_path_buf(struct strbuf *buf, const char *fmt, ...) + __attribute__((format (printf, 2, 3))); +extern void strbuf_git_path_submodule(struct strbuf *sb, const char *path, + const char *fmt, ...) __attribute__((format (printf, 3, 4))); extern char *git_pathdup(const char *fmt, ...) __attribute__((format (printf, 1, 2))); extern char *mkpathdup(const char *fmt, ...) __attribute__((format (printf, 1, 2))); - -/* Return a statically allocated filename matching the sha1 signature */ -extern char *mkpath(const char *fmt, ...) __attribute__((format (printf, 1, 2))); -extern char *git_path(const char *fmt, ...) __attribute__((format (printf, 1, 2))); -extern char *git_path_submodule(const char *path, const char *fmt, ...) +extern char *git_pathdup_submodule(const char *path, const char *fmt, ...) __attribute__((format (printf, 2, 3))); +extern void report_linked_checkout_garbage(void); + +/* + * You can define a static memoized git path like: + * + * static GIT_PATH_FUNC(git_path_foo, "FOO"); + * + * or use one of the global ones below. + */ +#define GIT_PATH_FUNC(func, filename) \ + const char *func(void) \ + { \ + static char *ret; \ + if (!ret) \ + ret = git_pathdup(filename); \ + return ret; \ + } + +const char *git_path_cherry_pick_head(void); +const char *git_path_revert_head(void); +const char *git_path_squash_msg(void); +const char *git_path_merge_msg(void); +const char *git_path_merge_rr(void); +const char *git_path_merge_mode(void); +const char *git_path_merge_head(void); +const char *git_path_fetch_head(void); +const char *git_path_shallow(void); + /* * Return the name of the file in the local object database that would * be used to store a loose object with the specified sha1. The @@@ -834,32 -717,14 +834,32 @@@ extern char *sha1_pack_name(const unsig */ extern char *sha1_pack_index_name(const unsigned char *sha1); -extern const char *find_unique_abbrev(const unsigned char *sha1, int); -extern const unsigned char null_sha1[20]; +/* + * Return an abbreviated sha1 unique within this repository's object database. + * The result will be at least `len` characters long, and will be NUL + * terminated. + * + * The non-`_r` version returns a static buffer which will be overwritten by + * subsequent calls. + * + * The `_r` variant writes to a buffer supplied by the caller, which must be at + * least `GIT_SHA1_HEXSZ + 1` bytes. The return value is the number of bytes + * written (excluding the NUL terminator). + * + * Note that while this version avoids the static buffer, it is not fully + * reentrant, as it calls into other non-reentrant git code. + */ +extern const char *find_unique_abbrev(const unsigned char *sha1, int len); +extern int find_unique_abbrev_r(char *hex, const unsigned char *sha1, int len); + +extern const unsigned char null_sha1[GIT_SHA1_RAWSZ]; +extern const struct object_id null_oid; static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2) { int i; - for (i = 0; i < 20; i++, sha1++, sha2++) { + for (i = 0; i < GIT_SHA1_RAWSZ; i++, sha1++, sha2++) { if (*sha1 != *sha2) return *sha1 - *sha2; } @@@ -867,42 -732,20 +867,42 @@@ return 0; } +static inline int oidcmp(const struct object_id *oid1, const struct object_id *oid2) +{ + return hashcmp(oid1->hash, oid2->hash); +} + static inline int is_null_sha1(const unsigned char *sha1) { return !hashcmp(sha1, null_sha1); } +static inline int is_null_oid(const struct object_id *oid) +{ + return !hashcmp(oid->hash, null_sha1); +} + static inline void hashcpy(unsigned char *sha_dst, const unsigned char *sha_src) { - memcpy(sha_dst, sha_src, 20); + memcpy(sha_dst, sha_src, GIT_SHA1_RAWSZ); } + +static inline void oidcpy(struct object_id *dst, const struct object_id *src) +{ + hashcpy(dst->hash, src->hash); +} + static inline void hashclr(unsigned char *hash) { - memset(hash, 0, 20); + memset(hash, 0, GIT_SHA1_RAWSZ); } +static inline void oidclr(struct object_id *oid) +{ + hashclr(oid->hash); +} + + #define EMPTY_TREE_SHA1_HEX \ "4b825dc642cb6eb9a060e54bf8d69288fbee4904" #define EMPTY_TREE_SHA1_BIN_LITERAL \ @@@ -1001,7 -844,6 +1001,7 @@@ extern char *xdg_config_home(const cha /* object replacement */ #define LOOKUP_REPLACE_OBJECT 1 +#define LOOKUP_UNKNOWN_OBJECT 2 extern void *read_sha1_file_extended(const unsigned char *sha1, enum object_type *type, unsigned long *size, unsigned flag); static inline void *read_sha1_file(const unsigned char *sha1, enum object_type *type, unsigned long *size) { @@@ -1051,7 -893,7 +1051,7 @@@ extern int do_check_packed_object_crc extern int check_sha1_signature(const unsigned char *sha1, void *buf, unsigned long size, const char *type); -extern int move_temp_to_file(const char *tmpfile, const char *filename); +extern int finalize_object_file(const char *tmpfile, const char *filename); extern int has_sha1_pack(const unsigned char *sha1); @@@ -1071,9 -913,6 +1071,9 @@@ static inline int has_sha1_file(const u return has_sha1_file_with_flags(sha1, 0); } +/* Same as the above, except for struct object_id. */ +extern int has_object_file(const struct object_id *oid); + /* * Return true iff an alternate object database has a loose object * with the specified name. This function does not respect replace @@@ -1099,21 -938,15 +1099,21 @@@ struct object_context unsigned char tree[20]; char path[PATH_MAX]; unsigned mode; + /* + * symlink_path is only used by get_tree_entry_follow_symlinks, + * and only for symlinks that point outside the repository. + */ + struct strbuf symlink_path; }; -#define GET_SHA1_QUIETLY 01 -#define GET_SHA1_COMMIT 02 -#define GET_SHA1_COMMITTISH 04 -#define GET_SHA1_TREE 010 -#define GET_SHA1_TREEISH 020 -#define GET_SHA1_BLOB 040 -#define GET_SHA1_ONLY_TO_DIE 04000 +#define GET_SHA1_QUIETLY 01 +#define GET_SHA1_COMMIT 02 +#define GET_SHA1_COMMITTISH 04 +#define GET_SHA1_TREE 010 +#define GET_SHA1_TREEISH 020 +#define GET_SHA1_BLOB 040 +#define GET_SHA1_FOLLOW_SYMLINKS 0100 +#define GET_SHA1_ONLY_TO_DIE 04000 extern int get_sha1(const char *str, unsigned char *sha1); extern int get_sha1_commit(const char *str, unsigned char *sha1); @@@ -1135,26 -968,78 +1135,26 @@@ extern int for_each_abbrev(const char * * null-terminated string. */ extern int get_sha1_hex(const char *hex, unsigned char *sha1); - -extern char *sha1_to_hex(const unsigned char *sha1); /* static buffer result! */ -extern int read_ref_full(const char *refname, int resolve_flags, - unsigned char *sha1, int *flags); -extern int read_ref(const char *refname, unsigned char *sha1); +extern int get_oid_hex(const char *hex, struct object_id *sha1); /* - * Resolve a reference, recursively following symbolic refererences. - * - * Store the referred-to object's name in sha1 and return the name of - * the non-symbolic reference that ultimately pointed at it. The - * return value, if not NULL, is a pointer into either a static buffer - * or the input ref. - * - * If the reference cannot be resolved to an object, the behavior - * depends on the RESOLVE_REF_READING flag: - * - * - If RESOLVE_REF_READING is set, return NULL. - * - * - If RESOLVE_REF_READING is not set, clear sha1 and return the name of - * the last reference name in the chain, which will either be a non-symbolic - * reference or an undefined reference. If this is a prelude to - * "writing" to the ref, the return value is the name of the ref - * that will actually be created or changed. - * - * If the RESOLVE_REF_NO_RECURSE flag is passed, only resolves one - * level of symbolic reference. The value stored in sha1 for a symbolic - * reference will always be null_sha1 in this case, and the return - * value is the reference that the symref refers to directly. + * Convert a binary sha1 to its hex equivalent. The `_r` variant is reentrant, + * and writes the NUL-terminated output to the buffer `out`, which must be at + * least `GIT_SHA1_HEXSZ + 1` bytes, and returns a pointer to out for + * convenience. * - * If flags is non-NULL, set the value that it points to the - * combination of REF_ISPACKED (if the reference was found among the - * packed references), REF_ISSYMREF (if the initial reference was a - * symbolic reference), REF_BAD_NAME (if the reference name is ill - * formed --- see RESOLVE_REF_ALLOW_BAD_NAME below), and REF_ISBROKEN - * (if the ref is malformed or has a bad name). See refs.h for more detail - * on each flag. + * The non-`_r` variant returns a static buffer, but uses a ring of 4 + * buffers, making it safe to make multiple calls for a single statement, like: * - * If ref is not a properly-formatted, normalized reference, return - * NULL. If more than MAXDEPTH recursive symbolic lookups are needed, - * give up and return NULL. - * - * RESOLVE_REF_ALLOW_BAD_NAME allows resolving refs even when their - * name is invalid according to git-check-ref-format(1). If the name - * is bad then the value stored in sha1 will be null_sha1 and the two - * flags REF_ISBROKEN and REF_BAD_NAME will be set. - * - * Even with RESOLVE_REF_ALLOW_BAD_NAME, names that escape the refs/ - * directory and do not consist of all caps and underscores cannot be - * resolved. The function returns NULL for such ref names. - * Caps and underscores refers to the special refs, such as HEAD, - * FETCH_HEAD and friends, that all live outside of the refs/ directory. + * printf("%s -> %s", sha1_to_hex(one), sha1_to_hex(two)); */ -#define RESOLVE_REF_READING 0x01 -#define RESOLVE_REF_NO_RECURSE 0x02 -#define RESOLVE_REF_ALLOW_BAD_NAME 0x04 -extern const char *resolve_ref_unsafe(const char *ref, int resolve_flags, unsigned char *sha1, int *flags); -extern char *resolve_refdup(const char *ref, int resolve_flags, unsigned char *sha1, int *flags); - -extern int dwim_ref(const char *str, int len, unsigned char *sha1, char **ref); -extern int dwim_log(const char *str, int len, unsigned char *sha1, char **ref); +extern char *sha1_to_hex_r(char *out, const unsigned char *sha1); +extern char *sha1_to_hex(const unsigned char *sha1); /* static buffer result! */ +extern char *oid_to_hex(const struct object_id *oid); /* same static buffer as sha1_to_hex */ + extern int interpret_branch_name(const char *str, int len, struct strbuf *); extern int get_sha1_mb(const char *str, unsigned char *sha1); -/* - * Return true iff abbrev_name is a possible abbreviation for - * full_name according to the rules defined by ref_rev_parse_rules in - * refs.c. - */ -extern int refname_match(const char *abbrev_name, const char *full_name); - -extern int create_symref(const char *ref, const char *refs_heads_master, const char *logmsg); extern int validate_headref(const char *ref); extern int base_name_compare(const char *name1, int len1, int mode1, const char *name2, int len2, int mode2); @@@ -1170,30 -1055,18 +1170,30 @@@ extern void *read_object_with_reference extern struct object *peel_to_type(const char *name, int namelen, struct object *o, enum object_type); -enum date_mode { - DATE_NORMAL = 0, - DATE_RELATIVE, - DATE_SHORT, - DATE_LOCAL, - DATE_ISO8601, - DATE_ISO8601_STRICT, - DATE_RFC2822, - DATE_RAW +struct date_mode { + enum date_mode_type { + DATE_NORMAL = 0, + DATE_RELATIVE, + DATE_SHORT, + DATE_ISO8601, + DATE_ISO8601_STRICT, + DATE_RFC2822, + DATE_STRFTIME, + DATE_RAW + } type; + const char *strftime_fmt; + int local; }; -const char *show_date(unsigned long time, int timezone, enum date_mode mode); +/* + * Convenience helper for passing a constant type, like: + * + * show_date(t, tz, DATE_MODE(NORMAL)); + */ +#define DATE_MODE(t) date_mode_from_type(DATE_##t) +struct date_mode *date_mode_from_type(enum date_mode_type type); + +const char *show_date(unsigned long time, int timezone, const struct date_mode *mode); void show_date_relative(unsigned long time, int tz, const struct timeval *now, struct strbuf *timebuf); int parse_date(const char *date, struct strbuf *out); @@@ -1203,7 -1076,7 +1203,7 @@@ void datestamp(struct strbuf *out) #define approxidate(s) approxidate_careful((s), NULL) unsigned long approxidate_careful(const char *, int *); unsigned long approxidate_relative(const char *date, const struct timeval *now); -enum date_mode parse_date_format(const char *format); +void parse_date_format(const char *format, struct date_mode *mode); int date_overflows(unsigned long date); #define IDENT_STRICT 1 @@@ -1240,8 -1113,7 +1240,8 @@@ extern int split_ident_line(struct iden * the ident_split. It will also sanity-check the values and produce * a well-known sentinel date if they appear bogus. */ -const char *show_ident_date(const struct ident_split *id, enum date_mode mode); +const char *show_ident_date(const struct ident_split *id, + const struct date_mode *mode); /* * Compare split idents for equality or strict ordering. Note that we @@@ -1321,7 -1193,6 +1321,7 @@@ extern struct packed_git freshened:1, do_not_close:1; unsigned char sha1[20]; + struct revindex_entry *revindex; /* something like ".git/objects/pack/xxxxx.pack" */ char pack_name[FLEX_ARRAY]; /* more */ } *packed_git; @@@ -1334,11 -1205,8 +1334,11 @@@ struct pack_entry extern struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path); -/* A hook for count-objects to report invalid files in pack directory */ -extern void (*report_garbage)(const char *desc, const char *path); +/* A hook to report invalid files in pack directory */ +#define PACKDIR_FILE_PACK 1 +#define PACKDIR_FILE_IDX 2 +#define PACKDIR_FILE_GARBAGE 4 +extern void (*report_garbage)(unsigned seen_bits, const char *path); extern void prepare_packed_git(void); extern void reprepare_packed_git(void); @@@ -1363,12 -1231,21 +1363,22 @@@ extern void close_pack_index(struct pac extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); extern void close_pack_windows(struct packed_git *); +extern void close_all_packs(void); extern void unuse_pack(struct pack_window **); extern void free_pack_by_name(const char *); extern void clear_delta_base_cache(void); -extern struct packed_git *add_packed_git(const char *, int, int); +extern struct packed_git *add_packed_git(const char *path, size_t path_len, int local); + /* + * Make sure that a pointer access into an mmap'd index file is within bounds, + * and can provide at least 8 bytes of data. + * + * Note that this is only necessary for variable-length segments of the file + * (like the 64-bit extended offset table), as we compare the size to the + * fixed-length parts when we open the file. + */ + extern void check_pack_index_ptr(const struct packed_git *p, const void *ptr); + /* * Return the SHA-1 of the nth object within the specified packfile. * Open the index if it is not already open. The return value points @@@ -1456,7 -1333,6 +1466,7 @@@ struct object_info unsigned long *sizep; unsigned long *disk_sizep; unsigned char *delta_base_sha1; + struct strbuf *typename; /* Response */ enum { @@@ -1489,7 -1365,7 +1499,7 @@@ extern int update_server_info(int) /* git_config_parse_key() returns these negated: */ #define CONFIG_INVALID_KEY 1 #define CONFIG_NO_SECTION_OR_NAME 2 -/* git_config_set(), git_config_set_multivar() return the above or these: */ +/* git_config_set_gently(), git_config_set_multivar_gently() return the above or these: */ #define CONFIG_NO_LOCK -1 #define CONFIG_INVALID_FILE 3 #define CONFIG_NO_WRITE 4 @@@ -1508,8 -1384,8 +1518,8 @@@ struct git_config_source typedef int (*config_fn_t)(const char *, const char *, void *); extern int git_default_config(const char *, const char *, void *); extern int git_config_from_file(config_fn_t fn, const char *, void *); -extern int git_config_from_buf(config_fn_t fn, const char *name, - const char *buf, size_t len, void *data); +extern int git_config_from_mem(config_fn_t fn, const char *origin_type, + const char *name, const char *buf, size_t len, void *data); extern void git_config_push_parameter(const char *text); extern int git_config_from_parameters(config_fn_t fn, void *data); extern void git_config(config_fn_t fn, void *); @@@ -1518,7 -1394,6 +1528,7 @@@ extern int git_config_with_options(conf int respect_includes); extern int git_config_early(config_fn_t fn, void *, const char *repo_config); extern int git_parse_ulong(const char *, unsigned long *); +extern int git_parse_maybe_bool(const char *); extern int git_config_int(const char *, const char *); extern int64_t git_config_int64(const char *, const char *); extern unsigned long git_config_ulong(const char *, const char *); @@@ -1527,16 -1402,11 +1537,16 @@@ extern int git_config_bool(const char * extern int git_config_maybe_bool(const char *, const char *); extern int git_config_string(const char **, const char *, const char *); extern int git_config_pathname(const char **, const char *, const char *); -extern int git_config_set_in_file(const char *, const char *, const char *); -extern int git_config_set(const char *, const char *); +extern int git_config_set_in_file_gently(const char *, const char *, const char *); +extern void git_config_set_in_file(const char *, const char *, const char *); +extern int git_config_set_gently(const char *, const char *); +extern void git_config_set(const char *, const char *); extern int git_config_parse_key(const char *, char **, int *); -extern int git_config_set_multivar(const char *, const char *, const char *, int); -extern int git_config_set_multivar_in_file(const char *, const char *, const char *, const char *, int); +extern int git_config_key_is_valid(const char *key); +extern int git_config_set_multivar_gently(const char *, const char *, const char *, int); +extern void git_config_set_multivar(const char *, const char *, const char *, int); +extern int git_config_set_multivar_in_file_gently(const char *, const char *, const char *, const char *, int); +extern void git_config_set_multivar_in_file(const char *, const char *, const char *, const char *, int); extern int git_config_rename_section(const char *, const char *); extern int git_config_rename_section_in_file(const char *, const char *, const char *); extern const char *git_etc_gitconfig(void); @@@ -1552,8 -1422,6 +1562,8 @@@ extern const char *get_log_output_encod extern const char *get_commit_output_encoding(void); extern int git_config_parse_parameter(const char *, config_fn_t fn, void *data); +extern const char *current_config_origin_type(void); +extern const char *current_config_name(void); struct config_include_data { int depth; @@@ -1632,14 -1500,6 +1642,14 @@@ extern int git_config_get_bool(const ch extern int git_config_get_bool_or_int(const char *key, int *is_bool, int *dest); extern int git_config_get_maybe_bool(const char *key, int *dest); extern int git_config_get_pathname(const char *key, const char **dest); +extern int git_config_get_untracked_cache(void); + +/* + * This is a hack for test programs like test-dump-untracked-cache to + * ensure that they do not modify the untracked cache when reading it. + * Do not use it otherwise! + */ +extern int ignore_untracked_cache_config; struct key_value_info { const char *filename; @@@ -1661,13 -1521,9 +1671,13 @@@ extern const char *git_mailmap_blob extern void maybe_flush_or_die(FILE *, const char *); __attribute__((format (printf, 2, 3))) extern void fprintf_or_die(FILE *, const char *fmt, ...); + +#define COPY_READ_ERROR (-2) +#define COPY_WRITE_ERROR (-3) extern int copy_fd(int ifd, int ofd); extern int copy_file(const char *dst, const char *src, int mode); extern int copy_file_with_time(const char *dst, const char *src, int mode); + extern void write_or_die(int fd, const void *buf, size_t count); extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg); extern int write_or_whine_pipe(int fd, const void *buf, size_t count, const char *msg); @@@ -1682,9 -1538,6 +1692,9 @@@ static inline ssize_t write_str_in_full return write_in_full(fd, str, strlen(str)); } +extern int write_file(const char *path, const char *fmt, ...); +extern int write_file_gently(const char *path, const char *fmt, ...); + /* pager.c */ extern void setup_pager(void); extern const char *pager_program; @@@ -1693,7 -1546,6 +1703,7 @@@ extern int pager_use_color extern int term_columns(void); extern int decimal_width(uintmax_t); extern int check_pager_config(const char *cmd); +extern void prepare_pager_args(struct child_process *, const char *pager); extern const char *editor_program; extern const char *askpass_program; @@@ -1808,14 -1660,5 +1818,14 @@@ int stat_validity_check(struct stat_val void stat_validity_update(struct stat_validity *sv, int fd); int versioncmp(const char *s1, const char *s2); +void sleep_millisec(int millisec); + +/* + * Create a directory and (if share is nonzero) adjust its permissions + * according to the shared_repository setting. Only use this for + * directories under $GIT_DIR. Don't use it for working tree + * directories. + */ +void safe_create_dir(const char *dir, int share); #endif /* CACHE_H */ diff --combined sha1_file.c index 02517009c1,b8da68baa6..d0f2aa029b --- a/sha1_file.c +++ b/sha1_file.c @@@ -36,7 -36,6 +36,7 @@@ static inline uintmax_t sz_fmt(size_t s) { return s; } const unsigned char null_sha1[20]; +const struct object_id null_oid; /* * This is meant to hold a *small* number of objects that you would @@@ -209,25 -208,44 +209,25 @@@ const char *sha1_file_name(const unsign * provided by the caller. which should be "pack" or "idx". */ static char *sha1_get_pack_name(const unsigned char *sha1, - char **name, char **base, const char *which) + struct strbuf *buf, + const char *which) { - static const char hex[] = "0123456789abcdef"; - char *buf; - int i; - - if (!*base) { - const char *sha1_file_directory = get_object_directory(); - int len = strlen(sha1_file_directory); - *base = xmalloc(len + 60); - sprintf(*base, "%s/pack/pack-1234567890123456789012345678901234567890.%s", - sha1_file_directory, which); - *name = *base + len + 11; - } - - buf = *name; - - for (i = 0; i < 20; i++) { - unsigned int val = *sha1++; - *buf++ = hex[val >> 4]; - *buf++ = hex[val & 0xf]; - } - - return *base; + strbuf_reset(buf); + strbuf_addf(buf, "%s/pack/pack-%s.%s", get_object_directory(), + sha1_to_hex(sha1), which); + return buf->buf; } char *sha1_pack_name(const unsigned char *sha1) { - static char *name, *base; - - return sha1_get_pack_name(sha1, &name, &base, "pack"); + static struct strbuf buf = STRBUF_INIT; + return sha1_get_pack_name(sha1, &buf, "pack"); } char *sha1_pack_index_name(const unsigned char *sha1) { - static char *name, *base; - - return sha1_get_pack_name(sha1, &name, &base, "idx"); + static struct strbuf buf = STRBUF_INIT; + return sha1_get_pack_name(sha1, &buf, "idx"); } struct alternate_object_database *alt_odb_list; @@@ -253,7 -271,7 +253,7 @@@ static int link_alt_odb_entry(const cha { struct alternate_object_database *ent; struct alternate_object_database *alt; - int pfxlen, entlen; + size_t pfxlen, entlen; struct strbuf pathbuf = STRBUF_INIT; if (!is_absolute_path(entry) && relative_base) { @@@ -273,8 -291,8 +273,8 @@@ while (pfxlen && pathbuf.buf[pfxlen-1] == '/') pfxlen -= 1; - entlen = pfxlen + 43; /* '/' + 2 hex + '/' + 38 hex + NUL */ - ent = xmalloc(sizeof(*ent) + entlen); + entlen = st_add(pfxlen, 43); /* '/' + 2 hex + '/' + 38 hex + NUL */ + ent = xmalloc(st_add(sizeof(*ent), entlen)); memcpy(ent->base, pathbuf.buf, pfxlen); strbuf_release(&pathbuf); @@@ -383,46 -401,13 +383,46 @@@ void read_info_alternates(const char * void add_to_alternates_file(const char *reference) { struct lock_file *lock = xcalloc(1, sizeof(struct lock_file)); - int fd = hold_lock_file_for_append(lock, git_path("objects/info/alternates"), LOCK_DIE_ON_ERROR); - char *alt = mkpath("%s\n", reference); - write_or_die(fd, alt, strlen(alt)); - if (commit_lock_file(lock)) - die("could not close alternates file"); - if (alt_odb_tail) - link_alt_odb_entries(alt, strlen(alt), '\n', NULL, 0); + char *alts = git_pathdup("objects/info/alternates"); + FILE *in, *out; + + hold_lock_file_for_update(lock, alts, LOCK_DIE_ON_ERROR); + out = fdopen_lock_file(lock, "w"); + if (!out) + die_errno("unable to fdopen alternates lockfile"); + + in = fopen(alts, "r"); + if (in) { + struct strbuf line = STRBUF_INIT; + int found = 0; + + while (strbuf_getline(&line, in) != EOF) { + if (!strcmp(reference, line.buf)) { + found = 1; + break; + } + fprintf_or_die(out, "%s\n", line.buf); + } + + strbuf_release(&line); + fclose(in); + + if (found) { + rollback_lock_file(lock); + lock = NULL; + } + } + else if (errno != ENOENT) + die_errno("unable to read alternates file"); + + if (lock) { + fprintf_or_die(out, "%s\n", reference); + if (commit_lock_file(lock)) + die_errno("unable to move new alternates file into place"); + if (alt_odb_tail) + link_alt_odb_entries(reference, strlen(reference), '\n', NULL, 0); + } + free(alts); } int foreach_alt_odb(alt_odb_fn fn, void *cb) @@@ -653,15 -638,13 +653,15 @@@ static int check_packed_git_idx(const c int open_pack_index(struct packed_git *p) { char *idx_name; + size_t len; int ret; if (p->index_data) return 0; - idx_name = xstrdup(p->pack_name); - strcpy(idx_name + strlen(idx_name) - strlen(".pack"), ".idx"); + if (!strip_suffix(p->pack_name, ".pack", &len)) + die("BUG: pack_name does not end in .pack"); + idx_name = xstrfmt("%.*s.idx", (int)len, p->pack_name); ret = check_packed_git_idx(idx_name, p); free(idx_name); return ret; @@@ -770,37 -753,6 +770,37 @@@ void close_pack_windows(struct packed_g } } +static int close_pack_fd(struct packed_git *p) +{ + if (p->pack_fd < 0) + return 0; + + close(p->pack_fd); + pack_open_fds--; + p->pack_fd = -1; + + return 1; +} + +static void close_pack(struct packed_git *p) +{ + close_pack_windows(p); + close_pack_fd(p); + close_pack_index(p); +} + +void close_all_packs(void) +{ + struct packed_git *p; + + for (p = packed_git; p; p = p->next) + if (p->do_not_close) + die("BUG! Want to close pack marked 'do-not-close'"); + else + close_pack(p); +} + + /* * The LRU pack is the one with the oldest MRU window, preferring packs * with no used windows, or the oldest mtime if it has no windows allocated. @@@ -868,8 -820,12 +868,8 @@@ static int close_one_pack(void find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse); } - if (lru_p) { - close(lru_p->pack_fd); - pack_open_fds--; - lru_p->pack_fd = -1; - return 1; - } + if (lru_p) + return close_pack_fd(lru_p); return 0; } @@@ -909,7 -865,12 +909,7 @@@ void free_pack_by_name(const char *pack p = *pp; if (strcmp(pack_name, p->pack_name) == 0) { clear_delta_base_cache(); - close_pack_windows(p); - if (p->pack_fd != -1) { - close(p->pack_fd); - pack_open_fds--; - } - close_pack_index(p); + close_pack(p); free(p->bad_object_sha1); *pp = p->next; if (last_found_pack == p) @@@ -1043,7 -1004,11 +1043,7 @@@ static int open_packed_git(struct packe { if (!open_packed_git_1(p)) return 0; - if (p->pack_fd != -1) { - close(p->pack_fd); - pack_open_fds--; - p->pack_fd = -1; - } + close_pack_fd(p); return -1; } @@@ -1076,6 -1041,8 +1076,8 @@@ unsigned char *use_pack(struct packed_g die("packfile %s cannot be accessed", p->pack_name); if (offset > (p->pack_size - 20)) die("offset beyond end of packfile (truncated pack?)"); + if (offset < 0) + die(_("offset before end of packfile (broken .idx?)")); if (!win || !in_window(win, offset)) { if (win) @@@ -1109,8 -1076,11 +1111,8 @@@ p->pack_name, strerror(errno)); if (!win->offset && win->len == p->pack_size - && !p->do_not_close) { - close(p->pack_fd); - pack_open_fds--; - p->pack_fd = -1; - } + && !p->do_not_close) + close_pack_fd(p); pack_mmap_calls++; pack_open_windows++; if (pack_mapped > peak_pack_mapped) @@@ -1134,7 -1104,7 +1136,7 @@@ static struct packed_git *alloc_packed_git(int extra) { - struct packed_git *p = xmalloc(sizeof(*p) + extra); + struct packed_git *p = xmalloc(st_add(sizeof(*p), extra)); memset(p, 0, sizeof(*p)); p->pack_fd = -1; return p; @@@ -1145,12 -1115,11 +1147,12 @@@ static void try_to_free_pack_memory(siz release_pack_memory(size); } -struct packed_git *add_packed_git(const char *path, int path_len, int local) +struct packed_git *add_packed_git(const char *path, size_t path_len, int local) { static int have_set_try_to_free_routine; struct stat st; - struct packed_git *p = alloc_packed_git(path_len + 2); + size_t alloc; + struct packed_git *p; if (!have_set_try_to_free_routine) { have_set_try_to_free_routine = 1; @@@ -1161,22 -1130,18 +1163,22 @@@ * Make sure a corresponding .pack file exists and that * the index looks sane. */ - path_len -= strlen(".idx"); - if (path_len < 1) { - free(p); + if (!strip_suffix_mem(path, &path_len, ".idx")) return NULL; - } + + /* + * ".pack" is long enough to hold any suffix we're adding (and + * the use xsnprintf double-checks that) + */ + alloc = st_add3(path_len, strlen(".pack"), 1); + p = alloc_packed_git(alloc); memcpy(p->pack_name, path, path_len); - strcpy(p->pack_name + path_len, ".keep"); + xsnprintf(p->pack_name + path_len, alloc - path_len, ".keep"); if (!access(p->pack_name, F_OK)) p->pack_keep = 1; - strcpy(p->pack_name + path_len, ".pack"); + xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack"); if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) { free(p); return NULL; @@@ -1196,10 -1161,9 +1198,10 @@@ struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path) { const char *path = sha1_pack_name(sha1); - struct packed_git *p = alloc_packed_git(strlen(path) + 1); + size_t alloc = st_add(strlen(path), 1); + struct packed_git *p = alloc_packed_git(alloc); - strcpy(p->pack_name, path); + memcpy(p->pack_name, path, alloc); /* includes NUL */ hashcpy(p->sha1, sha1); if (check_packed_git_idx(idx_path, p)) { free(p); @@@ -1218,16 -1182,27 +1220,16 @@@ void install_packed_git(struct packed_g packed_git = pack; } -void (*report_garbage)(const char *desc, const char *path); +void (*report_garbage)(unsigned seen_bits, const char *path); static void report_helper(const struct string_list *list, int seen_bits, int first, int last) { - const char *msg; - switch (seen_bits) { - case 0: - msg = "no corresponding .idx or .pack"; - break; - case 1: - msg = "no corresponding .idx"; - break; - case 2: - msg = "no corresponding .pack"; - break; - default: + if (seen_bits == (PACKDIR_FILE_PACK|PACKDIR_FILE_IDX)) return; - } + for (; first < last; first++) - report_garbage(msg, list->items[first].string); + report_garbage(seen_bits, list->items[first].string); } static void report_pack_garbage(struct string_list *list) @@@ -1250,7 -1225,7 +1252,7 @@@ if (baselen == -1) { const char *dot = strrchr(path, '.'); if (!dot) { - report_garbage("garbage found", path); + report_garbage(PACKDIR_FILE_GARBAGE, path); continue; } baselen = dot - path + 1; @@@ -1322,7 -1297,7 +1324,7 @@@ static void prepare_packed_git_one(cha ends_with(de->d_name, ".keep")) string_list_append(&garbage, path.buf); else - report_garbage("garbage found", path.buf); + report_garbage(PACKDIR_FILE_GARBAGE, path.buf); } closedir(dir); report_pack_garbage(&garbage); @@@ -1413,12 -1388,10 +1415,12 @@@ static void mark_bad_packed_object(stru { unsigned i; for (i = 0; i < p->num_bad_objects; i++) - if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i)) + if (!hashcmp(sha1, p->bad_object_sha1 + GIT_SHA1_RAWSZ * i)) return; - p->bad_object_sha1 = xrealloc(p->bad_object_sha1, 20 * (p->num_bad_objects + 1)); - hashcpy(p->bad_object_sha1 + 20 * p->num_bad_objects, sha1); + p->bad_object_sha1 = xrealloc(p->bad_object_sha1, + st_mult(GIT_SHA1_RAWSZ, + st_add(p->num_bad_objects, 1))); + hashcpy(p->bad_object_sha1 + GIT_SHA1_RAWSZ * p->num_bad_objects, sha1); p->num_bad_objects++; } @@@ -1460,7 -1433,7 +1462,7 @@@ int check_sha1_signature(const unsigne return -1; /* Generate the header */ - hdrlen = sprintf(hdr, "%s %lu", typename(obj_type), size) + 1; + hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(obj_type), size) + 1; /* Sha1.. */ git_SHA1_Init(&c); @@@ -1487,10 -1460,7 +1489,10 @@@ int git_open_noatime(const char *name static int sha1_file_open_flag = O_NOATIME; for (;;) { - int fd = open(name, O_RDONLY | sha1_file_open_flag); + int fd; + + errno = 0; + fd = open(name, O_RDONLY | sha1_file_open_flag); if (fd >= 0) return fd; @@@ -1608,40 -1578,6 +1610,40 @@@ int unpack_sha1_header(git_zstream *str return git_inflate(stream, 0); } +static int unpack_sha1_header_to_strbuf(git_zstream *stream, unsigned char *map, + unsigned long mapsize, void *buffer, + unsigned long bufsiz, struct strbuf *header) +{ + int status; + + status = unpack_sha1_header(stream, map, mapsize, buffer, bufsiz); + + /* + * Check if entire header is unpacked in the first iteration. + */ + if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer)) + return 0; + + /* + * buffer[0..bufsiz] was not large enough. Copy the partial + * result out to header, and then append the result of further + * reading the stream. + */ + strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer); + stream->next_out = buffer; + stream->avail_out = bufsiz; + + do { + status = git_inflate(stream, 0); + strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer); + if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer)) + return 0; + stream->next_out = buffer; + stream->avail_out = bufsiz; + } while (status != Z_STREAM_END); + return -1; +} + static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1) { int bytes = strlen(buffer) + 1; @@@ -1692,38 -1628,27 +1694,38 @@@ * too permissive for what we want to check. So do an anal * object header parse by hand. */ -int parse_sha1_header(const char *hdr, unsigned long *sizep) +static int parse_sha1_header_extended(const char *hdr, struct object_info *oi, + unsigned int flags) { - char type[10]; - int i; + const char *type_buf = hdr; unsigned long size; + int type, type_len = 0; /* - * The type can be at most ten bytes (including the - * terminating '\0' that we add), and is followed by + * The type can be of any size but is followed by * a space. */ - i = 0; for (;;) { char c = *hdr++; if (c == ' ') break; - type[i++] = c; - if (i >= sizeof(type)) - return -1; + type_len++; } - type[i] = 0; + + type = type_from_string_gently(type_buf, type_len, 1); + if (oi->typename) + strbuf_add(oi->typename, type_buf, type_len); + /* + * Set type to 0 if its an unknown object and + * we're obtaining the type using '--allow-unkown-type' + * option. + */ + if ((flags & LOOKUP_UNKNOWN_OBJECT) && (type < 0)) + type = 0; + else if (type < 0) + die("invalid object type"); + if (oi->typep) + *oi->typep = type; /* * The length must follow immediately, and be in canonical @@@ -1741,24 -1666,12 +1743,24 @@@ size = size * 10 + c; } } - *sizep = size; + + if (oi->sizep) + *oi->sizep = size; /* * The length must be followed by a zero byte */ - return *hdr ? -1 : type_from_string(type); + return *hdr ? -1 : type; +} + +int parse_sha1_header(const char *hdr, unsigned long *sizep) +{ + struct object_info oi; + + oi.sizep = sizep; + oi.typename = NULL; + oi.typep = NULL; + return parse_sha1_header_extended(hdr, &oi, LOOKUP_REPLACE_OBJECT); } static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1) @@@ -1944,7 -1857,7 +1946,7 @@@ static enum object_type packed_to_objec /* Push the object we're going to leave behind */ if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) { poi_stack_alloc = alloc_nr(poi_stack_nr); - poi_stack = xmalloc(sizeof(off_t)*poi_stack_alloc); + ALLOC_ARRAY(poi_stack, poi_stack_alloc); memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr); } else { ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc); @@@ -2118,7 -2031,7 +2120,7 @@@ static unsigned long pack_entry_hash(st { unsigned long hash; - hash = (unsigned long)p + (unsigned long)base_offset; + hash = (unsigned long)(intptr_t)p + (unsigned long)base_offset; hash += (hash >> 8) + (hash >> 16); return hash % MAX_DELTA_CACHE; } @@@ -2310,7 -2223,7 +2312,7 @@@ void *unpack_entry(struct packed_git *p if (delta_stack_nr >= delta_stack_alloc && delta_stack == small_delta_stack) { delta_stack_alloc = alloc_nr(delta_stack_nr); - delta_stack = xmalloc(sizeof(*delta_stack)*delta_stack_alloc); + ALLOC_ARRAY(delta_stack, delta_stack_alloc); memcpy(delta_stack, small_delta_stack, sizeof(*delta_stack)*delta_stack_nr); } else { @@@ -2448,6 -2361,20 +2450,20 @@@ const unsigned char *nth_packed_object_ } } + void check_pack_index_ptr(const struct packed_git *p, const void *vptr) + { + const unsigned char *ptr = vptr; + const unsigned char *start = p->index_data; + const unsigned char *end = start + p->index_size; + if (ptr < start) + die(_("offset before start of pack index for %s (corrupt index?)"), + p->pack_name); + /* No need to check for underflow; .idx files must be at least 8 bytes */ + if (ptr >= end - 8) + die(_("offset beyond end of pack index for %s (truncated index?)"), + p->pack_name); + } + off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n) { const unsigned char *index = p->index_data; @@@ -2461,6 -2388,7 +2477,7 @@@ if (!(off & 0x80000000)) return off; index += p->num_objects * 4 + (off & 0x7fffffff) * 8; + check_pack_index_ptr(p, index); return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) | ntohl(*((uint32_t *)(index + 4))); } @@@ -2623,15 -2551,13 +2640,15 @@@ struct packed_git *find_sha1_pack(cons } static int sha1_loose_object_info(const unsigned char *sha1, - struct object_info *oi) + struct object_info *oi, + int flags) { - int status; - unsigned long mapsize, size; + int status = 0; + unsigned long mapsize; void *map; git_zstream stream; char hdr[32]; + struct strbuf hdrbuf = STRBUF_INIT; if (oi->delta_base_sha1) hashclr(oi->delta_base_sha1); @@@ -2644,7 -2570,7 +2661,7 @@@ * return value implicitly indicates whether the * object even exists. */ - if (!oi->typep && !oi->sizep) { + if (!oi->typep && !oi->typename && !oi->sizep) { struct stat st; if (stat_sha1_file(sha1, &st) < 0) return -1; @@@ -2658,26 -2584,17 +2675,26 @@@ return -1; if (oi->disk_sizep) *oi->disk_sizep = mapsize; - if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) + if ((flags & LOOKUP_UNKNOWN_OBJECT)) { + if (unpack_sha1_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0) + status = error("unable to unpack %s header with --allow-unknown-type", + sha1_to_hex(sha1)); + } else if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) status = error("unable to unpack %s header", sha1_to_hex(sha1)); - else if ((status = parse_sha1_header(hdr, &size)) < 0) + if (status < 0) + ; /* Do nothing */ + else if (hdrbuf.len) { + if ((status = parse_sha1_header_extended(hdrbuf.buf, oi, flags)) < 0) + status = error("unable to parse %s header with --allow-unknown-type", + sha1_to_hex(sha1)); + } else if ((status = parse_sha1_header_extended(hdr, oi, flags)) < 0) status = error("unable to parse %s header", sha1_to_hex(sha1)); - else if (oi->sizep) - *oi->sizep = size; git_inflate_end(&stream); munmap(map, mapsize); - if (oi->typep) + if (status && oi->typep) *oi->typep = status; + strbuf_release(&hdrbuf); return 0; } @@@ -2686,7 -2603,6 +2703,7 @@@ int sha1_object_info_extended(const uns struct cached_object *co; struct pack_entry e; int rtype; + enum object_type real_type; const unsigned char *real = lookup_replace_object_extended(sha1, flags); co = find_cached_object(real); @@@ -2699,15 -2615,13 +2716,15 @@@ *(oi->disk_sizep) = 0; if (oi->delta_base_sha1) hashclr(oi->delta_base_sha1); + if (oi->typename) + strbuf_addstr(oi->typename, typename(co->type)); oi->whence = OI_CACHED; return 0; } if (!find_pack_entry(real, &e)) { /* Most likely it's a loose object. */ - if (!sha1_loose_object_info(real, oi)) { + if (!sha1_loose_object_info(real, oi, flags)) { oi->whence = OI_LOOSE; return 0; } @@@ -2718,18 -2632,9 +2735,18 @@@ return -1; } + /* + * packed_object_info() does not follow the delta chain to + * find out the real type, unless it is given oi->typep. + */ + if (oi->typename && !oi->typep) + oi->typep = &real_type; + rtype = packed_object_info(e.p, e.offset, oi); if (rtype < 0) { mark_bad_packed_object(e.p, real); + if (oi->typep == &real_type) + oi->typep = NULL; return sha1_object_info_extended(real, oi, 0); } else if (in_delta_base_cache(e.p, e.offset)) { oi->whence = OI_DBCACHED; @@@ -2740,10 -2645,6 +2757,10 @@@ oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA || rtype == OBJ_OFS_DELTA); } + if (oi->typename) + strbuf_addstr(oi->typename, typename(*oi->typep)); + if (oi->typep == &real_type) + oi->typep = NULL; return 0; } @@@ -2926,7 -2827,7 +2943,7 @@@ static void write_sha1_file_prepare(con git_SHA_CTX c; /* Generate the header */ - *hdrlen = sprintf(hdr, "%s %lu", type, len)+1; + *hdrlen = xsnprintf(hdr, *hdrlen, "%s %lu", type, len)+1; /* Sha1.. */ git_SHA1_Init(&c); @@@ -2937,8 -2838,11 +2954,8 @@@ /* * Move the just written object into its final resting place. - * NEEDSWORK: this should be renamed to finalize_temp_file() as - * "moving" is only a part of what it does, when no patch between - * master to pu changes the call sites of this function. */ -int move_temp_to_file(const char *tmpfile, const char *filename) +int finalize_object_file(const char *tmpfile, const char *filename) { int ret = 0; @@@ -2989,7 -2893,7 +3006,7 @@@ int hash_sha1_file(const void *buf, uns unsigned char *sha1) { char hdr[32]; - int hdrlen; + int hdrlen = sizeof(hdr); write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen); return 0; } @@@ -3019,31 -2923,29 +3036,31 @@@ static inline int directory_size(const * We want to avoid cross-directory filename renames, because those * can have problems on various filesystems (FAT, NFS, Coda). */ -static int create_tmpfile(char *buffer, size_t bufsiz, const char *filename) +static int create_tmpfile(struct strbuf *tmp, const char *filename) { int fd, dirlen = directory_size(filename); - if (dirlen + 20 > bufsiz) { - errno = ENAMETOOLONG; - return -1; - } - memcpy(buffer, filename, dirlen); - strcpy(buffer + dirlen, "tmp_obj_XXXXXX"); - fd = git_mkstemp_mode(buffer, 0444); + strbuf_reset(tmp); + strbuf_add(tmp, filename, dirlen); + strbuf_addstr(tmp, "tmp_obj_XXXXXX"); + fd = git_mkstemp_mode(tmp->buf, 0444); if (fd < 0 && dirlen && errno == ENOENT) { - /* Make sure the directory exists */ - memcpy(buffer, filename, dirlen); - buffer[dirlen-1] = 0; - if (mkdir(buffer, 0777) && errno != EEXIST) + /* + * Make sure the directory exists; note that the contents + * of the buffer are undefined after mkstemp returns an + * error, so we have to rewrite the whole buffer from + * scratch. + */ + strbuf_reset(tmp); + strbuf_add(tmp, filename, dirlen - 1); + if (mkdir(tmp->buf, 0777) && errno != EEXIST) return -1; - if (adjust_shared_perm(buffer)) + if (adjust_shared_perm(tmp->buf)) return -1; /* Try again */ - strcpy(buffer + dirlen - 1, "/tmp_obj_XXXXXX"); - fd = git_mkstemp_mode(buffer, 0444); + strbuf_addstr(tmp, "/tmp_obj_XXXXXX"); + fd = git_mkstemp_mode(tmp->buf, 0444); } return fd; } @@@ -3056,10 -2958,10 +3073,10 @@@ static int write_loose_object(const uns git_zstream stream; git_SHA_CTX c; unsigned char parano_sha1[20]; - static char tmp_file[PATH_MAX]; + static struct strbuf tmp_file = STRBUF_INIT; const char *filename = sha1_file_name(sha1); - fd = create_tmpfile(tmp_file, sizeof(tmp_file), filename); + fd = create_tmpfile(&tmp_file, filename); if (fd < 0) { if (errno == EACCES) return error("insufficient permission for adding an object to repository database %s", get_object_directory()); @@@ -3108,12 -3010,12 +3125,12 @@@ struct utimbuf utb; utb.actime = mtime; utb.modtime = mtime; - if (utime(tmp_file, &utb) < 0) + if (utime(tmp_file.buf, &utb) < 0) warning("failed utime() on %s: %s", - tmp_file, strerror(errno)); + tmp_file.buf, strerror(errno)); } - return move_temp_to_file(tmp_file, filename); + return finalize_object_file(tmp_file.buf, filename); } static int freshen_loose_object(const unsigned char *sha1) @@@ -3137,7 -3039,7 +3154,7 @@@ static int freshen_packed_object(const int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1) { char hdr[32]; - int hdrlen; + int hdrlen = sizeof(hdr); /* Normally if we have it in the pack then we do not bother writing * it out into .git/objects/??/?{38} file. @@@ -3155,8 -3057,7 +3172,8 @@@ int hash_sha1_file_literally(const voi int hdrlen, status = 0; /* type string, SP, %lu of the length plus NUL must fit this */ - header = xmalloc(strlen(type) + 32); + hdrlen = strlen(type) + 32; + header = xmalloc(hdrlen); write_sha1_file_prepare(buf, len, type, sha1, header, &hdrlen); if (!(flags & HASH_WRITE_OBJECT)) @@@ -3184,7 -3085,7 +3201,7 @@@ int force_object_loose(const unsigned c buf = read_packed_sha1(sha1, &type, &len); if (!buf) return error("cannot read sha1_file for %s", sha1_to_hex(sha1)); - hdrlen = sprintf(hdr, "%s %lu", typename(type), len) + 1; + hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(type), len) + 1; ret = write_loose_object(sha1, hdr, hdrlen, buf, len, mtime); free(buf); @@@ -3219,11 -3120,6 +3236,11 @@@ int has_sha1_file_with_flags(const unsi return find_pack_entry(sha1, &e); } +int has_object_file(const struct object_id *oid) +{ + return has_sha1_file(oid->hash); +} + static void check_tree(const void *buf, size_t size) { struct tree_desc desc; @@@ -3507,12 -3403,12 +3524,12 @@@ static int for_each_file_in_obj_subdir( break; } } - strbuf_setlen(path, baselen); + closedir(dir); + strbuf_setlen(path, baselen); if (!r && subdir_cb) r = subdir_cb(subdir_nr, path->buf, data); - closedir(dir); return r; } @@@ -3617,19 -3513,14 +3634,19 @@@ int for_each_packed_object(each_packed_ { struct packed_git *p; int r = 0; + int pack_errors = 0; prepare_packed_git(); for (p = packed_git; p; p = p->next) { if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local) continue; + if (open_pack_index(p)) { + pack_errors = 1; + continue; + } r = for_each_object_in_pack(p, cb, data); if (r) break; } - return r; + return r ? r : pack_errors; }