From: Junio C Hamano Date: Thu, 10 Mar 2016 19:13:46 +0000 (-0800) Subject: Merge branch 'jk/pack-idx-corruption-safety' into maint X-Git-Tag: v2.7.3~6 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/80047fa084f994969d42338bc3fbbb89e7d957d6?ds=inline;hp=-c Merge branch 'jk/pack-idx-corruption-safety' into maint The code to read the pack data using the offsets stored in the pack idx file has been made more carefully check the validity of the data in the idx. * jk/pack-idx-corruption-safety: sha1_file.c: mark strings for translation use_pack: handle signed off_t overflow nth_packed_object_offset: bounds-check extended offset t5313: test bounds-checks of corrupted/malicious pack/idx files --- 80047fa084f994969d42338bc3fbbb89e7d957d6 diff --combined builtin/index-pack.c index 193908a619,98bdbb5e6c..45245199ae --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@@ -18,14 -18,16 +18,14 @@@ static const char index_pack_usage[] struct object_entry { struct pack_idx_entry idx; unsigned long size; - unsigned int hdr_size; - enum object_type type; - enum object_type real_type; - unsigned delta_depth; - int base_object_no; + unsigned char hdr_size; + signed char type; + signed char real_type; }; -union delta_base { - unsigned char sha1[20]; - off_t offset; +struct object_stat { + unsigned delta_depth; + int base_object_no; }; struct base_data { @@@ -47,35 -49,31 +47,35 @@@ struct thread_local int pack_fd; }; -/* - * Even if sizeof(union delta_base) == 24 on 64-bit archs, we really want - * to memcmp() only the first 20 bytes. - */ -#define UNION_BASE_SZ 20 - #define FLAG_LINK (1u<<20) #define FLAG_CHECKED (1u<<21) -struct delta_entry { - union delta_base base; +struct ofs_delta_entry { + off_t offset; + int obj_no; +}; + +struct ref_delta_entry { + unsigned char sha1[20]; int obj_no; }; static struct object_entry *objects; -static struct delta_entry *deltas; +static struct object_stat *obj_stat; +static struct ofs_delta_entry *ofs_deltas; +static struct ref_delta_entry *ref_deltas; static struct thread_local nothread_data; static int nr_objects; -static int nr_deltas; +static int nr_ofs_deltas; +static int nr_ref_deltas; +static int ref_deltas_alloc; static int nr_resolved_deltas; static int nr_threads; static int from_stdin; static int strict; static int do_fsck_object; +static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT; static int verbose; static int show_stat; static int check_self_contained_and_connected; @@@ -193,13 -191,13 +193,13 @@@ static void cleanup_thread(void #endif -static int mark_link(struct object *obj, int type, void *data) +static int mark_link(struct object *obj, int type, void *data, struct fsck_options *options) { if (!obj) return -1; if (type != OBJ_ANY && obj->type != type) - die(_("object type mismatch at %s"), sha1_to_hex(obj->sha1)); + die(_("object type mismatch at %s"), oid_to_hex(&obj->oid)); obj->flags |= FLAG_LINK; return 0; @@@ -217,13 -215,13 +217,13 @@@ static unsigned check_object(struct obj if (!(obj->flags & FLAG_CHECKED)) { unsigned long size; - int type = sha1_object_info(obj->sha1, &size); + int type = sha1_object_info(obj->oid.hash, &size); if (type <= 0) die(_("did not receive expected object %s"), - sha1_to_hex(obj->sha1)); + oid_to_hex(&obj->oid)); if (type != obj->type) die(_("object %s: expected type %s, found %s"), - sha1_to_hex(obj->sha1), + oid_to_hex(&obj->oid), typename(obj->type), typename(type)); obj->flags |= FLAG_CHECKED; return 1; @@@ -441,7 -439,7 +441,7 @@@ static void *unpack_entry_data(unsigne int hdrlen; if (!is_delta_type(type)) { - hdrlen = sprintf(hdr, "%s %lu", typename(type), size) + 1; + hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(type), size) + 1; git_SHA1_Init(&c); git_SHA1_Update(&c, hdr, hdrlen); } else @@@ -478,8 -476,7 +478,8 @@@ } static void *unpack_raw_entry(struct object_entry *obj, - union delta_base *delta_base, + off_t *ofs_offset, + unsigned char *ref_sha1, unsigned char *sha1) { unsigned char *p; @@@ -508,10 -505,11 +508,10 @@@ switch (obj->type) { case OBJ_REF_DELTA: - hashcpy(delta_base->sha1, fill(20)); + hashcpy(ref_sha1, fill(20)); use(20); break; case OBJ_OFS_DELTA: - memset(delta_base, 0, sizeof(*delta_base)); p = fill(1); c = *p; use(1); @@@ -525,8 -523,8 +525,8 @@@ use(1); base_offset = (base_offset << 7) + (c & 127); } - delta_base->offset = obj->idx.offset - base_offset; - if (delta_base->offset <= 0 || delta_base->offset >= obj->idx.offset) + *ofs_offset = obj->idx.offset - base_offset; + if (*ofs_offset <= 0 || *ofs_offset >= obj->idx.offset) bad_object(obj->idx.offset, _("delta base offset is out of bound")); break; case OBJ_COMMIT: @@@ -610,110 -608,55 +610,110 @@@ static void *get_data_from_pack(struct return unpack_data(obj, NULL, NULL); } -static int compare_delta_bases(const union delta_base *base1, - const union delta_base *base2, - enum object_type type1, - enum object_type type2) +static int compare_ofs_delta_bases(off_t offset1, off_t offset2, + enum object_type type1, + enum object_type type2) +{ + int cmp = type1 - type2; + if (cmp) + return cmp; + return offset1 < offset2 ? -1 : + offset1 > offset2 ? 1 : + 0; +} + +static int find_ofs_delta(const off_t offset, enum object_type type) +{ + int first = 0, last = nr_ofs_deltas; + + while (first < last) { + int next = (first + last) / 2; + struct ofs_delta_entry *delta = &ofs_deltas[next]; + int cmp; + + cmp = compare_ofs_delta_bases(offset, delta->offset, + type, objects[delta->obj_no].type); + if (!cmp) + return next; + if (cmp < 0) { + last = next; + continue; + } + first = next+1; + } + return -first-1; +} + +static void find_ofs_delta_children(off_t offset, + int *first_index, int *last_index, + enum object_type type) +{ + int first = find_ofs_delta(offset, type); + int last = first; + int end = nr_ofs_deltas - 1; + + if (first < 0) { + *first_index = 0; + *last_index = -1; + return; + } + while (first > 0 && ofs_deltas[first - 1].offset == offset) + --first; + while (last < end && ofs_deltas[last + 1].offset == offset) + ++last; + *first_index = first; + *last_index = last; +} + +static int compare_ref_delta_bases(const unsigned char *sha1, + const unsigned char *sha2, + enum object_type type1, + enum object_type type2) { int cmp = type1 - type2; if (cmp) return cmp; - return memcmp(base1, base2, UNION_BASE_SZ); + return hashcmp(sha1, sha2); } -static int find_delta(const union delta_base *base, enum object_type type) +static int find_ref_delta(const unsigned char *sha1, enum object_type type) { - int first = 0, last = nr_deltas; - - while (first < last) { - int next = (first + last) / 2; - struct delta_entry *delta = &deltas[next]; - int cmp; - - cmp = compare_delta_bases(base, &delta->base, - type, objects[delta->obj_no].type); - if (!cmp) - return next; - if (cmp < 0) { - last = next; - continue; - } - first = next+1; - } - return -first-1; + int first = 0, last = nr_ref_deltas; + + while (first < last) { + int next = (first + last) / 2; + struct ref_delta_entry *delta = &ref_deltas[next]; + int cmp; + + cmp = compare_ref_delta_bases(sha1, delta->sha1, + type, objects[delta->obj_no].type); + if (!cmp) + return next; + if (cmp < 0) { + last = next; + continue; + } + first = next+1; + } + return -first-1; } -static void find_delta_children(const union delta_base *base, - int *first_index, int *last_index, - enum object_type type) +static void find_ref_delta_children(const unsigned char *sha1, + int *first_index, int *last_index, + enum object_type type) { - int first = find_delta(base, type); + int first = find_ref_delta(sha1, type); int last = first; - int end = nr_deltas - 1; + int end = nr_ref_deltas - 1; if (first < 0) { *first_index = 0; *last_index = -1; return; } - while (first > 0 && !memcmp(&deltas[first - 1].base, base, UNION_BASE_SZ)) + while (first > 0 && !hashcmp(ref_deltas[first - 1].sha1, sha1)) --first; - while (last < end && !memcmp(&deltas[last + 1].base, base, UNION_BASE_SZ)) + while (last < end && !hashcmp(ref_deltas[last + 1].sha1, sha1)) ++last; *first_index = first; *last_index = last; @@@ -839,10 -782,11 +839,10 @@@ static void sha1_object(const void *dat if (!obj) die(_("invalid %s"), typename(type)); if (do_fsck_object && - fsck_object(obj, buf, size, 1, - fsck_error_function)) + fsck_object(obj, buf, size, &fsck_options)) die(_("Error in object")); - if (fsck_walk(obj, mark_link, NULL)) - die(_("Not all child objects of %s are reachable"), sha1_to_hex(obj->sha1)); + if (fsck_walk(obj, NULL, &fsck_options)) + die(_("Not all child objects of %s are reachable"), oid_to_hex(&obj->oid)); if (obj->type == OBJ_TREE) { struct tree *item = (struct tree *) obj; @@@ -929,15 -873,13 +929,15 @@@ static void resolve_delta(struct object void *base_data, *delta_data; if (show_stat) { - delta_obj->delta_depth = base->obj->delta_depth + 1; + int i = delta_obj - objects; + int j = base->obj - objects; + obj_stat[i].delta_depth = obj_stat[j].delta_depth + 1; deepest_delta_lock(); - if (deepest_delta < delta_obj->delta_depth) - deepest_delta = delta_obj->delta_depth; + if (deepest_delta < obj_stat[i].delta_depth) + deepest_delta = obj_stat[i].delta_depth; deepest_delta_unlock(); + obj_stat[i].base_object_no = j; } - delta_obj->base_object_no = base->obj - objects; delta_data = get_data_from_pack(delta_obj); base_data = get_base_data(base); result->obj = delta_obj; @@@ -960,7 -902,7 +960,7 @@@ * "want"; if so, swap in "set" and return true. Otherwise, leave it untouched * and return false. */ -static int compare_and_swap_type(enum object_type *type, +static int compare_and_swap_type(signed char *type, enum object_type want, enum object_type set) { @@@ -979,13 -921,16 +979,13 @@@ static struct base_data *find_unresolve struct base_data *prev_base) { if (base->ref_last == -1 && base->ofs_last == -1) { - union delta_base base_spec; + find_ref_delta_children(base->obj->idx.sha1, + &base->ref_first, &base->ref_last, + OBJ_REF_DELTA); - hashcpy(base_spec.sha1, base->obj->idx.sha1); - find_delta_children(&base_spec, - &base->ref_first, &base->ref_last, OBJ_REF_DELTA); - - memset(&base_spec, 0, sizeof(base_spec)); - base_spec.offset = base->obj->idx.offset; - find_delta_children(&base_spec, - &base->ofs_first, &base->ofs_last, OBJ_OFS_DELTA); + find_ofs_delta_children(base->obj->idx.offset, + &base->ofs_first, &base->ofs_last, + OBJ_OFS_DELTA); if (base->ref_last == -1 && base->ofs_last == -1) { free(base->data); @@@ -996,7 -941,7 +996,7 @@@ } if (base->ref_first <= base->ref_last) { - struct object_entry *child = objects + deltas[base->ref_first].obj_no; + struct object_entry *child = objects + ref_deltas[base->ref_first].obj_no; struct base_data *result = alloc_base_data(); if (!compare_and_swap_type(&child->real_type, OBJ_REF_DELTA, @@@ -1012,7 -957,7 +1012,7 @@@ } if (base->ofs_first <= base->ofs_last) { - struct object_entry *child = objects + deltas[base->ofs_first].obj_no; + struct object_entry *child = objects + ofs_deltas[base->ofs_first].obj_no; struct base_data *result = alloc_base_data(); assert(child->real_type == OBJ_OFS_DELTA); @@@ -1048,22 -993,15 +1048,22 @@@ static void find_unresolved_deltas(stru } } -static int compare_delta_entry(const void *a, const void *b) +static int compare_ofs_delta_entry(const void *a, const void *b) +{ + const struct ofs_delta_entry *delta_a = a; + const struct ofs_delta_entry *delta_b = b; + + return delta_a->offset < delta_b->offset ? -1 : + delta_a->offset > delta_b->offset ? 1 : + 0; +} + +static int compare_ref_delta_entry(const void *a, const void *b) { - const struct delta_entry *delta_a = a; - const struct delta_entry *delta_b = b; + const struct ref_delta_entry *delta_a = a; + const struct ref_delta_entry *delta_b = b; - /* group by type (ref vs ofs) and then by value (sha-1 or offset) */ - return compare_delta_bases(&delta_a->base, &delta_b->base, - objects[delta_a->obj_no].type, - objects[delta_b->obj_no].type); + return hashcmp(delta_a->sha1, delta_b->sha1); } static void resolve_base(struct object_entry *obj) @@@ -1109,8 -1047,7 +1109,8 @@@ static void *threaded_second_pass(void static void parse_pack_objects(unsigned char *sha1) { int i, nr_delays = 0; - struct delta_entry *delta = deltas; + struct ofs_delta_entry *ofs_delta = ofs_deltas; + unsigned char ref_delta_sha1[20]; struct stat st; if (verbose) @@@ -1119,18 -1056,12 +1119,18 @@@ nr_objects); for (i = 0; i < nr_objects; i++) { struct object_entry *obj = &objects[i]; - void *data = unpack_raw_entry(obj, &delta->base, obj->idx.sha1); + void *data = unpack_raw_entry(obj, &ofs_delta->offset, + ref_delta_sha1, obj->idx.sha1); obj->real_type = obj->type; - if (is_delta_type(obj->type)) { - nr_deltas++; - delta->obj_no = i; - delta++; + if (obj->type == OBJ_OFS_DELTA) { + nr_ofs_deltas++; + ofs_delta->obj_no = i; + ofs_delta++; + } else if (obj->type == OBJ_REF_DELTA) { + ALLOC_GROW(ref_deltas, nr_ref_deltas + 1, ref_deltas_alloc); + hashcpy(ref_deltas[nr_ref_deltas].sha1, ref_delta_sha1); + ref_deltas[nr_ref_deltas].obj_no = i; + nr_ref_deltas++; } else if (!data) { /* large blobs, check later */ obj->real_type = OBJ_BAD; @@@ -1181,18 -1112,15 +1181,18 @@@ static void resolve_deltas(void { int i; - if (!nr_deltas) + if (!nr_ofs_deltas && !nr_ref_deltas) return; /* Sort deltas by base SHA1/offset for fast searching */ - qsort(deltas, nr_deltas, sizeof(struct delta_entry), - compare_delta_entry); + qsort(ofs_deltas, nr_ofs_deltas, sizeof(struct ofs_delta_entry), + compare_ofs_delta_entry); + qsort(ref_deltas, nr_ref_deltas, sizeof(struct ref_delta_entry), + compare_ref_delta_entry); if (verbose) - progress = start_progress(_("Resolving deltas"), nr_deltas); + progress = start_progress(_("Resolving deltas"), + nr_ref_deltas + nr_ofs_deltas); #ifndef NO_PTHREADS nr_dispatched = 0; @@@ -1227,10 -1155,10 +1227,10 @@@ * - append objects to convert thin pack to full pack if required * - write the final 20-byte SHA-1 */ -static void fix_unresolved_deltas(struct sha1file *f, int nr_unresolved); +static void fix_unresolved_deltas(struct sha1file *f); static void conclude_pack(int fix_thin_pack, const char *curr_pack, unsigned char *pack_sha1) { - if (nr_deltas == nr_resolved_deltas) { + if (nr_ref_deltas + nr_ofs_deltas == nr_resolved_deltas) { stop_progress(&progress); /* Flush remaining pack final 20-byte SHA1. */ flush(); @@@ -1241,7 -1169,7 +1241,7 @@@ struct sha1file *f; unsigned char read_sha1[20], tail_sha1[20]; struct strbuf msg = STRBUF_INIT; - int nr_unresolved = nr_deltas - nr_resolved_deltas; + int nr_unresolved = nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas; int nr_objects_initial = nr_objects; if (nr_unresolved <= 0) die(_("confusion beyond insanity")); @@@ -1249,7 -1177,7 +1249,7 @@@ memset(objects + nr_objects + 1, 0, nr_unresolved * sizeof(*objects)); f = sha1fd(output_fd, curr_pack); - fix_unresolved_deltas(f, nr_unresolved); + fix_unresolved_deltas(f); strbuf_addf(&msg, _("completed with %d local objects"), nr_objects - nr_objects_initial); stop_progress_msg(&progress, msg.buf); @@@ -1263,11 -1191,11 +1263,11 @@@ die(_("Unexpected tail checksum for %s " "(disk corruption?)"), curr_pack); } - if (nr_deltas != nr_resolved_deltas) + if (nr_ofs_deltas + nr_ref_deltas != nr_resolved_deltas) die(Q_("pack has %d unresolved delta", "pack has %d unresolved deltas", - nr_deltas - nr_resolved_deltas), - nr_deltas - nr_resolved_deltas); + nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas), + nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas); } static int write_compressed(struct sha1file *f, void *in, unsigned int size) @@@ -1326,15 -1254,15 +1326,15 @@@ static struct object_entry *append_obj_ static int delta_pos_compare(const void *_a, const void *_b) { - struct delta_entry *a = *(struct delta_entry **)_a; - struct delta_entry *b = *(struct delta_entry **)_b; + struct ref_delta_entry *a = *(struct ref_delta_entry **)_a; + struct ref_delta_entry *b = *(struct ref_delta_entry **)_b; return a->obj_no - b->obj_no; } -static void fix_unresolved_deltas(struct sha1file *f, int nr_unresolved) +static void fix_unresolved_deltas(struct sha1file *f) { - struct delta_entry **sorted_by_pos; - int i, n = 0; + struct ref_delta_entry **sorted_by_pos; + int i; /* * Since many unresolved deltas may well be themselves base objects @@@ -1346,26 -1274,29 +1346,26 @@@ * before deltas depending on them, a good heuristic is to start * resolving deltas in the same order as their position in the pack. */ - sorted_by_pos = xmalloc(nr_unresolved * sizeof(*sorted_by_pos)); - for (i = 0; i < nr_deltas; i++) { - if (objects[deltas[i].obj_no].real_type != OBJ_REF_DELTA) - continue; - sorted_by_pos[n++] = &deltas[i]; - } - qsort(sorted_by_pos, n, sizeof(*sorted_by_pos), delta_pos_compare); + ALLOC_ARRAY(sorted_by_pos, nr_ref_deltas); + for (i = 0; i < nr_ref_deltas; i++) + sorted_by_pos[i] = &ref_deltas[i]; + qsort(sorted_by_pos, nr_ref_deltas, sizeof(*sorted_by_pos), delta_pos_compare); - for (i = 0; i < n; i++) { - struct delta_entry *d = sorted_by_pos[i]; + for (i = 0; i < nr_ref_deltas; i++) { + struct ref_delta_entry *d = sorted_by_pos[i]; enum object_type type; struct base_data *base_obj = alloc_base_data(); if (objects[d->obj_no].real_type != OBJ_REF_DELTA) continue; - base_obj->data = read_sha1_file(d->base.sha1, &type, &base_obj->size); + base_obj->data = read_sha1_file(d->sha1, &type, &base_obj->size); if (!base_obj->data) continue; - if (check_sha1_signature(d->base.sha1, base_obj->data, + if (check_sha1_signature(d->sha1, base_obj->data, base_obj->size, typename(type))) - die(_("local object %s is corrupt"), sha1_to_hex(d->base.sha1)); - base_obj->obj = append_obj_to_pack(f, d->base.sha1, + die(_("local object %s is corrupt"), sha1_to_hex(d->sha1)); + base_obj->obj = append_obj_to_pack(f, d->sha1, base_obj->data, base_obj->size, type); find_unresolved_deltas(base_obj); display_progress(progress, nr_resolved_deltas); @@@ -1421,7 -1352,7 +1421,7 @@@ static void final(const char *final_pac get_object_directory(), sha1_to_hex(sha1)); final_pack_name = name; } - if (move_temp_to_file(curr_pack_name, final_pack_name)) + if (finalize_object_file(curr_pack_name, final_pack_name)) die(_("cannot store pack file")); } else if (from_stdin) chmod(final_pack_name, 0444); @@@ -1432,7 -1363,7 +1432,7 @@@ get_object_directory(), sha1_to_hex(sha1)); final_index_name = name; } - if (move_temp_to_file(curr_index_name, final_index_name)) + if (finalize_object_file(curr_index_name, final_index_name)) die(_("cannot store index file")); } else chmod(final_index_name, 0444); @@@ -1514,6 -1445,7 +1514,7 @@@ static void read_v2_anomalous_offsets(s if (!(off & 0x80000000)) continue; off = off & 0x7fffffff; + check_pack_index_ptr(p, &idx2[off * 2]); if (idx2[off * 2]) continue; /* @@@ -1557,7 -1489,7 +1558,7 @@@ static void read_idx_option(struct pack static void show_pack_info(int stat_only) { - int i, baseobjects = nr_objects - nr_deltas; + int i, baseobjects = nr_objects - nr_ref_deltas - nr_ofs_deltas; unsigned long *chain_histogram = NULL; if (deepest_delta) @@@ -1567,7 -1499,7 +1568,7 @@@ struct object_entry *obj = &objects[i]; if (is_delta_type(obj->type)) - chain_histogram[obj->delta_depth - 1]++; + chain_histogram[obj_stat[i].delta_depth - 1]++; if (stat_only) continue; printf("%s %-6s %lu %lu %"PRIuMAX, @@@ -1576,8 -1508,8 +1577,8 @@@ (unsigned long)(obj[1].idx.offset - obj->idx.offset), (uintmax_t)obj->idx.offset); if (is_delta_type(obj->type)) { - struct object_entry *bobj = &objects[obj->base_object_no]; - printf(" %u %s", obj->delta_depth, sha1_to_hex(bobj->idx.sha1)); + struct object_entry *bobj = &objects[obj_stat[i].base_object_no]; + printf(" %u %s", obj_stat[i].delta_depth, sha1_to_hex(bobj->idx.sha1)); } putchar('\n'); } @@@ -1615,7 -1547,6 +1616,7 @@@ int cmd_index_pack(int argc, const cha usage(index_pack_usage); check_replace_refs = 0; + fsck_options.walk = mark_link; reset_pack_idx_option(&opts); git_config(git_index_pack_config, &opts); @@@ -1633,10 -1564,6 +1634,10 @@@ } else if (!strcmp(arg, "--strict")) { strict = 1; do_fsck_object = 1; + } else if (skip_prefix(arg, "--strict=", &arg)) { + strict = 1; + do_fsck_object = 1; + fsck_set_msg_types(&fsck_options, arg); } else if (!strcmp(arg, "--check-self-contained-and-connected")) { strict = 1; check_self_contained_and_connected = 1; @@@ -1744,22 -1671,19 +1745,22 @@@ curr_pack = open_pack_file(pack_name); parse_pack_header(); - objects = xcalloc(nr_objects + 1, sizeof(struct object_entry)); - deltas = xcalloc(nr_objects, sizeof(struct delta_entry)); + objects = xcalloc(st_add(nr_objects, 1), sizeof(struct object_entry)); + if (show_stat) + obj_stat = xcalloc(st_add(nr_objects, 1), sizeof(struct object_stat)); + ofs_deltas = xcalloc(nr_objects, sizeof(struct ofs_delta_entry)); parse_pack_objects(pack_sha1); resolve_deltas(); conclude_pack(fix_thin_pack, curr_pack, pack_sha1); - free(deltas); + free(ofs_deltas); + free(ref_deltas); if (strict) foreign_nr = check_objects(); if (show_stat) show_pack_info(stat_only); - idx_objects = xmalloc((nr_objects) * sizeof(struct pack_idx_entry *)); + ALLOC_ARRAY(idx_objects, nr_objects); for (i = 0; i < nr_objects; i++) idx_objects[i] = &objects[i].idx; curr_index = write_idx_file(index_name, idx_objects, nr_objects, &opts, pack_sha1); diff --combined cache.h index 1f145c2c6a,6c9aaa1ae6..1a2cec0b88 --- a/cache.h +++ b/cache.h @@@ -11,29 -11,11 +11,29 @@@ #include "string-list.h" #include SHA1_HEADER -#ifndef git_SHA_CTX -#define git_SHA_CTX SHA_CTX -#define git_SHA1_Init SHA1_Init -#define git_SHA1_Update SHA1_Update -#define git_SHA1_Final SHA1_Final +#ifndef platform_SHA_CTX +/* + * platform's underlying implementation of SHA-1; could be OpenSSL, + * blk_SHA, Apple CommonCrypto, etc... Note that including + * SHA1_HEADER may have already defined platform_SHA_CTX for our + * own implementations like block-sha1 and ppc-sha1, so we list + * the default for OpenSSL compatible SHA-1 implementations here. + */ +#define platform_SHA_CTX SHA_CTX +#define platform_SHA1_Init SHA1_Init +#define platform_SHA1_Update SHA1_Update +#define platform_SHA1_Final SHA1_Final +#endif + +#define git_SHA_CTX platform_SHA_CTX +#define git_SHA1_Init platform_SHA1_Init +#define git_SHA1_Update platform_SHA1_Update +#define git_SHA1_Final platform_SHA1_Final + +#ifdef SHA1_MAX_BLOCK_SIZE +#include "compat/sha1-chunked.h" +#undef git_SHA1_Update +#define git_SHA1_Update git_SHA1_Update_Chunked #endif #include @@@ -61,14 -43,6 +61,14 @@@ int git_deflate_end_gently(git_zstream int git_deflate(git_zstream *, int flush); unsigned long git_deflate_bound(git_zstream *, unsigned long); +/* The length in bytes and in hex digits of an object name (SHA-1 value). */ +#define GIT_SHA1_RAWSZ 20 +#define GIT_SHA1_HEXSZ (2 * GIT_SHA1_RAWSZ) + +struct object_id { + unsigned char hash[GIT_SHA1_RAWSZ]; +}; + #if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT) #define DTYPE(de) ((de)->d_type) #else @@@ -214,7 -188,7 +214,7 @@@ struct cache_entry #define CE_INTENT_TO_ADD (1 << 29) #define CE_SKIP_WORKTREE (1 << 30) /* CE_EXTENDED2 is for future extension */ -#define CE_EXTENDED2 (1 << 31) +#define CE_EXTENDED2 (1U << 31) #define CE_EXTENDED_FLAGS (CE_INTENT_TO_ADD | CE_SKIP_WORKTREE) @@@ -228,9 -202,7 +228,9 @@@ #error "CE_EXTENDED_FLAGS out of range" #endif +/* Forward structure decls */ struct pathspec; +struct child_process; /* * Copy the sha1 and stat state of a cache entry from one to @@@ -261,7 -233,6 +261,7 @@@ static inline unsigned create_ce_flags( #define ce_uptodate(ce) ((ce)->ce_flags & CE_UPTODATE) #define ce_skip_worktree(ce) ((ce)->ce_flags & CE_SKIP_WORKTREE) #define ce_mark_uptodate(ce) ((ce)->ce_flags |= CE_UPTODATE) +#define ce_intent_to_add(ce) ((ce)->ce_flags & CE_INTENT_TO_ADD) #define ce_permissions(mode) (((mode) & 0100) ? 0755 : 0644) static inline unsigned int create_ce_mode(unsigned int mode) @@@ -318,11 -289,8 +318,11 @@@ static inline unsigned int canon_mode(u #define RESOLVE_UNDO_CHANGED (1 << 4) #define CACHE_TREE_CHANGED (1 << 5) #define SPLIT_INDEX_ORDERED (1 << 6) +#define UNTRACKED_CHANGED (1 << 7) struct split_index; +struct untracked_cache; + struct index_state { struct cache_entry **cache; unsigned int version; @@@ -336,7 -304,6 +336,7 @@@ struct hashmap name_hash; struct hashmap dir_hash; unsigned char sha1[20]; + struct untracked_cache *untracked; }; extern struct index_state the_index; @@@ -403,7 -370,6 +403,7 @@@ static inline enum object_type object_t /* Double-check local_repo_env below if you add to this list. */ #define GIT_DIR_ENVIRONMENT "GIT_DIR" +#define GIT_COMMON_DIR_ENVIRONMENT "GIT_COMMON_DIR" #define GIT_NAMESPACE_ENVIRONMENT "GIT_NAMESPACE" #define GIT_WORK_TREE_ENVIRONMENT "GIT_WORK_TREE" #define GIT_PREFIX_ENVIRONMENT "GIT_PREFIX" @@@ -418,7 -384,6 +418,7 @@@ #define EXEC_PATH_ENVIRONMENT "GIT_EXEC_PATH" #define CEILING_DIRECTORIES_ENVIRONMENT "GIT_CEILING_DIRECTORIES" #define NO_REPLACE_OBJECTS_ENVIRONMENT "GIT_NO_REPLACE_OBJECTS" +#define GIT_REPLACE_REF_BASE_ENVIRONMENT "GIT_REPLACE_REF_BASE" #define GITATTRIBUTES_FILE ".gitattributes" #define INFOATTRIBUTES_FILE "info/attributes" #define ATTRIBUTE_MACRO_PREFIX "[attr]" @@@ -458,46 -423,15 +458,46 @@@ extern int is_inside_git_dir(void) extern char *git_work_tree_cfg; extern int is_inside_work_tree(void); extern const char *get_git_dir(void); -extern int is_git_directory(const char *path); +extern const char *get_git_common_dir(void); extern char *get_object_directory(void); extern char *get_index_file(void); extern char *get_graft_file(void); extern int set_git_dir(const char *path); +extern int get_common_dir_noenv(struct strbuf *sb, const char *gitdir); +extern int get_common_dir(struct strbuf *sb, const char *gitdir); extern const char *get_git_namespace(void); extern const char *strip_namespace(const char *namespaced_ref); extern const char *get_git_work_tree(void); -extern const char *read_gitfile(const char *path); + +/* + * Return true if the given path is a git directory; note that this _just_ + * looks at the directory itself. If you want to know whether "foo/.git" + * is a repository, you must feed that path, not just "foo". + */ +extern int is_git_directory(const char *path); + +/* + * Return 1 if the given path is the root of a git repository or + * submodule, else 0. Will not return 1 for bare repositories with the + * exception of creating a bare repository in "foo/.git" and calling + * is_git_repository("foo"). + * + * If we run into read errors, we err on the side of saying "yes, it is", + * as we usually consider sub-repos precious, and would prefer to err on the + * side of not disrupting or deleting them. + */ +extern int is_nonbare_repository_dir(struct strbuf *path); + +#define READ_GITFILE_ERR_STAT_FAILED 1 +#define READ_GITFILE_ERR_NOT_A_FILE 2 +#define READ_GITFILE_ERR_OPEN_FAILED 3 +#define READ_GITFILE_ERR_READ_FAILED 4 +#define READ_GITFILE_ERR_INVALID_FORMAT 5 +#define READ_GITFILE_ERR_NO_PATH 6 +#define READ_GITFILE_ERR_NOT_A_REPO 7 +#define READ_GITFILE_ERR_TOO_LARGE 8 +extern const char *read_gitfile_gently(const char *path, int *return_error_code); +#define read_gitfile(path) read_gitfile_gently((path), NULL) extern const char *resolve_gitdir(const char *suspect); extern void set_git_work_tree(const char *tree); @@@ -560,8 -494,7 +560,8 @@@ extern int write_locked_index(struct in extern int discard_index(struct index_state *); extern int unmerged_index(const struct index_state *); extern int verify_path(const char *path); -extern struct cache_entry *index_dir_exists(struct index_state *istate, const char *name, int namelen); +extern int index_dir_exists(struct index_state *istate, const char *name, int namelen); +extern void adjust_dirname_case(struct index_state *istate, char *name); extern struct cache_entry *index_file_exists(struct index_state *istate, const char *name, int namelen, int igncase); extern int index_name_pos(const struct index_state *, const char *name, int namelen); #define ADD_CACHE_OK_TO_ADD 1 /* Ok to add */ @@@ -619,8 -552,6 +619,8 @@@ extern void fill_stat_data(struct stat_ * INODE_CHANGED, and DATA_CHANGED. */ extern int match_stat_data(const struct stat_data *sd, struct stat *st); +extern int match_stat_data_racy(const struct index_state *istate, + const struct stat_data *sd, struct stat *st); extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st); @@@ -637,6 -568,8 +637,6 @@@ extern void update_index_if_able(struc extern int hold_locked_index(struct lock_file *, int); extern void set_alternate_index_output(const char *); -extern int delete_ref(const char *, const unsigned char *sha1, unsigned int flags); - /* Environment bits from configuration mechanism */ extern int trust_executable_bit; extern int trust_ctime; @@@ -672,7 -605,6 +672,7 @@@ extern unsigned long pack_size_limit_cf * been sought but there were none. */ extern int check_replace_refs; +extern char *git_replace_ref_base; extern int fsync_object_files; extern int core_preload_index; @@@ -680,7 -612,6 +680,7 @@@ extern int core_apply_sparse_checkout extern int precomposed_unicode; extern int protect_hfs; extern int protect_ntfs; +extern int git_db_env, git_index_env, git_graft_env, git_common_dir_env; /* * Include broken refs in all ref iterations, which will @@@ -737,15 -668,8 +737,15 @@@ extern char *notes_ref_name extern int grafts_replace_parents; +/* + * GIT_REPO_VERSION is the version we write by default. The + * _READ variant is the highest number we know how to + * handle. + */ #define GIT_REPO_VERSION 0 +#define GIT_REPO_VERSION_READ 1 extern int repository_format_version; +extern int repository_format_precious_objects; extern int check_repository_format(void); #define MTIME_CHANGED 0x0001 @@@ -756,61 -680,21 +756,61 @@@ #define DATA_CHANGED 0x0020 #define TYPE_CHANGED 0x0040 +/* + * Return a statically allocated filename, either generically (mkpath), in + * the repository directory (git_path), or in a submodule's repository + * directory (git_path_submodule). In all cases, note that the result + * may be overwritten by another call to _any_ of the functions. Consider + * using the safer "dup" or "strbuf" formats below (in some cases, the + * unsafe versions have already been removed). + */ +extern const char *mkpath(const char *fmt, ...) __attribute__((format (printf, 1, 2))); +extern const char *git_path(const char *fmt, ...) __attribute__((format (printf, 1, 2))); + extern char *mksnpath(char *buf, size_t n, const char *fmt, ...) __attribute__((format (printf, 3, 4))); -extern char *git_snpath(char *buf, size_t n, const char *fmt, ...) +extern void strbuf_git_path(struct strbuf *sb, const char *fmt, ...) + __attribute__((format (printf, 2, 3))); +extern char *git_path_buf(struct strbuf *buf, const char *fmt, ...) + __attribute__((format (printf, 2, 3))); +extern void strbuf_git_path_submodule(struct strbuf *sb, const char *path, + const char *fmt, ...) __attribute__((format (printf, 3, 4))); extern char *git_pathdup(const char *fmt, ...) __attribute__((format (printf, 1, 2))); extern char *mkpathdup(const char *fmt, ...) __attribute__((format (printf, 1, 2))); - -/* Return a statically allocated filename matching the sha1 signature */ -extern char *mkpath(const char *fmt, ...) __attribute__((format (printf, 1, 2))); -extern char *git_path(const char *fmt, ...) __attribute__((format (printf, 1, 2))); -extern char *git_path_submodule(const char *path, const char *fmt, ...) +extern char *git_pathdup_submodule(const char *path, const char *fmt, ...) __attribute__((format (printf, 2, 3))); +extern void report_linked_checkout_garbage(void); + +/* + * You can define a static memoized git path like: + * + * static GIT_PATH_FUNC(git_path_foo, "FOO"); + * + * or use one of the global ones below. + */ +#define GIT_PATH_FUNC(func, filename) \ + const char *func(void) \ + { \ + static char *ret; \ + if (!ret) \ + ret = git_pathdup(filename); \ + return ret; \ + } + +const char *git_path_cherry_pick_head(void); +const char *git_path_revert_head(void); +const char *git_path_squash_msg(void); +const char *git_path_merge_msg(void); +const char *git_path_merge_rr(void); +const char *git_path_merge_mode(void); +const char *git_path_merge_head(void); +const char *git_path_fetch_head(void); +const char *git_path_shallow(void); + /* * Return the name of the file in the local object database that would * be used to store a loose object with the specified sha1. The @@@ -833,32 -717,14 +833,32 @@@ extern char *sha1_pack_name(const unsig */ extern char *sha1_pack_index_name(const unsigned char *sha1); -extern const char *find_unique_abbrev(const unsigned char *sha1, int); -extern const unsigned char null_sha1[20]; +/* + * Return an abbreviated sha1 unique within this repository's object database. + * The result will be at least `len` characters long, and will be NUL + * terminated. + * + * The non-`_r` version returns a static buffer which will be overwritten by + * subsequent calls. + * + * The `_r` variant writes to a buffer supplied by the caller, which must be at + * least `GIT_SHA1_HEXSZ + 1` bytes. The return value is the number of bytes + * written (excluding the NUL terminator). + * + * Note that while this version avoids the static buffer, it is not fully + * reentrant, as it calls into other non-reentrant git code. + */ +extern const char *find_unique_abbrev(const unsigned char *sha1, int len); +extern int find_unique_abbrev_r(char *hex, const unsigned char *sha1, int len); + +extern const unsigned char null_sha1[GIT_SHA1_RAWSZ]; +extern const struct object_id null_oid; static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2) { int i; - for (i = 0; i < 20; i++, sha1++, sha2++) { + for (i = 0; i < GIT_SHA1_RAWSZ; i++, sha1++, sha2++) { if (*sha1 != *sha2) return *sha1 - *sha2; } @@@ -866,42 -732,20 +866,42 @@@ return 0; } +static inline int oidcmp(const struct object_id *oid1, const struct object_id *oid2) +{ + return hashcmp(oid1->hash, oid2->hash); +} + static inline int is_null_sha1(const unsigned char *sha1) { return !hashcmp(sha1, null_sha1); } +static inline int is_null_oid(const struct object_id *oid) +{ + return !hashcmp(oid->hash, null_sha1); +} + static inline void hashcpy(unsigned char *sha_dst, const unsigned char *sha_src) { - memcpy(sha_dst, sha_src, 20); + memcpy(sha_dst, sha_src, GIT_SHA1_RAWSZ); +} + +static inline void oidcpy(struct object_id *dst, const struct object_id *src) +{ + hashcpy(dst->hash, src->hash); } + static inline void hashclr(unsigned char *hash) { - memset(hash, 0, 20); + memset(hash, 0, GIT_SHA1_RAWSZ); } +static inline void oidclr(struct object_id *oid) +{ + hashclr(oid->hash); +} + + #define EMPTY_TREE_SHA1_HEX \ "4b825dc642cb6eb9a060e54bf8d69288fbee4904" #define EMPTY_TREE_SHA1_BIN_LITERAL \ @@@ -1000,7 -844,6 +1000,7 @@@ extern char *xdg_config_home(const cha /* object replacement */ #define LOOKUP_REPLACE_OBJECT 1 +#define LOOKUP_UNKNOWN_OBJECT 2 extern void *read_sha1_file_extended(const unsigned char *sha1, enum object_type *type, unsigned long *size, unsigned flag); static inline void *read_sha1_file(const unsigned char *sha1, enum object_type *type, unsigned long *size) { @@@ -1050,7 -893,7 +1050,7 @@@ extern int do_check_packed_object_crc extern int check_sha1_signature(const unsigned char *sha1, void *buf, unsigned long size, const char *type); -extern int move_temp_to_file(const char *tmpfile, const char *filename); +extern int finalize_object_file(const char *tmpfile, const char *filename); extern int has_sha1_pack(const unsigned char *sha1); @@@ -1070,9 -913,6 +1070,9 @@@ static inline int has_sha1_file(const u return has_sha1_file_with_flags(sha1, 0); } +/* Same as the above, except for struct object_id. */ +extern int has_object_file(const struct object_id *oid); + /* * Return true iff an alternate object database has a loose object * with the specified name. This function does not respect replace @@@ -1098,21 -938,15 +1098,21 @@@ struct object_context unsigned char tree[20]; char path[PATH_MAX]; unsigned mode; + /* + * symlink_path is only used by get_tree_entry_follow_symlinks, + * and only for symlinks that point outside the repository. + */ + struct strbuf symlink_path; }; -#define GET_SHA1_QUIETLY 01 -#define GET_SHA1_COMMIT 02 -#define GET_SHA1_COMMITTISH 04 -#define GET_SHA1_TREE 010 -#define GET_SHA1_TREEISH 020 -#define GET_SHA1_BLOB 040 -#define GET_SHA1_ONLY_TO_DIE 04000 +#define GET_SHA1_QUIETLY 01 +#define GET_SHA1_COMMIT 02 +#define GET_SHA1_COMMITTISH 04 +#define GET_SHA1_TREE 010 +#define GET_SHA1_TREEISH 020 +#define GET_SHA1_BLOB 040 +#define GET_SHA1_FOLLOW_SYMLINKS 0100 +#define GET_SHA1_ONLY_TO_DIE 04000 extern int get_sha1(const char *str, unsigned char *sha1); extern int get_sha1_commit(const char *str, unsigned char *sha1); @@@ -1134,26 -968,78 +1134,26 @@@ extern int for_each_abbrev(const char * * null-terminated string. */ extern int get_sha1_hex(const char *hex, unsigned char *sha1); - -extern char *sha1_to_hex(const unsigned char *sha1); /* static buffer result! */ -extern int read_ref_full(const char *refname, int resolve_flags, - unsigned char *sha1, int *flags); -extern int read_ref(const char *refname, unsigned char *sha1); +extern int get_oid_hex(const char *hex, struct object_id *sha1); /* - * Resolve a reference, recursively following symbolic refererences. - * - * Store the referred-to object's name in sha1 and return the name of - * the non-symbolic reference that ultimately pointed at it. The - * return value, if not NULL, is a pointer into either a static buffer - * or the input ref. - * - * If the reference cannot be resolved to an object, the behavior - * depends on the RESOLVE_REF_READING flag: + * Convert a binary sha1 to its hex equivalent. The `_r` variant is reentrant, + * and writes the NUL-terminated output to the buffer `out`, which must be at + * least `GIT_SHA1_HEXSZ + 1` bytes, and returns a pointer to out for + * convenience. * - * - If RESOLVE_REF_READING is set, return NULL. + * The non-`_r` variant returns a static buffer, but uses a ring of 4 + * buffers, making it safe to make multiple calls for a single statement, like: * - * - If RESOLVE_REF_READING is not set, clear sha1 and return the name of - * the last reference name in the chain, which will either be a non-symbolic - * reference or an undefined reference. If this is a prelude to - * "writing" to the ref, the return value is the name of the ref - * that will actually be created or changed. - * - * If the RESOLVE_REF_NO_RECURSE flag is passed, only resolves one - * level of symbolic reference. The value stored in sha1 for a symbolic - * reference will always be null_sha1 in this case, and the return - * value is the reference that the symref refers to directly. - * - * If flags is non-NULL, set the value that it points to the - * combination of REF_ISPACKED (if the reference was found among the - * packed references), REF_ISSYMREF (if the initial reference was a - * symbolic reference), REF_BAD_NAME (if the reference name is ill - * formed --- see RESOLVE_REF_ALLOW_BAD_NAME below), and REF_ISBROKEN - * (if the ref is malformed or has a bad name). See refs.h for more detail - * on each flag. - * - * If ref is not a properly-formatted, normalized reference, return - * NULL. If more than MAXDEPTH recursive symbolic lookups are needed, - * give up and return NULL. - * - * RESOLVE_REF_ALLOW_BAD_NAME allows resolving refs even when their - * name is invalid according to git-check-ref-format(1). If the name - * is bad then the value stored in sha1 will be null_sha1 and the two - * flags REF_ISBROKEN and REF_BAD_NAME will be set. - * - * Even with RESOLVE_REF_ALLOW_BAD_NAME, names that escape the refs/ - * directory and do not consist of all caps and underscores cannot be - * resolved. The function returns NULL for such ref names. - * Caps and underscores refers to the special refs, such as HEAD, - * FETCH_HEAD and friends, that all live outside of the refs/ directory. + * printf("%s -> %s", sha1_to_hex(one), sha1_to_hex(two)); */ -#define RESOLVE_REF_READING 0x01 -#define RESOLVE_REF_NO_RECURSE 0x02 -#define RESOLVE_REF_ALLOW_BAD_NAME 0x04 -extern const char *resolve_ref_unsafe(const char *ref, int resolve_flags, unsigned char *sha1, int *flags); -extern char *resolve_refdup(const char *ref, int resolve_flags, unsigned char *sha1, int *flags); - -extern int dwim_ref(const char *str, int len, unsigned char *sha1, char **ref); -extern int dwim_log(const char *str, int len, unsigned char *sha1, char **ref); +extern char *sha1_to_hex_r(char *out, const unsigned char *sha1); +extern char *sha1_to_hex(const unsigned char *sha1); /* static buffer result! */ +extern char *oid_to_hex(const struct object_id *oid); /* same static buffer as sha1_to_hex */ + extern int interpret_branch_name(const char *str, int len, struct strbuf *); extern int get_sha1_mb(const char *str, unsigned char *sha1); -/* - * Return true iff abbrev_name is a possible abbreviation for - * full_name according to the rules defined by ref_rev_parse_rules in - * refs.c. - */ -extern int refname_match(const char *abbrev_name, const char *full_name); - -extern int create_symref(const char *ref, const char *refs_heads_master, const char *logmsg); extern int validate_headref(const char *ref); extern int base_name_compare(const char *name1, int len1, int mode1, const char *name2, int len2, int mode2); @@@ -1169,30 -1055,18 +1169,30 @@@ extern void *read_object_with_reference extern struct object *peel_to_type(const char *name, int namelen, struct object *o, enum object_type); -enum date_mode { - DATE_NORMAL = 0, - DATE_RELATIVE, - DATE_SHORT, - DATE_LOCAL, - DATE_ISO8601, - DATE_ISO8601_STRICT, - DATE_RFC2822, - DATE_RAW +struct date_mode { + enum date_mode_type { + DATE_NORMAL = 0, + DATE_RELATIVE, + DATE_SHORT, + DATE_ISO8601, + DATE_ISO8601_STRICT, + DATE_RFC2822, + DATE_STRFTIME, + DATE_RAW + } type; + const char *strftime_fmt; + int local; }; -const char *show_date(unsigned long time, int timezone, enum date_mode mode); +/* + * Convenience helper for passing a constant type, like: + * + * show_date(t, tz, DATE_MODE(NORMAL)); + */ +#define DATE_MODE(t) date_mode_from_type(DATE_##t) +struct date_mode *date_mode_from_type(enum date_mode_type type); + +const char *show_date(unsigned long time, int timezone, const struct date_mode *mode); void show_date_relative(unsigned long time, int tz, const struct timeval *now, struct strbuf *timebuf); int parse_date(const char *date, struct strbuf *out); @@@ -1202,7 -1076,7 +1202,7 @@@ void datestamp(struct strbuf *out) #define approxidate(s) approxidate_careful((s), NULL) unsigned long approxidate_careful(const char *, int *); unsigned long approxidate_relative(const char *date, const struct timeval *now); -enum date_mode parse_date_format(const char *format); +void parse_date_format(const char *format, struct date_mode *mode); int date_overflows(unsigned long date); #define IDENT_STRICT 1 @@@ -1239,8 -1113,7 +1239,8 @@@ extern int split_ident_line(struct iden * the ident_split. It will also sanity-check the values and produce * a well-known sentinel date if they appear bogus. */ -const char *show_ident_date(const struct ident_split *id, enum date_mode mode); +const char *show_ident_date(const struct ident_split *id, + const struct date_mode *mode); /* * Compare split idents for equality or strict ordering. Note that we @@@ -1332,11 -1205,8 +1332,11 @@@ struct pack_entry extern struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path); -/* A hook for count-objects to report invalid files in pack directory */ -extern void (*report_garbage)(const char *desc, const char *path); +/* A hook to report invalid files in pack directory */ +#define PACKDIR_FILE_PACK 1 +#define PACKDIR_FILE_IDX 2 +#define PACKDIR_FILE_GARBAGE 4 +extern void (*report_garbage)(unsigned seen_bits, const char *path); extern void prepare_packed_git(void); extern void reprepare_packed_git(void); @@@ -1361,12 -1231,21 +1361,22 @@@ extern void close_pack_index(struct pac extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); extern void close_pack_windows(struct packed_git *); +extern void close_all_packs(void); extern void unuse_pack(struct pack_window **); extern void free_pack_by_name(const char *); extern void clear_delta_base_cache(void); -extern struct packed_git *add_packed_git(const char *, int, int); +extern struct packed_git *add_packed_git(const char *path, size_t path_len, int local); + /* + * Make sure that a pointer access into an mmap'd index file is within bounds, + * and can provide at least 8 bytes of data. + * + * Note that this is only necessary for variable-length segments of the file + * (like the 64-bit extended offset table), as we compare the size to the + * fixed-length parts when we open the file. + */ + extern void check_pack_index_ptr(const struct packed_git *p, const void *ptr); + /* * Return the SHA-1 of the nth object within the specified packfile. * Open the index if it is not already open. The return value points @@@ -1454,7 -1333,6 +1464,7 @@@ struct object_info unsigned long *sizep; unsigned long *disk_sizep; unsigned char *delta_base_sha1; + struct strbuf *typename; /* Response */ enum { @@@ -1487,7 -1365,7 +1497,7 @@@ extern int update_server_info(int) /* git_config_parse_key() returns these negated: */ #define CONFIG_INVALID_KEY 1 #define CONFIG_NO_SECTION_OR_NAME 2 -/* git_config_set(), git_config_set_multivar() return the above or these: */ +/* git_config_set_gently(), git_config_set_multivar_gently() return the above or these: */ #define CONFIG_NO_LOCK -1 #define CONFIG_INVALID_FILE 3 #define CONFIG_NO_WRITE 4 @@@ -1516,7 -1394,6 +1526,7 @@@ extern int git_config_with_options(conf int respect_includes); extern int git_config_early(config_fn_t fn, void *, const char *repo_config); extern int git_parse_ulong(const char *, unsigned long *); +extern int git_parse_maybe_bool(const char *); extern int git_config_int(const char *, const char *); extern int64_t git_config_int64(const char *, const char *); extern unsigned long git_config_ulong(const char *, const char *); @@@ -1525,16 -1402,11 +1535,16 @@@ extern int git_config_bool(const char * extern int git_config_maybe_bool(const char *, const char *); extern int git_config_string(const char **, const char *, const char *); extern int git_config_pathname(const char **, const char *, const char *); -extern int git_config_set_in_file(const char *, const char *, const char *); -extern int git_config_set(const char *, const char *); +extern int git_config_set_in_file_gently(const char *, const char *, const char *); +extern void git_config_set_in_file(const char *, const char *, const char *); +extern int git_config_set_gently(const char *, const char *); +extern void git_config_set(const char *, const char *); extern int git_config_parse_key(const char *, char **, int *); -extern int git_config_set_multivar(const char *, const char *, const char *, int); -extern int git_config_set_multivar_in_file(const char *, const char *, const char *, const char *, int); +extern int git_config_key_is_valid(const char *key); +extern int git_config_set_multivar_gently(const char *, const char *, const char *, int); +extern void git_config_set_multivar(const char *, const char *, const char *, int); +extern int git_config_set_multivar_in_file_gently(const char *, const char *, const char *, const char *, int); +extern void git_config_set_multivar_in_file(const char *, const char *, const char *, const char *, int); extern int git_config_rename_section(const char *, const char *); extern int git_config_rename_section_in_file(const char *, const char *, const char *); extern const char *git_etc_gitconfig(void); @@@ -1649,13 -1521,9 +1659,13 @@@ extern const char *git_mailmap_blob extern void maybe_flush_or_die(FILE *, const char *); __attribute__((format (printf, 2, 3))) extern void fprintf_or_die(FILE *, const char *fmt, ...); + +#define COPY_READ_ERROR (-2) +#define COPY_WRITE_ERROR (-3) extern int copy_fd(int ifd, int ofd); extern int copy_file(const char *dst, const char *src, int mode); extern int copy_file_with_time(const char *dst, const char *src, int mode); + extern void write_or_die(int fd, const void *buf, size_t count); extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg); extern int write_or_whine_pipe(int fd, const void *buf, size_t count, const char *msg); @@@ -1670,9 -1538,6 +1680,9 @@@ static inline ssize_t write_str_in_full return write_in_full(fd, str, strlen(str)); } +extern int write_file(const char *path, const char *fmt, ...); +extern int write_file_gently(const char *path, const char *fmt, ...); + /* pager.c */ extern void setup_pager(void); extern const char *pager_program; @@@ -1681,7 -1546,6 +1691,7 @@@ extern int pager_use_color extern int term_columns(void); extern int decimal_width(uintmax_t); extern int check_pager_config(const char *cmd); +extern void prepare_pager_args(struct child_process *, const char *pager); extern const char *editor_program; extern const char *askpass_program; @@@ -1796,14 -1660,5 +1806,14 @@@ int stat_validity_check(struct stat_val void stat_validity_update(struct stat_validity *sv, int fd); int versioncmp(const char *s1, const char *s2); +void sleep_millisec(int millisec); + +/* + * Create a directory and (if share is nonzero) adjust its permissions + * according to the shared_repository setting. Only use this for + * directories under $GIT_DIR. Don't use it for working tree + * directories. + */ +void safe_create_dir(const char *dir, int share); #endif /* CACHE_H */ diff --combined sha1_file.c index ab16c7b98c,b8da68baa6..b516874473 --- a/sha1_file.c +++ b/sha1_file.c @@@ -36,7 -36,6 +36,7 @@@ static inline uintmax_t sz_fmt(size_t s) { return s; } const unsigned char null_sha1[20]; +const struct object_id null_oid; /* * This is meant to hold a *small* number of objects that you would @@@ -209,25 -208,44 +209,25 @@@ const char *sha1_file_name(const unsign * provided by the caller. which should be "pack" or "idx". */ static char *sha1_get_pack_name(const unsigned char *sha1, - char **name, char **base, const char *which) + struct strbuf *buf, + const char *which) { - static const char hex[] = "0123456789abcdef"; - char *buf; - int i; - - if (!*base) { - const char *sha1_file_directory = get_object_directory(); - int len = strlen(sha1_file_directory); - *base = xmalloc(len + 60); - sprintf(*base, "%s/pack/pack-1234567890123456789012345678901234567890.%s", - sha1_file_directory, which); - *name = *base + len + 11; - } - - buf = *name; - - for (i = 0; i < 20; i++) { - unsigned int val = *sha1++; - *buf++ = hex[val >> 4]; - *buf++ = hex[val & 0xf]; - } - - return *base; + strbuf_reset(buf); + strbuf_addf(buf, "%s/pack/pack-%s.%s", get_object_directory(), + sha1_to_hex(sha1), which); + return buf->buf; } char *sha1_pack_name(const unsigned char *sha1) { - static char *name, *base; - - return sha1_get_pack_name(sha1, &name, &base, "pack"); + static struct strbuf buf = STRBUF_INIT; + return sha1_get_pack_name(sha1, &buf, "pack"); } char *sha1_pack_index_name(const unsigned char *sha1) { - static char *name, *base; - - return sha1_get_pack_name(sha1, &name, &base, "idx"); + static struct strbuf buf = STRBUF_INIT; + return sha1_get_pack_name(sha1, &buf, "idx"); } struct alternate_object_database *alt_odb_list; @@@ -253,7 -271,7 +253,7 @@@ static int link_alt_odb_entry(const cha { struct alternate_object_database *ent; struct alternate_object_database *alt; - int pfxlen, entlen; + size_t pfxlen, entlen; struct strbuf pathbuf = STRBUF_INIT; if (!is_absolute_path(entry) && relative_base) { @@@ -273,8 -291,8 +273,8 @@@ while (pfxlen && pathbuf.buf[pfxlen-1] == '/') pfxlen -= 1; - entlen = pfxlen + 43; /* '/' + 2 hex + '/' + 38 hex + NUL */ - ent = xmalloc(sizeof(*ent) + entlen); + entlen = st_add(pfxlen, 43); /* '/' + 2 hex + '/' + 38 hex + NUL */ + ent = xmalloc(st_add(sizeof(*ent), entlen)); memcpy(ent->base, pathbuf.buf, pfxlen); strbuf_release(&pathbuf); @@@ -383,46 -401,13 +383,46 @@@ void read_info_alternates(const char * void add_to_alternates_file(const char *reference) { struct lock_file *lock = xcalloc(1, sizeof(struct lock_file)); - int fd = hold_lock_file_for_append(lock, git_path("objects/info/alternates"), LOCK_DIE_ON_ERROR); - char *alt = mkpath("%s\n", reference); - write_or_die(fd, alt, strlen(alt)); - if (commit_lock_file(lock)) - die("could not close alternates file"); - if (alt_odb_tail) - link_alt_odb_entries(alt, strlen(alt), '\n', NULL, 0); + char *alts = git_pathdup("objects/info/alternates"); + FILE *in, *out; + + hold_lock_file_for_update(lock, alts, LOCK_DIE_ON_ERROR); + out = fdopen_lock_file(lock, "w"); + if (!out) + die_errno("unable to fdopen alternates lockfile"); + + in = fopen(alts, "r"); + if (in) { + struct strbuf line = STRBUF_INIT; + int found = 0; + + while (strbuf_getline(&line, in, '\n') != EOF) { + if (!strcmp(reference, line.buf)) { + found = 1; + break; + } + fprintf_or_die(out, "%s\n", line.buf); + } + + strbuf_release(&line); + fclose(in); + + if (found) { + rollback_lock_file(lock); + lock = NULL; + } + } + else if (errno != ENOENT) + die_errno("unable to read alternates file"); + + if (lock) { + fprintf_or_die(out, "%s\n", reference); + if (commit_lock_file(lock)) + die_errno("unable to move new alternates file into place"); + if (alt_odb_tail) + link_alt_odb_entries(reference, strlen(reference), '\n', NULL, 0); + } + free(alts); } int foreach_alt_odb(alt_odb_fn fn, void *cb) @@@ -653,15 -638,13 +653,15 @@@ static int check_packed_git_idx(const c int open_pack_index(struct packed_git *p) { char *idx_name; + size_t len; int ret; if (p->index_data) return 0; - idx_name = xstrdup(p->pack_name); - strcpy(idx_name + strlen(idx_name) - strlen(".pack"), ".idx"); + if (!strip_suffix(p->pack_name, ".pack", &len)) + die("BUG: pack_name does not end in .pack"); + idx_name = xstrfmt("%.*s.idx", (int)len, p->pack_name); ret = check_packed_git_idx(idx_name, p); free(idx_name); return ret; @@@ -770,37 -753,6 +770,37 @@@ void close_pack_windows(struct packed_g } } +static int close_pack_fd(struct packed_git *p) +{ + if (p->pack_fd < 0) + return 0; + + close(p->pack_fd); + pack_open_fds--; + p->pack_fd = -1; + + return 1; +} + +static void close_pack(struct packed_git *p) +{ + close_pack_windows(p); + close_pack_fd(p); + close_pack_index(p); +} + +void close_all_packs(void) +{ + struct packed_git *p; + + for (p = packed_git; p; p = p->next) + if (p->do_not_close) + die("BUG! Want to close pack marked 'do-not-close'"); + else + close_pack(p); +} + + /* * The LRU pack is the one with the oldest MRU window, preferring packs * with no used windows, or the oldest mtime if it has no windows allocated. @@@ -868,8 -820,12 +868,8 @@@ static int close_one_pack(void find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse); } - if (lru_p) { - close(lru_p->pack_fd); - pack_open_fds--; - lru_p->pack_fd = -1; - return 1; - } + if (lru_p) + return close_pack_fd(lru_p); return 0; } @@@ -909,7 -865,12 +909,7 @@@ void free_pack_by_name(const char *pack p = *pp; if (strcmp(pack_name, p->pack_name) == 0) { clear_delta_base_cache(); - close_pack_windows(p); - if (p->pack_fd != -1) { - close(p->pack_fd); - pack_open_fds--; - } - close_pack_index(p); + close_pack(p); free(p->bad_object_sha1); *pp = p->next; if (last_found_pack == p) @@@ -1043,7 -1004,11 +1043,7 @@@ static int open_packed_git(struct packe { if (!open_packed_git_1(p)) return 0; - if (p->pack_fd != -1) { - close(p->pack_fd); - pack_open_fds--; - p->pack_fd = -1; - } + close_pack_fd(p); return -1; } @@@ -1076,6 -1041,8 +1076,8 @@@ unsigned char *use_pack(struct packed_g die("packfile %s cannot be accessed", p->pack_name); if (offset > (p->pack_size - 20)) die("offset beyond end of packfile (truncated pack?)"); + if (offset < 0) + die(_("offset before end of packfile (broken .idx?)")); if (!win || !in_window(win, offset)) { if (win) @@@ -1109,8 -1076,11 +1111,8 @@@ p->pack_name, strerror(errno)); if (!win->offset && win->len == p->pack_size - && !p->do_not_close) { - close(p->pack_fd); - pack_open_fds--; - p->pack_fd = -1; - } + && !p->do_not_close) + close_pack_fd(p); pack_mmap_calls++; pack_open_windows++; if (pack_mapped > peak_pack_mapped) @@@ -1134,7 -1104,7 +1136,7 @@@ static struct packed_git *alloc_packed_git(int extra) { - struct packed_git *p = xmalloc(sizeof(*p) + extra); + struct packed_git *p = xmalloc(st_add(sizeof(*p), extra)); memset(p, 0, sizeof(*p)); p->pack_fd = -1; return p; @@@ -1145,12 -1115,11 +1147,12 @@@ static void try_to_free_pack_memory(siz release_pack_memory(size); } -struct packed_git *add_packed_git(const char *path, int path_len, int local) +struct packed_git *add_packed_git(const char *path, size_t path_len, int local) { static int have_set_try_to_free_routine; struct stat st; - struct packed_git *p = alloc_packed_git(path_len + 2); + size_t alloc; + struct packed_git *p; if (!have_set_try_to_free_routine) { have_set_try_to_free_routine = 1; @@@ -1161,22 -1130,18 +1163,22 @@@ * Make sure a corresponding .pack file exists and that * the index looks sane. */ - path_len -= strlen(".idx"); - if (path_len < 1) { - free(p); + if (!strip_suffix_mem(path, &path_len, ".idx")) return NULL; - } + + /* + * ".pack" is long enough to hold any suffix we're adding (and + * the use xsnprintf double-checks that) + */ + alloc = st_add3(path_len, strlen(".pack"), 1); + p = alloc_packed_git(alloc); memcpy(p->pack_name, path, path_len); - strcpy(p->pack_name + path_len, ".keep"); + xsnprintf(p->pack_name + path_len, alloc - path_len, ".keep"); if (!access(p->pack_name, F_OK)) p->pack_keep = 1; - strcpy(p->pack_name + path_len, ".pack"); + xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack"); if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) { free(p); return NULL; @@@ -1196,10 -1161,9 +1198,10 @@@ struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path) { const char *path = sha1_pack_name(sha1); - struct packed_git *p = alloc_packed_git(strlen(path) + 1); + size_t alloc = st_add(strlen(path), 1); + struct packed_git *p = alloc_packed_git(alloc); - strcpy(p->pack_name, path); + memcpy(p->pack_name, path, alloc); /* includes NUL */ hashcpy(p->sha1, sha1); if (check_packed_git_idx(idx_path, p)) { free(p); @@@ -1218,16 -1182,27 +1220,16 @@@ void install_packed_git(struct packed_g packed_git = pack; } -void (*report_garbage)(const char *desc, const char *path); +void (*report_garbage)(unsigned seen_bits, const char *path); static void report_helper(const struct string_list *list, int seen_bits, int first, int last) { - const char *msg; - switch (seen_bits) { - case 0: - msg = "no corresponding .idx or .pack"; - break; - case 1: - msg = "no corresponding .idx"; - break; - case 2: - msg = "no corresponding .pack"; - break; - default: + if (seen_bits == (PACKDIR_FILE_PACK|PACKDIR_FILE_IDX)) return; - } + for (; first < last; first++) - report_garbage(msg, list->items[first].string); + report_garbage(seen_bits, list->items[first].string); } static void report_pack_garbage(struct string_list *list) @@@ -1250,7 -1225,7 +1252,7 @@@ if (baselen == -1) { const char *dot = strrchr(path, '.'); if (!dot) { - report_garbage("garbage found", path); + report_garbage(PACKDIR_FILE_GARBAGE, path); continue; } baselen = dot - path + 1; @@@ -1322,7 -1297,7 +1324,7 @@@ static void prepare_packed_git_one(cha ends_with(de->d_name, ".keep")) string_list_append(&garbage, path.buf); else - report_garbage("garbage found", path.buf); + report_garbage(PACKDIR_FILE_GARBAGE, path.buf); } closedir(dir); report_pack_garbage(&garbage); @@@ -1413,12 -1388,10 +1415,12 @@@ static void mark_bad_packed_object(stru { unsigned i; for (i = 0; i < p->num_bad_objects; i++) - if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i)) + if (!hashcmp(sha1, p->bad_object_sha1 + GIT_SHA1_RAWSZ * i)) return; - p->bad_object_sha1 = xrealloc(p->bad_object_sha1, 20 * (p->num_bad_objects + 1)); - hashcpy(p->bad_object_sha1 + 20 * p->num_bad_objects, sha1); + p->bad_object_sha1 = xrealloc(p->bad_object_sha1, + st_mult(GIT_SHA1_RAWSZ, + st_add(p->num_bad_objects, 1))); + hashcpy(p->bad_object_sha1 + GIT_SHA1_RAWSZ * p->num_bad_objects, sha1); p->num_bad_objects++; } @@@ -1460,7 -1433,7 +1462,7 @@@ int check_sha1_signature(const unsigne return -1; /* Generate the header */ - hdrlen = sprintf(hdr, "%s %lu", typename(obj_type), size) + 1; + hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(obj_type), size) + 1; /* Sha1.. */ git_SHA1_Init(&c); @@@ -1487,10 -1460,7 +1489,10 @@@ int git_open_noatime(const char *name static int sha1_file_open_flag = O_NOATIME; for (;;) { - int fd = open(name, O_RDONLY | sha1_file_open_flag); + int fd; + + errno = 0; + fd = open(name, O_RDONLY | sha1_file_open_flag); if (fd >= 0) return fd; @@@ -1608,40 -1578,6 +1610,40 @@@ int unpack_sha1_header(git_zstream *str return git_inflate(stream, 0); } +static int unpack_sha1_header_to_strbuf(git_zstream *stream, unsigned char *map, + unsigned long mapsize, void *buffer, + unsigned long bufsiz, struct strbuf *header) +{ + int status; + + status = unpack_sha1_header(stream, map, mapsize, buffer, bufsiz); + + /* + * Check if entire header is unpacked in the first iteration. + */ + if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer)) + return 0; + + /* + * buffer[0..bufsiz] was not large enough. Copy the partial + * result out to header, and then append the result of further + * reading the stream. + */ + strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer); + stream->next_out = buffer; + stream->avail_out = bufsiz; + + do { + status = git_inflate(stream, 0); + strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer); + if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer)) + return 0; + stream->next_out = buffer; + stream->avail_out = bufsiz; + } while (status != Z_STREAM_END); + return -1; +} + static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1) { int bytes = strlen(buffer) + 1; @@@ -1692,38 -1628,27 +1694,38 @@@ * too permissive for what we want to check. So do an anal * object header parse by hand. */ -int parse_sha1_header(const char *hdr, unsigned long *sizep) +static int parse_sha1_header_extended(const char *hdr, struct object_info *oi, + unsigned int flags) { - char type[10]; - int i; + const char *type_buf = hdr; unsigned long size; + int type, type_len = 0; /* - * The type can be at most ten bytes (including the - * terminating '\0' that we add), and is followed by + * The type can be of any size but is followed by * a space. */ - i = 0; for (;;) { char c = *hdr++; if (c == ' ') break; - type[i++] = c; - if (i >= sizeof(type)) - return -1; + type_len++; } - type[i] = 0; + + type = type_from_string_gently(type_buf, type_len, 1); + if (oi->typename) + strbuf_add(oi->typename, type_buf, type_len); + /* + * Set type to 0 if its an unknown object and + * we're obtaining the type using '--allow-unkown-type' + * option. + */ + if ((flags & LOOKUP_UNKNOWN_OBJECT) && (type < 0)) + type = 0; + else if (type < 0) + die("invalid object type"); + if (oi->typep) + *oi->typep = type; /* * The length must follow immediately, and be in canonical @@@ -1741,24 -1666,12 +1743,24 @@@ size = size * 10 + c; } } - *sizep = size; + + if (oi->sizep) + *oi->sizep = size; /* * The length must be followed by a zero byte */ - return *hdr ? -1 : type_from_string(type); + return *hdr ? -1 : type; +} + +int parse_sha1_header(const char *hdr, unsigned long *sizep) +{ + struct object_info oi; + + oi.sizep = sizep; + oi.typename = NULL; + oi.typep = NULL; + return parse_sha1_header_extended(hdr, &oi, LOOKUP_REPLACE_OBJECT); } static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1) @@@ -1944,7 -1857,7 +1946,7 @@@ static enum object_type packed_to_objec /* Push the object we're going to leave behind */ if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) { poi_stack_alloc = alloc_nr(poi_stack_nr); - poi_stack = xmalloc(sizeof(off_t)*poi_stack_alloc); + ALLOC_ARRAY(poi_stack, poi_stack_alloc); memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr); } else { ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc); @@@ -2118,7 -2031,7 +2120,7 @@@ static unsigned long pack_entry_hash(st { unsigned long hash; - hash = (unsigned long)p + (unsigned long)base_offset; + hash = (unsigned long)(intptr_t)p + (unsigned long)base_offset; hash += (hash >> 8) + (hash >> 16); return hash % MAX_DELTA_CACHE; } @@@ -2310,7 -2223,7 +2312,7 @@@ void *unpack_entry(struct packed_git *p if (delta_stack_nr >= delta_stack_alloc && delta_stack == small_delta_stack) { delta_stack_alloc = alloc_nr(delta_stack_nr); - delta_stack = xmalloc(sizeof(*delta_stack)*delta_stack_alloc); + ALLOC_ARRAY(delta_stack, delta_stack_alloc); memcpy(delta_stack, small_delta_stack, sizeof(*delta_stack)*delta_stack_nr); } else { @@@ -2448,6 -2361,20 +2450,20 @@@ const unsigned char *nth_packed_object_ } } + void check_pack_index_ptr(const struct packed_git *p, const void *vptr) + { + const unsigned char *ptr = vptr; + const unsigned char *start = p->index_data; + const unsigned char *end = start + p->index_size; + if (ptr < start) + die(_("offset before start of pack index for %s (corrupt index?)"), + p->pack_name); + /* No need to check for underflow; .idx files must be at least 8 bytes */ + if (ptr >= end - 8) + die(_("offset beyond end of pack index for %s (truncated index?)"), + p->pack_name); + } + off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n) { const unsigned char *index = p->index_data; @@@ -2461,6 -2388,7 +2477,7 @@@ if (!(off & 0x80000000)) return off; index += p->num_objects * 4 + (off & 0x7fffffff) * 8; + check_pack_index_ptr(p, index); return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) | ntohl(*((uint32_t *)(index + 4))); } @@@ -2623,15 -2551,13 +2640,15 @@@ struct packed_git *find_sha1_pack(cons } static int sha1_loose_object_info(const unsigned char *sha1, - struct object_info *oi) + struct object_info *oi, + int flags) { - int status; - unsigned long mapsize, size; + int status = 0; + unsigned long mapsize; void *map; git_zstream stream; char hdr[32]; + struct strbuf hdrbuf = STRBUF_INIT; if (oi->delta_base_sha1) hashclr(oi->delta_base_sha1); @@@ -2644,7 -2570,7 +2661,7 @@@ * return value implicitly indicates whether the * object even exists. */ - if (!oi->typep && !oi->sizep) { + if (!oi->typep && !oi->typename && !oi->sizep) { struct stat st; if (stat_sha1_file(sha1, &st) < 0) return -1; @@@ -2658,26 -2584,17 +2675,26 @@@ return -1; if (oi->disk_sizep) *oi->disk_sizep = mapsize; - if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) + if ((flags & LOOKUP_UNKNOWN_OBJECT)) { + if (unpack_sha1_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0) + status = error("unable to unpack %s header with --allow-unknown-type", + sha1_to_hex(sha1)); + } else if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) status = error("unable to unpack %s header", sha1_to_hex(sha1)); - else if ((status = parse_sha1_header(hdr, &size)) < 0) + if (status < 0) + ; /* Do nothing */ + else if (hdrbuf.len) { + if ((status = parse_sha1_header_extended(hdrbuf.buf, oi, flags)) < 0) + status = error("unable to parse %s header with --allow-unknown-type", + sha1_to_hex(sha1)); + } else if ((status = parse_sha1_header_extended(hdr, oi, flags)) < 0) status = error("unable to parse %s header", sha1_to_hex(sha1)); - else if (oi->sizep) - *oi->sizep = size; git_inflate_end(&stream); munmap(map, mapsize); - if (oi->typep) + if (status && oi->typep) *oi->typep = status; + strbuf_release(&hdrbuf); return 0; } @@@ -2686,7 -2603,6 +2703,7 @@@ int sha1_object_info_extended(const uns struct cached_object *co; struct pack_entry e; int rtype; + enum object_type real_type; const unsigned char *real = lookup_replace_object_extended(sha1, flags); co = find_cached_object(real); @@@ -2699,15 -2615,13 +2716,15 @@@ *(oi->disk_sizep) = 0; if (oi->delta_base_sha1) hashclr(oi->delta_base_sha1); + if (oi->typename) + strbuf_addstr(oi->typename, typename(co->type)); oi->whence = OI_CACHED; return 0; } if (!find_pack_entry(real, &e)) { /* Most likely it's a loose object. */ - if (!sha1_loose_object_info(real, oi)) { + if (!sha1_loose_object_info(real, oi, flags)) { oi->whence = OI_LOOSE; return 0; } @@@ -2718,18 -2632,9 +2735,18 @@@ return -1; } + /* + * packed_object_info() does not follow the delta chain to + * find out the real type, unless it is given oi->typep. + */ + if (oi->typename && !oi->typep) + oi->typep = &real_type; + rtype = packed_object_info(e.p, e.offset, oi); if (rtype < 0) { mark_bad_packed_object(e.p, real); + if (oi->typep == &real_type) + oi->typep = NULL; return sha1_object_info_extended(real, oi, 0); } else if (in_delta_base_cache(e.p, e.offset)) { oi->whence = OI_DBCACHED; @@@ -2740,10 -2645,6 +2757,10 @@@ oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA || rtype == OBJ_OFS_DELTA); } + if (oi->typename) + strbuf_addstr(oi->typename, typename(*oi->typep)); + if (oi->typep == &real_type) + oi->typep = NULL; return 0; } @@@ -2926,7 -2827,7 +2943,7 @@@ static void write_sha1_file_prepare(con git_SHA_CTX c; /* Generate the header */ - *hdrlen = sprintf(hdr, "%s %lu", type, len)+1; + *hdrlen = xsnprintf(hdr, *hdrlen, "%s %lu", type, len)+1; /* Sha1.. */ git_SHA1_Init(&c); @@@ -2937,8 -2838,11 +2954,8 @@@ /* * Move the just written object into its final resting place. - * NEEDSWORK: this should be renamed to finalize_temp_file() as - * "moving" is only a part of what it does, when no patch between - * master to pu changes the call sites of this function. */ -int move_temp_to_file(const char *tmpfile, const char *filename) +int finalize_object_file(const char *tmpfile, const char *filename) { int ret = 0; @@@ -2989,7 -2893,7 +3006,7 @@@ int hash_sha1_file(const void *buf, uns unsigned char *sha1) { char hdr[32]; - int hdrlen; + int hdrlen = sizeof(hdr); write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen); return 0; } @@@ -3019,31 -2923,29 +3036,31 @@@ static inline int directory_size(const * We want to avoid cross-directory filename renames, because those * can have problems on various filesystems (FAT, NFS, Coda). */ -static int create_tmpfile(char *buffer, size_t bufsiz, const char *filename) +static int create_tmpfile(struct strbuf *tmp, const char *filename) { int fd, dirlen = directory_size(filename); - if (dirlen + 20 > bufsiz) { - errno = ENAMETOOLONG; - return -1; - } - memcpy(buffer, filename, dirlen); - strcpy(buffer + dirlen, "tmp_obj_XXXXXX"); - fd = git_mkstemp_mode(buffer, 0444); + strbuf_reset(tmp); + strbuf_add(tmp, filename, dirlen); + strbuf_addstr(tmp, "tmp_obj_XXXXXX"); + fd = git_mkstemp_mode(tmp->buf, 0444); if (fd < 0 && dirlen && errno == ENOENT) { - /* Make sure the directory exists */ - memcpy(buffer, filename, dirlen); - buffer[dirlen-1] = 0; - if (mkdir(buffer, 0777) && errno != EEXIST) + /* + * Make sure the directory exists; note that the contents + * of the buffer are undefined after mkstemp returns an + * error, so we have to rewrite the whole buffer from + * scratch. + */ + strbuf_reset(tmp); + strbuf_add(tmp, filename, dirlen - 1); + if (mkdir(tmp->buf, 0777) && errno != EEXIST) return -1; - if (adjust_shared_perm(buffer)) + if (adjust_shared_perm(tmp->buf)) return -1; /* Try again */ - strcpy(buffer + dirlen - 1, "/tmp_obj_XXXXXX"); - fd = git_mkstemp_mode(buffer, 0444); + strbuf_addstr(tmp, "/tmp_obj_XXXXXX"); + fd = git_mkstemp_mode(tmp->buf, 0444); } return fd; } @@@ -3056,10 -2958,10 +3073,10 @@@ static int write_loose_object(const uns git_zstream stream; git_SHA_CTX c; unsigned char parano_sha1[20]; - static char tmp_file[PATH_MAX]; + static struct strbuf tmp_file = STRBUF_INIT; const char *filename = sha1_file_name(sha1); - fd = create_tmpfile(tmp_file, sizeof(tmp_file), filename); + fd = create_tmpfile(&tmp_file, filename); if (fd < 0) { if (errno == EACCES) return error("insufficient permission for adding an object to repository database %s", get_object_directory()); @@@ -3108,12 -3010,12 +3125,12 @@@ struct utimbuf utb; utb.actime = mtime; utb.modtime = mtime; - if (utime(tmp_file, &utb) < 0) + if (utime(tmp_file.buf, &utb) < 0) warning("failed utime() on %s: %s", - tmp_file, strerror(errno)); + tmp_file.buf, strerror(errno)); } - return move_temp_to_file(tmp_file, filename); + return finalize_object_file(tmp_file.buf, filename); } static int freshen_loose_object(const unsigned char *sha1) @@@ -3137,7 -3039,7 +3154,7 @@@ static int freshen_packed_object(const int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1) { char hdr[32]; - int hdrlen; + int hdrlen = sizeof(hdr); /* Normally if we have it in the pack then we do not bother writing * it out into .git/objects/??/?{38} file. @@@ -3155,8 -3057,7 +3172,8 @@@ int hash_sha1_file_literally(const voi int hdrlen, status = 0; /* type string, SP, %lu of the length plus NUL must fit this */ - header = xmalloc(strlen(type) + 32); + hdrlen = strlen(type) + 32; + header = xmalloc(hdrlen); write_sha1_file_prepare(buf, len, type, sha1, header, &hdrlen); if (!(flags & HASH_WRITE_OBJECT)) @@@ -3184,7 -3085,7 +3201,7 @@@ int force_object_loose(const unsigned c buf = read_packed_sha1(sha1, &type, &len); if (!buf) return error("cannot read sha1_file for %s", sha1_to_hex(sha1)); - hdrlen = sprintf(hdr, "%s %lu", typename(type), len) + 1; + hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(type), len) + 1; ret = write_loose_object(sha1, hdr, hdrlen, buf, len, mtime); free(buf); @@@ -3219,11 -3120,6 +3236,11 @@@ int has_sha1_file_with_flags(const unsi return find_pack_entry(sha1, &e); } +int has_object_file(const struct object_id *oid) +{ + return has_sha1_file(oid->hash); +} + static void check_tree(const void *buf, size_t size) { struct tree_desc desc; @@@ -3507,12 -3403,12 +3524,12 @@@ static int for_each_file_in_obj_subdir( break; } } - strbuf_setlen(path, baselen); + closedir(dir); + strbuf_setlen(path, baselen); if (!r && subdir_cb) r = subdir_cb(subdir_nr, path->buf, data); - closedir(dir); return r; } @@@ -3617,19 -3513,14 +3634,19 @@@ int for_each_packed_object(each_packed_ { struct packed_git *p; int r = 0; + int pack_errors = 0; prepare_packed_git(); for (p = packed_git; p; p = p->next) { if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local) continue; + if (open_pack_index(p)) { + pack_errors = 1; + continue; + } r = for_each_object_in_pack(p, cb, data); if (r) break; } - return r; + return r ? r : pack_errors; }