From: Junio C Hamano Date: Wed, 24 Jun 2015 19:21:54 +0000 (-0700) Subject: Merge branch 'jk/index-pack-reduce-recheck' X-Git-Tag: v2.5.0-rc0~12 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/712b351bd37d4e4676d1165993cad4cc2176b346?hp=-c Merge branch 'jk/index-pack-reduce-recheck' Disable "have we lost a race with competing repack?" check while receiving a huge object transfer that runs index-pack. * jk/index-pack-reduce-recheck: index-pack: avoid excessive re-reading of pack directory --- 712b351bd37d4e4676d1165993cad4cc2176b346 diff --combined builtin/index-pack.c index 2ecfbae239,96b110445d..48fa4724aa --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@@ -18,14 -18,16 +18,14 @@@ static const char index_pack_usage[] struct object_entry { struct pack_idx_entry idx; unsigned long size; - unsigned int hdr_size; - enum object_type type; - enum object_type real_type; - unsigned delta_depth; - int base_object_no; + unsigned char hdr_size; + signed char type; + signed char real_type; }; -union delta_base { - unsigned char sha1[20]; - off_t offset; +struct object_stat { + unsigned delta_depth; + int base_object_no; }; struct base_data { @@@ -47,28 -49,25 +47,28 @@@ struct thread_local int pack_fd; }; -/* - * Even if sizeof(union delta_base) == 24 on 64-bit archs, we really want - * to memcmp() only the first 20 bytes. - */ -#define UNION_BASE_SZ 20 - #define FLAG_LINK (1u<<20) #define FLAG_CHECKED (1u<<21) -struct delta_entry { - union delta_base base; +struct ofs_delta_entry { + off_t offset; + int obj_no; +}; + +struct ref_delta_entry { + unsigned char sha1[20]; int obj_no; }; static struct object_entry *objects; -static struct delta_entry *deltas; +static struct object_stat *obj_stat; +static struct ofs_delta_entry *ofs_deltas; +static struct ref_delta_entry *ref_deltas; static struct thread_local nothread_data; static int nr_objects; -static int nr_deltas; +static int nr_ofs_deltas; +static int nr_ref_deltas; +static int ref_deltas_alloc; static int nr_resolved_deltas; static int nr_threads; @@@ -113,10 -112,6 +113,10 @@@ static pthread_mutex_t deepest_delta_mu #define deepest_delta_lock() lock_mutex(&deepest_delta_mutex) #define deepest_delta_unlock() unlock_mutex(&deepest_delta_mutex) +static pthread_mutex_t type_cas_mutex; +#define type_cas_lock() lock_mutex(&type_cas_mutex) +#define type_cas_unlock() unlock_mutex(&type_cas_mutex) + static pthread_key_t key; static inline void lock_mutex(pthread_mutex_t *mutex) @@@ -140,7 -135,6 +140,7 @@@ static void init_thread(void init_recursive_mutex(&read_mutex); pthread_mutex_init(&counter_mutex, NULL); pthread_mutex_init(&work_mutex, NULL); + pthread_mutex_init(&type_cas_mutex, NULL); if (show_stat) pthread_mutex_init(&deepest_delta_mutex, NULL); pthread_key_create(&key, NULL); @@@ -163,7 -157,6 +163,7 @@@ static void cleanup_thread(void pthread_mutex_destroy(&read_mutex); pthread_mutex_destroy(&counter_mutex); pthread_mutex_destroy(&work_mutex); + pthread_mutex_destroy(&type_cas_mutex); if (show_stat) pthread_mutex_destroy(&deepest_delta_mutex); for (i = 0; i < nr_threads; i++) @@@ -186,9 -179,6 +186,9 @@@ #define deepest_delta_lock() #define deepest_delta_unlock() +#define type_cas_lock() +#define type_cas_unlock() + #endif @@@ -448,7 -438,7 +448,7 @@@ static void *unpack_entry_data(unsigne if (type == OBJ_BLOB && size > big_file_threshold) buf = fixed_buf; else - buf = xmalloc(size); + buf = xmallocz(size); memset(&stream, 0, sizeof(stream)); git_inflate_init(&stream); @@@ -477,8 -467,7 +477,8 @@@ } static void *unpack_raw_entry(struct object_entry *obj, - union delta_base *delta_base, + off_t *ofs_offset, + unsigned char *ref_sha1, unsigned char *sha1) { unsigned char *p; @@@ -507,10 -496,11 +507,10 @@@ switch (obj->type) { case OBJ_REF_DELTA: - hashcpy(delta_base->sha1, fill(20)); + hashcpy(ref_sha1, fill(20)); use(20); break; case OBJ_OFS_DELTA: - memset(delta_base, 0, sizeof(*delta_base)); p = fill(1); c = *p; use(1); @@@ -524,8 -514,8 +524,8 @@@ use(1); base_offset = (base_offset << 7) + (c & 127); } - delta_base->offset = obj->idx.offset - base_offset; - if (delta_base->offset <= 0 || delta_base->offset >= obj->idx.offset) + *ofs_offset = obj->idx.offset - base_offset; + if (*ofs_offset <= 0 || *ofs_offset >= obj->idx.offset) bad_object(obj->idx.offset, _("delta base offset is out of bound")); break; case OBJ_COMMIT: @@@ -553,7 -543,7 +553,7 @@@ static void *unpack_data(struct object_ git_zstream stream; int status; - data = xmalloc(consume ? 64*1024 : obj->size); + data = xmallocz(consume ? 64*1024 : obj->size); inbuf = xmalloc((len < 64*1024) ? len : 64*1024); memset(&stream, 0, sizeof(stream)); @@@ -609,110 -599,55 +609,110 @@@ static void *get_data_from_pack(struct return unpack_data(obj, NULL, NULL); } -static int compare_delta_bases(const union delta_base *base1, - const union delta_base *base2, - enum object_type type1, - enum object_type type2) +static int compare_ofs_delta_bases(off_t offset1, off_t offset2, + enum object_type type1, + enum object_type type2) +{ + int cmp = type1 - type2; + if (cmp) + return cmp; + return offset1 < offset2 ? -1 : + offset1 > offset2 ? 1 : + 0; +} + +static int find_ofs_delta(const off_t offset, enum object_type type) +{ + int first = 0, last = nr_ofs_deltas; + + while (first < last) { + int next = (first + last) / 2; + struct ofs_delta_entry *delta = &ofs_deltas[next]; + int cmp; + + cmp = compare_ofs_delta_bases(offset, delta->offset, + type, objects[delta->obj_no].type); + if (!cmp) + return next; + if (cmp < 0) { + last = next; + continue; + } + first = next+1; + } + return -first-1; +} + +static void find_ofs_delta_children(off_t offset, + int *first_index, int *last_index, + enum object_type type) +{ + int first = find_ofs_delta(offset, type); + int last = first; + int end = nr_ofs_deltas - 1; + + if (first < 0) { + *first_index = 0; + *last_index = -1; + return; + } + while (first > 0 && ofs_deltas[first - 1].offset == offset) + --first; + while (last < end && ofs_deltas[last + 1].offset == offset) + ++last; + *first_index = first; + *last_index = last; +} + +static int compare_ref_delta_bases(const unsigned char *sha1, + const unsigned char *sha2, + enum object_type type1, + enum object_type type2) { int cmp = type1 - type2; if (cmp) return cmp; - return memcmp(base1, base2, UNION_BASE_SZ); + return hashcmp(sha1, sha2); } -static int find_delta(const union delta_base *base, enum object_type type) +static int find_ref_delta(const unsigned char *sha1, enum object_type type) { - int first = 0, last = nr_deltas; - - while (first < last) { - int next = (first + last) / 2; - struct delta_entry *delta = &deltas[next]; - int cmp; - - cmp = compare_delta_bases(base, &delta->base, - type, objects[delta->obj_no].type); - if (!cmp) - return next; - if (cmp < 0) { - last = next; - continue; - } - first = next+1; - } - return -first-1; + int first = 0, last = nr_ref_deltas; + + while (first < last) { + int next = (first + last) / 2; + struct ref_delta_entry *delta = &ref_deltas[next]; + int cmp; + + cmp = compare_ref_delta_bases(sha1, delta->sha1, + type, objects[delta->obj_no].type); + if (!cmp) + return next; + if (cmp < 0) { + last = next; + continue; + } + first = next+1; + } + return -first-1; } -static void find_delta_children(const union delta_base *base, - int *first_index, int *last_index, - enum object_type type) +static void find_ref_delta_children(const unsigned char *sha1, + int *first_index, int *last_index, + enum object_type type) { - int first = find_delta(base, type); + int first = find_ref_delta(sha1, type); int last = first; - int end = nr_deltas - 1; + int end = nr_ref_deltas - 1; if (first < 0) { *first_index = 0; *last_index = -1; return; } - while (first > 0 && !memcmp(&deltas[first - 1].base, base, UNION_BASE_SZ)) + while (first > 0 && !hashcmp(ref_deltas[first - 1].sha1, sha1)) --first; - while (last < end && !memcmp(&deltas[last + 1].base, base, UNION_BASE_SZ)) + while (last < end && !hashcmp(ref_deltas[last + 1].sha1, sha1)) ++last; *first_index = first; *last_index = last; @@@ -786,7 -721,7 +786,7 @@@ static void sha1_object(const void *dat assert(data || obj_entry); read_lock(); - collision_test_needed = has_sha1_file(sha1); + collision_test_needed = has_sha1_file_with_flags(sha1, HAS_SHA1_QUICK); read_unlock(); if (collision_test_needed && !data) { @@@ -838,8 -773,7 +838,8 @@@ if (!obj) die(_("invalid %s"), typename(type)); if (do_fsck_object && - fsck_object(obj, 1, fsck_error_function)) + fsck_object(obj, buf, size, 1, + fsck_error_function)) die(_("Error in object")); if (fsck_walk(obj, mark_link, NULL)) die(_("Not all child objects of %s are reachable"), sha1_to_hex(obj->sha1)); @@@ -928,16 -862,15 +928,16 @@@ static void resolve_delta(struct object { void *base_data, *delta_data; - delta_obj->real_type = base->obj->real_type; if (show_stat) { - delta_obj->delta_depth = base->obj->delta_depth + 1; + int i = delta_obj - objects; + int j = base->obj - objects; + obj_stat[i].delta_depth = obj_stat[j].delta_depth + 1; deepest_delta_lock(); - if (deepest_delta < delta_obj->delta_depth) - deepest_delta = delta_obj->delta_depth; + if (deepest_delta < obj_stat[i].delta_depth) + deepest_delta = obj_stat[i].delta_depth; deepest_delta_unlock(); + obj_stat[i].base_object_no = j; } - delta_obj->base_object_no = base->obj - objects; delta_data = get_data_from_pack(delta_obj); base_data = get_base_data(base); result->obj = delta_obj; @@@ -955,37 -888,20 +955,37 @@@ counter_unlock(); } +/* + * Standard boolean compare-and-swap: atomically check whether "*type" is + * "want"; if so, swap in "set" and return true. Otherwise, leave it untouched + * and return false. + */ +static int compare_and_swap_type(signed char *type, + enum object_type want, + enum object_type set) +{ + enum object_type old; + + type_cas_lock(); + old = *type; + if (old == want) + *type = set; + type_cas_unlock(); + + return old == want; +} + static struct base_data *find_unresolved_deltas_1(struct base_data *base, struct base_data *prev_base) { if (base->ref_last == -1 && base->ofs_last == -1) { - union delta_base base_spec; - - hashcpy(base_spec.sha1, base->obj->idx.sha1); - find_delta_children(&base_spec, - &base->ref_first, &base->ref_last, OBJ_REF_DELTA); + find_ref_delta_children(base->obj->idx.sha1, + &base->ref_first, &base->ref_last, + OBJ_REF_DELTA); - memset(&base_spec, 0, sizeof(base_spec)); - base_spec.offset = base->obj->idx.offset; - find_delta_children(&base_spec, - &base->ofs_first, &base->ofs_last, OBJ_OFS_DELTA); + find_ofs_delta_children(base->obj->idx.offset, + &base->ofs_first, &base->ofs_last, + OBJ_OFS_DELTA); if (base->ref_last == -1 && base->ofs_last == -1) { free(base->data); @@@ -996,13 -912,10 +996,13 @@@ } if (base->ref_first <= base->ref_last) { - struct object_entry *child = objects + deltas[base->ref_first].obj_no; + struct object_entry *child = objects + ref_deltas[base->ref_first].obj_no; struct base_data *result = alloc_base_data(); - assert(child->real_type == OBJ_REF_DELTA); + if (!compare_and_swap_type(&child->real_type, OBJ_REF_DELTA, + base->obj->real_type)) + die("BUG: child->real_type != OBJ_REF_DELTA"); + resolve_delta(child, base, result); if (base->ref_first == base->ref_last && base->ofs_last == -1) free_base_data(base); @@@ -1012,11 -925,10 +1012,11 @@@ } if (base->ofs_first <= base->ofs_last) { - struct object_entry *child = objects + deltas[base->ofs_first].obj_no; + struct object_entry *child = objects + ofs_deltas[base->ofs_first].obj_no; struct base_data *result = alloc_base_data(); assert(child->real_type == OBJ_OFS_DELTA); + child->real_type = base->obj->real_type; resolve_delta(child, base, result); if (base->ofs_first == base->ofs_last) free_base_data(base); @@@ -1048,22 -960,15 +1048,22 @@@ static void find_unresolved_deltas(stru } } -static int compare_delta_entry(const void *a, const void *b) +static int compare_ofs_delta_entry(const void *a, const void *b) +{ + const struct ofs_delta_entry *delta_a = a; + const struct ofs_delta_entry *delta_b = b; + + return delta_a->offset < delta_b->offset ? -1 : + delta_a->offset > delta_b->offset ? 1 : + 0; +} + +static int compare_ref_delta_entry(const void *a, const void *b) { - const struct delta_entry *delta_a = a; - const struct delta_entry *delta_b = b; + const struct ref_delta_entry *delta_a = a; + const struct ref_delta_entry *delta_b = b; - /* group by type (ref vs ofs) and then by value (sha-1 or offset) */ - return compare_delta_bases(&delta_a->base, &delta_b->base, - objects[delta_a->obj_no].type, - objects[delta_b->obj_no].type); + return hashcmp(delta_a->sha1, delta_b->sha1); } static void resolve_base(struct object_entry *obj) @@@ -1109,8 -1014,7 +1109,8 @@@ static void *threaded_second_pass(void static void parse_pack_objects(unsigned char *sha1) { int i, nr_delays = 0; - struct delta_entry *delta = deltas; + struct ofs_delta_entry *ofs_delta = ofs_deltas; + unsigned char ref_delta_sha1[20]; struct stat st; if (verbose) @@@ -1119,18 -1023,12 +1119,18 @@@ nr_objects); for (i = 0; i < nr_objects; i++) { struct object_entry *obj = &objects[i]; - void *data = unpack_raw_entry(obj, &delta->base, obj->idx.sha1); + void *data = unpack_raw_entry(obj, &ofs_delta->offset, + ref_delta_sha1, obj->idx.sha1); obj->real_type = obj->type; - if (is_delta_type(obj->type)) { - nr_deltas++; - delta->obj_no = i; - delta++; + if (obj->type == OBJ_OFS_DELTA) { + nr_ofs_deltas++; + ofs_delta->obj_no = i; + ofs_delta++; + } else if (obj->type == OBJ_REF_DELTA) { + ALLOC_GROW(ref_deltas, nr_ref_deltas + 1, ref_deltas_alloc); + hashcpy(ref_deltas[nr_ref_deltas].sha1, ref_delta_sha1); + ref_deltas[nr_ref_deltas].obj_no = i; + nr_ref_deltas++; } else if (!data) { /* large blobs, check later */ obj->real_type = OBJ_BAD; @@@ -1181,18 -1079,15 +1181,18 @@@ static void resolve_deltas(void { int i; - if (!nr_deltas) + if (!nr_ofs_deltas && !nr_ref_deltas) return; /* Sort deltas by base SHA1/offset for fast searching */ - qsort(deltas, nr_deltas, sizeof(struct delta_entry), - compare_delta_entry); + qsort(ofs_deltas, nr_ofs_deltas, sizeof(struct ofs_delta_entry), + compare_ofs_delta_entry); + qsort(ref_deltas, nr_ref_deltas, sizeof(struct ref_delta_entry), + compare_ref_delta_entry); if (verbose) - progress = start_progress(_("Resolving deltas"), nr_deltas); + progress = start_progress(_("Resolving deltas"), + nr_ref_deltas + nr_ofs_deltas); #ifndef NO_PTHREADS nr_dispatched = 0; @@@ -1230,7 -1125,7 +1230,7 @@@ static void fix_unresolved_deltas(struct sha1file *f, int nr_unresolved); static void conclude_pack(int fix_thin_pack, const char *curr_pack, unsigned char *pack_sha1) { - if (nr_deltas == nr_resolved_deltas) { + if (nr_ref_deltas + nr_ofs_deltas == nr_resolved_deltas) { stop_progress(&progress); /* Flush remaining pack final 20-byte SHA1. */ flush(); @@@ -1241,11 -1136,13 +1241,11 @@@ struct sha1file *f; unsigned char read_sha1[20], tail_sha1[20]; struct strbuf msg = STRBUF_INIT; - int nr_unresolved = nr_deltas - nr_resolved_deltas; + int nr_unresolved = nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas; int nr_objects_initial = nr_objects; if (nr_unresolved <= 0) die(_("confusion beyond insanity")); - objects = xrealloc(objects, - (nr_objects + nr_unresolved + 1) - * sizeof(*objects)); + REALLOC_ARRAY(objects, nr_objects + nr_unresolved + 1); memset(objects + nr_objects + 1, 0, nr_unresolved * sizeof(*objects)); f = sha1fd(output_fd, curr_pack); @@@ -1263,11 -1160,11 +1263,11 @@@ die(_("Unexpected tail checksum for %s " "(disk corruption?)"), curr_pack); } - if (nr_deltas != nr_resolved_deltas) + if (nr_ofs_deltas + nr_ref_deltas != nr_resolved_deltas) die(Q_("pack has %d unresolved delta", "pack has %d unresolved deltas", - nr_deltas - nr_resolved_deltas), - nr_deltas - nr_resolved_deltas); + nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas), + nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas); } static int write_compressed(struct sha1file *f, void *in, unsigned int size) @@@ -1276,6 -1173,7 +1276,6 @@@ int status; unsigned char outbuf[4096]; - memset(&stream, 0, sizeof(stream)); git_deflate_init(&stream, zlib_compression_level); stream.next_in = in; stream.avail_in = size; @@@ -1326,14 -1224,14 +1326,14 @@@ static struct object_entry *append_obj_ static int delta_pos_compare(const void *_a, const void *_b) { - struct delta_entry *a = *(struct delta_entry **)_a; - struct delta_entry *b = *(struct delta_entry **)_b; + struct ref_delta_entry *a = *(struct ref_delta_entry **)_a; + struct ref_delta_entry *b = *(struct ref_delta_entry **)_b; return a->obj_no - b->obj_no; } static void fix_unresolved_deltas(struct sha1file *f, int nr_unresolved) { - struct delta_entry **sorted_by_pos; + struct ref_delta_entry **sorted_by_pos; int i, n = 0; /* @@@ -1347,25 -1245,28 +1347,25 @@@ * resolving deltas in the same order as their position in the pack. */ sorted_by_pos = xmalloc(nr_unresolved * sizeof(*sorted_by_pos)); - for (i = 0; i < nr_deltas; i++) { - if (objects[deltas[i].obj_no].real_type != OBJ_REF_DELTA) - continue; - sorted_by_pos[n++] = &deltas[i]; - } + for (i = 0; i < nr_ref_deltas; i++) + sorted_by_pos[n++] = &ref_deltas[i]; qsort(sorted_by_pos, n, sizeof(*sorted_by_pos), delta_pos_compare); for (i = 0; i < n; i++) { - struct delta_entry *d = sorted_by_pos[i]; + struct ref_delta_entry *d = sorted_by_pos[i]; enum object_type type; struct base_data *base_obj = alloc_base_data(); if (objects[d->obj_no].real_type != OBJ_REF_DELTA) continue; - base_obj->data = read_sha1_file(d->base.sha1, &type, &base_obj->size); + base_obj->data = read_sha1_file(d->sha1, &type, &base_obj->size); if (!base_obj->data) continue; - if (check_sha1_signature(d->base.sha1, base_obj->data, + if (check_sha1_signature(d->sha1, base_obj->data, base_obj->size, typename(type))) - die(_("local object %s is corrupt"), sha1_to_hex(d->base.sha1)); - base_obj->obj = append_obj_to_pack(f, d->base.sha1, + die(_("local object %s is corrupt"), sha1_to_hex(d->sha1)); + base_obj->obj = append_obj_to_pack(f, d->sha1, base_obj->data, base_obj->size, type); find_unresolved_deltas(base_obj); display_progress(progress, nr_resolved_deltas); @@@ -1557,7 -1458,7 +1557,7 @@@ static void read_idx_option(struct pack static void show_pack_info(int stat_only) { - int i, baseobjects = nr_objects - nr_deltas; + int i, baseobjects = nr_objects - nr_ref_deltas - nr_ofs_deltas; unsigned long *chain_histogram = NULL; if (deepest_delta) @@@ -1567,7 -1468,7 +1567,7 @@@ struct object_entry *obj = &objects[i]; if (is_delta_type(obj->type)) - chain_histogram[obj->delta_depth - 1]++; + chain_histogram[obj_stat[i].delta_depth - 1]++; if (stat_only) continue; printf("%s %-6s %lu %lu %"PRIuMAX, @@@ -1576,8 -1477,8 +1576,8 @@@ (unsigned long)(obj[1].idx.offset - obj->idx.offset), (uintmax_t)obj->idx.offset); if (is_delta_type(obj->type)) { - struct object_entry *bobj = &objects[obj->base_object_no]; - printf(" %u %s", obj->delta_depth, sha1_to_hex(bobj->idx.sha1)); + struct object_entry *bobj = &objects[obj_stat[i].base_object_no]; + printf(" %u %s", obj_stat[i].delta_depth, sha1_to_hex(bobj->idx.sha1)); } putchar('\n'); } @@@ -1604,8 -1505,7 +1604,8 @@@ int cmd_index_pack(int argc, const cha const char *curr_index; const char *index_name = NULL, *pack_name = NULL; const char *keep_name = NULL, *keep_msg = NULL; - char *index_name_buf = NULL, *keep_name_buf = NULL; + struct strbuf index_name_buf = STRBUF_INIT, + keep_name_buf = STRBUF_INIT; struct pack_idx_entry **idx_objects; struct pack_idx_option opts; unsigned char pack_sha1[20]; @@@ -1702,22 -1602,24 +1702,22 @@@ if (fix_thin_pack && !from_stdin) die(_("--fix-thin cannot be used without --stdin")); if (!index_name && pack_name) { - int len = strlen(pack_name); - if (!has_extension(pack_name, ".pack")) + size_t len; + if (!strip_suffix(pack_name, ".pack", &len)) die(_("packfile name '%s' does not end with '.pack'"), pack_name); - index_name_buf = xmalloc(len); - memcpy(index_name_buf, pack_name, len - 5); - strcpy(index_name_buf + len - 5, ".idx"); - index_name = index_name_buf; + strbuf_add(&index_name_buf, pack_name, len); + strbuf_addstr(&index_name_buf, ".idx"); + index_name = index_name_buf.buf; } if (keep_msg && !keep_name && pack_name) { - int len = strlen(pack_name); - if (!has_extension(pack_name, ".pack")) + size_t len; + if (!strip_suffix(pack_name, ".pack", &len)) die(_("packfile name '%s' does not end with '.pack'"), pack_name); - keep_name_buf = xmalloc(len); - memcpy(keep_name_buf, pack_name, len - 5); - strcpy(keep_name_buf + len - 5, ".keep"); - keep_name = keep_name_buf; + strbuf_add(&keep_name_buf, pack_name, len); + strbuf_addstr(&keep_name_buf, ".idx"); + keep_name = keep_name_buf.buf; } if (verify) { if (!index_name) @@@ -1740,14 -1642,11 +1740,14 @@@ curr_pack = open_pack_file(pack_name); parse_pack_header(); objects = xcalloc(nr_objects + 1, sizeof(struct object_entry)); - deltas = xcalloc(nr_objects, sizeof(struct delta_entry)); + if (show_stat) + obj_stat = xcalloc(nr_objects + 1, sizeof(struct object_stat)); + ofs_deltas = xcalloc(nr_objects, sizeof(struct ofs_delta_entry)); parse_pack_objects(pack_sha1); resolve_deltas(); conclude_pack(fix_thin_pack, curr_pack, pack_sha1); - free(deltas); + free(ofs_deltas); + free(ref_deltas); if (strict) foreign_nr = check_objects(); @@@ -1768,8 -1667,8 +1768,8 @@@ else close(input_fd); free(objects); - free(index_name_buf); - free(keep_name_buf); + strbuf_release(&index_name_buf); + strbuf_release(&keep_name_buf); if (pack_name == NULL) free((void *) curr_pack); if (index_name == NULL) diff --combined cache.h index fe840e34a5,3fb7570671..4f554664c5 --- a/cache.h +++ b/cache.h @@@ -7,8 -7,6 +7,8 @@@ #include "advice.h" #include "gettext.h" #include "convert.h" +#include "trace.h" +#include "string-list.h" #include SHA1_HEADER #ifndef git_SHA_CTX @@@ -43,14 -41,6 +43,14 @@@ int git_deflate_end_gently(git_zstream int git_deflate(git_zstream *, int flush); unsigned long git_deflate_bound(git_zstream *, unsigned long); +/* The length in bytes and in hex digits of an object name (SHA-1 value). */ +#define GIT_SHA1_RAWSZ 20 +#define GIT_SHA1_HEXSZ (2 * GIT_SHA1_RAWSZ) + +struct object_id { + unsigned char hash[GIT_SHA1_RAWSZ]; +}; + #if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT) #define DTYPE(de) ((de)->d_type) #else @@@ -73,25 -63,17 +73,25 @@@ * * The value 0160000 is not normally a valid mode, and * also just happens to be S_IFDIR + S_IFLNK - * - * NOTE! We *really* shouldn't depend on the S_IFxxx macros - * always having the same values everywhere. We should use - * our internal git values for these things, and then we can - * translate that to the OS-specific value. It just so - * happens that everybody shares the same bit representation - * in the UNIX world (and apparently wider too..) */ #define S_IFGITLINK 0160000 #define S_ISGITLINK(m) (((m) & S_IFMT) == S_IFGITLINK) +/* + * Some mode bits are also used internally for computations. + * + * They *must* not overlap with any valid modes, and they *must* not be emitted + * to outside world - i.e. appear on disk or network. In other words, it's just + * temporary fields, which we internally use, but they have to stay in-house. + * + * ( such approach is valid, as standard S_IF* fits into 16 bits, and in Git + * codebase mode is `unsigned int` which is assumed to be at least 32 bits ) + */ + +/* used internally in tree-diff */ +#define S_DIFFTREE_IFXMIN_NEQ 0x80000000 + + /* * Intensive research over the course of many years has shown that * port 9418 is totally unused by anything else. Or @@@ -153,7 -135,6 +153,7 @@@ struct cache_entry unsigned int ce_mode; unsigned int ce_flags; unsigned int ce_namelen; + unsigned int index; /* for link extension */ unsigned char sha1[20]; char name[FLEX_ARRAY]; /* more */ }; @@@ -164,7 -145,7 +164,7 @@@ #define CE_STAGESHIFT 12 /* - * Range 0xFFFF0000 in ce_flags is divided into + * Range 0xFFFF0FFF in ce_flags is divided into * two parts: in-memory flags and on-disk ones. * Flags in CE_EXTENDED_FLAGS will get saved on-disk * if you want to save a new flag, add it in @@@ -187,9 -168,6 +187,9 @@@ /* used to temporarily mark paths matched by pathspecs */ #define CE_MATCHED (1 << 26) +#define CE_UPDATE_IN_BASE (1 << 27) +#define CE_STRIP_NAME (1 << 28) + /* * Extended on-disk flags */ @@@ -290,32 -268,18 +290,32 @@@ static inline unsigned int canon_mode(u #define cache_entry_size(len) (offsetof(struct cache_entry,name) + (len) + 1) +#define SOMETHING_CHANGED (1 << 0) /* unclassified changes go here */ +#define CE_ENTRY_CHANGED (1 << 1) +#define CE_ENTRY_REMOVED (1 << 2) +#define CE_ENTRY_ADDED (1 << 3) +#define RESOLVE_UNDO_CHANGED (1 << 4) +#define CACHE_TREE_CHANGED (1 << 5) +#define SPLIT_INDEX_ORDERED (1 << 6) +#define UNTRACKED_CHANGED (1 << 7) + +struct split_index; +struct untracked_cache; + struct index_state { struct cache_entry **cache; unsigned int version; unsigned int cache_nr, cache_alloc, cache_changed; struct string_list *resolve_undo; struct cache_tree *cache_tree; + struct split_index *split_index; struct cache_time timestamp; unsigned name_hash_initialized : 1, initialized : 1; struct hashmap name_hash; struct hashmap dir_hash; unsigned char sha1[20]; + struct untracked_cache *untracked; }; extern struct index_state the_index; @@@ -338,6 -302,7 +338,6 @@@ extern void free_name_hash(struct index #define read_cache_preload(pathspec) read_index_preload(&the_index, (pathspec)) #define is_cache_unborn() is_index_unborn(&the_index) #define read_cache_unmerged() read_index_unmerged(&the_index) -#define write_cache(newfd, cache, entries) write_index(&the_index, (newfd)) #define discard_cache() discard_index(&the_index) #define unmerged_cache() unmerged_index(&the_index) #define cache_name_pos(name, namelen) index_name_pos(&the_index,(name),(namelen)) @@@ -382,7 -347,6 +382,7 @@@ static inline enum object_type object_t /* Double-check local_repo_env below if you add to this list. */ #define GIT_DIR_ENVIRONMENT "GIT_DIR" +#define GIT_COMMON_DIR_ENVIRONMENT "GIT_COMMON_DIR" #define GIT_NAMESPACE_ENVIRONMENT "GIT_NAMESPACE" #define GIT_WORK_TREE_ENVIRONMENT "GIT_WORK_TREE" #define GIT_PREFIX_ENVIRONMENT "GIT_PREFIX" @@@ -436,13 -400,11 +436,13 @@@ extern int is_inside_git_dir(void) extern char *git_work_tree_cfg; extern int is_inside_work_tree(void); extern const char *get_git_dir(void); +extern const char *get_git_common_dir(void); extern int is_git_directory(const char *path); extern char *get_object_directory(void); extern char *get_index_file(void); extern char *get_graft_file(void); extern int set_git_dir(const char *path); +extern int get_common_dir(struct strbuf *sb, const char *gitdir); extern const char *get_git_namespace(void); extern const char *strip_namespace(const char *namespaced_ref); extern const char *get_git_work_tree(void); @@@ -490,22 -452,17 +490,22 @@@ extern int daemonize(void) alloc = (nr); \ else \ alloc = alloc_nr(alloc); \ - x = xrealloc((x), alloc * sizeof(*(x))); \ + REALLOC_ARRAY(x, alloc); \ } \ } while (0) /* Initialize and use the cache information */ +struct lock_file; extern int read_index(struct index_state *); extern int read_index_preload(struct index_state *, const struct pathspec *pathspec); +extern int do_read_index(struct index_state *istate, const char *path, + int must_exist); /* for testting only! */ extern int read_index_from(struct index_state *, const char *path); extern int is_index_unborn(struct index_state *); extern int read_index_unmerged(struct index_state *); -extern int write_index(struct index_state *, int newfd); +#define COMMIT_LOCK (1 << 0) +#define CLOSE_LOCK (1 << 1) +extern int write_locked_index(struct index_state *, struct lock_file *lock, unsigned flags); extern int discard_index(struct index_state *); extern int unmerged_index(const struct index_state *); extern int verify_path(const char *path); @@@ -517,7 -474,6 +517,7 @@@ extern int index_name_pos(const struct #define ADD_CACHE_SKIP_DFCHECK 4 /* Ok to skip DF conflict checks */ #define ADD_CACHE_JUST_APPEND 8 /* Append only; tree.c::read_tree() */ #define ADD_CACHE_NEW_ONLY 16 /* Do not replace existing ones */ +#define ADD_CACHE_KEEP_CACHE_TREE 32 /* Do not invalidate cache-tree */ extern int add_index_entry(struct index_state *, struct cache_entry *ce, int option); extern void rename_index_entry_at(struct index_state *, int pos, const char *new_name); extern int remove_index_entry_at(struct index_state *, int pos); @@@ -567,8 -523,6 +567,8 @@@ extern void fill_stat_data(struct stat_ * INODE_CHANGED, and DATA_CHANGED. */ extern int match_stat_data(const struct stat_data *sd, struct stat *st); +extern int match_stat_data_racy(const struct index_state *istate, + const struct stat_data *sd, struct stat *st); extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st); @@@ -580,12 -534,28 +580,12 @@@ #define REFRESH_IN_PORCELAIN 0x0020 /* user friendly output, not "needs update" */ extern int refresh_index(struct index_state *, unsigned int flags, const struct pathspec *pathspec, char *seen, const char *header_msg); -struct lock_file { - struct lock_file *next; - int fd; - pid_t owner; - char on_list; - char filename[PATH_MAX]; -}; -#define LOCK_DIE_ON_ERROR 1 -#define LOCK_NODEREF 2 -extern int unable_to_lock_error(const char *path, int err); -extern NORETURN void unable_to_lock_index_die(const char *path, int err); -extern int hold_lock_file_for_update(struct lock_file *, const char *path, int); -extern int hold_lock_file_for_append(struct lock_file *, const char *path, int); -extern int commit_lock_file(struct lock_file *); extern void update_index_if_able(struct index_state *, struct lock_file *); extern int hold_locked_index(struct lock_file *, int); -extern int commit_locked_index(struct lock_file *); extern void set_alternate_index_output(const char *); -extern int close_lock_file(struct lock_file *); -extern void rollback_lock_file(struct lock_file *); -extern int delete_ref(const char *, const unsigned char *sha1, int delopt); + +extern int delete_ref(const char *, const unsigned char *sha1, unsigned int flags); /* Environment bits from configuration mechanism */ extern int trust_executable_bit; @@@ -629,22 -599,12 +629,22 @@@ extern int core_apply_sparse_checkout extern int precomposed_unicode; extern int protect_hfs; extern int protect_ntfs; +extern int git_db_env, git_index_env, git_graft_env, git_common_dir_env; + +/* + * Include broken refs in all ref iterations, which will + * generally choke dangerous operations rather than letting + * them silently proceed without taking the broken ref into + * account. + */ +extern int ref_paranoia; /* * The character that begins a commented line in user-editable file * that is subject to stripspace. */ extern char comment_line_char; +extern int auto_comment_line_char; enum branch_track { BRANCH_TRACK_UNSPECIFIED = -1, @@@ -700,19 -660,18 +700,19 @@@ extern int check_repository_format(void extern char *mksnpath(char *buf, size_t n, const char *fmt, ...) __attribute__((format (printf, 3, 4))); -extern char *git_snpath(char *buf, size_t n, const char *fmt, ...) - __attribute__((format (printf, 3, 4))); +extern void strbuf_git_path(struct strbuf *sb, const char *fmt, ...) + __attribute__((format (printf, 2, 3))); extern char *git_pathdup(const char *fmt, ...) __attribute__((format (printf, 1, 2))); extern char *mkpathdup(const char *fmt, ...) __attribute__((format (printf, 1, 2))); /* Return a statically allocated filename matching the sha1 signature */ -extern char *mkpath(const char *fmt, ...) __attribute__((format (printf, 1, 2))); -extern char *git_path(const char *fmt, ...) __attribute__((format (printf, 1, 2))); -extern char *git_path_submodule(const char *path, const char *fmt, ...) +extern const char *mkpath(const char *fmt, ...) __attribute__((format (printf, 1, 2))); +extern const char *git_path(const char *fmt, ...) __attribute__((format (printf, 1, 2))); +extern const char *git_path_submodule(const char *path, const char *fmt, ...) __attribute__((format (printf, 2, 3))); +extern void report_linked_checkout_garbage(void); /* * Return the name of the file in the local object database that would @@@ -737,13 -696,13 +737,13 @@@ extern char *sha1_pack_name(const unsig extern char *sha1_pack_index_name(const unsigned char *sha1); extern const char *find_unique_abbrev(const unsigned char *sha1, int); -extern const unsigned char null_sha1[20]; +extern const unsigned char null_sha1[GIT_SHA1_RAWSZ]; static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2) { int i; - for (i = 0; i < 20; i++, sha1++, sha2++) { + for (i = 0; i < GIT_SHA1_RAWSZ; i++, sha1++, sha2++) { if (*sha1 != *sha2) return *sha1 - *sha2; } @@@ -751,42 -710,20 +751,42 @@@ return 0; } +static inline int oidcmp(const struct object_id *oid1, const struct object_id *oid2) +{ + return hashcmp(oid1->hash, oid2->hash); +} + static inline int is_null_sha1(const unsigned char *sha1) { return !hashcmp(sha1, null_sha1); } +static inline int is_null_oid(const struct object_id *oid) +{ + return !hashcmp(oid->hash, null_sha1); +} + static inline void hashcpy(unsigned char *sha_dst, const unsigned char *sha_src) { - memcpy(sha_dst, sha_src, 20); + memcpy(sha_dst, sha_src, GIT_SHA1_RAWSZ); } + +static inline void oidcpy(struct object_id *dst, const struct object_id *src) +{ + hashcpy(dst->hash, src->hash); +} + static inline void hashclr(unsigned char *hash) { - memset(hash, 0, 20); + memset(hash, 0, GIT_SHA1_RAWSZ); +} + +static inline void oidclr(struct object_id *oid) +{ + hashclr(oid->hash); } + #define EMPTY_TREE_SHA1_HEX \ "4b825dc642cb6eb9a060e54bf8d69288fbee4904" #define EMPTY_TREE_SHA1_BIN_LITERAL \ @@@ -857,6 -794,7 +857,6 @@@ enum scld_error safe_create_leading_dir enum scld_error safe_create_leading_directories_const(const char *path); int mkdir_in_gitdir(const char *path); -extern void home_config_paths(char **global, char **xdg, char *file); extern char *expand_user_path(const char *path); const char *enter_repo(const char *path, int strict); static inline int is_absolute_path(const char *path) @@@ -874,18 -812,11 +874,18 @@@ int normalize_path_copy(char *dst, cons int longest_ancestor_length(const char *path, struct string_list *prefixes); char *strip_path_suffix(const char *path, const char *suffix); int daemon_avoid_alias(const char *path); -int offset_1st_component(const char *path); extern int is_ntfs_dotgit(const char *name); +/** + * Return a newly allocated string with the evaluation of + * "$XDG_CONFIG_HOME/git/$filename" if $XDG_CONFIG_HOME is non-empty, otherwise + * "$HOME/.config/git/$filename". Return NULL upon error. + */ +extern char *xdg_config_home(const char *filename); + /* object replacement */ #define LOOKUP_REPLACE_OBJECT 1 +#define LOOKUP_UNKNOWN_OBJECT 2 extern void *read_sha1_file_extended(const unsigned char *sha1, enum object_type *type, unsigned long *size, unsigned flag); static inline void *read_sha1_file(const unsigned char *sha1, enum object_type *type, unsigned long *size) { @@@ -922,7 -853,6 +922,7 @@@ static inline const unsigned char *look extern int sha1_object_info(const unsigned char *, unsigned long *); extern int hash_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1); extern int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *return_sha1); +extern int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type, unsigned char *sha1, unsigned flags); extern int pretend_sha1_file(void *, unsigned long, enum object_type, unsigned char *); extern int force_object_loose(const unsigned char *sha1, time_t mtime); extern int git_open_noatime(const char *name); @@@ -943,8 -873,17 +943,17 @@@ extern int has_sha1_pack(const unsigne * Return true iff we have an object named sha1, whether local or in * an alternate object database, and whether packed or loose. This * function does not respect replace references. + * + * If the QUICK flag is set, do not re-check the pack directory + * when we cannot find the object (this means we may give a false + * negative answer if another process is simultaneously repacking). */ - extern int has_sha1_file(const unsigned char *sha1); + #define HAS_SHA1_QUICK 0x1 + extern int has_sha1_file_with_flags(const unsigned char *sha1, int flags); + static inline int has_sha1_file(const unsigned char *sha1) + { + return has_sha1_file_with_flags(sha1, 0); + } /* * Return true iff an alternate object database has a loose object @@@ -971,21 -910,15 +980,21 @@@ struct object_context unsigned char tree[20]; char path[PATH_MAX]; unsigned mode; + /* + * symlink_path is only used by get_tree_entry_follow_symlinks, + * and only for symlinks that point outside the repository. + */ + struct strbuf symlink_path; }; -#define GET_SHA1_QUIETLY 01 -#define GET_SHA1_COMMIT 02 -#define GET_SHA1_COMMITTISH 04 -#define GET_SHA1_TREE 010 -#define GET_SHA1_TREEISH 020 -#define GET_SHA1_BLOB 040 -#define GET_SHA1_ONLY_TO_DIE 04000 +#define GET_SHA1_QUIETLY 01 +#define GET_SHA1_COMMIT 02 +#define GET_SHA1_COMMITTISH 04 +#define GET_SHA1_TREE 010 +#define GET_SHA1_TREEISH 020 +#define GET_SHA1_BLOB 040 +#define GET_SHA1_FOLLOW_SYMLINKS 0100 +#define GET_SHA1_ONLY_TO_DIE 04000 extern int get_sha1(const char *str, unsigned char *sha1); extern int get_sha1_commit(const char *str, unsigned char *sha1); @@@ -1007,12 -940,10 +1016,12 @@@ extern int for_each_abbrev(const char * * null-terminated string. */ extern int get_sha1_hex(const char *hex, unsigned char *sha1); +extern int get_oid_hex(const char *hex, struct object_id *sha1); extern char *sha1_to_hex(const unsigned char *sha1); /* static buffer result! */ -extern int read_ref_full(const char *refname, unsigned char *sha1, - int reading, int *flags); +extern char *oid_to_hex(const struct object_id *oid); /* same static buffer as sha1_to_hex */ +extern int read_ref_full(const char *refname, int resolve_flags, + unsigned char *sha1, int *flags); extern int read_ref(const char *refname, unsigned char *sha1); /* @@@ -1024,49 -955,29 +1033,49 @@@ * or the input ref. * * If the reference cannot be resolved to an object, the behavior - * depends on the "reading" argument: + * depends on the RESOLVE_REF_READING flag: * - * - If reading is set, return NULL. + * - If RESOLVE_REF_READING is set, return NULL. * - * - If reading is not set, clear sha1 and return the name of the last - * reference name in the chain, which will either be a non-symbolic + * - If RESOLVE_REF_READING is not set, clear sha1 and return the name of + * the last reference name in the chain, which will either be a non-symbolic * reference or an undefined reference. If this is a prelude to * "writing" to the ref, the return value is the name of the ref * that will actually be created or changed. * - * If flag is non-NULL, set the value that it points to the + * If the RESOLVE_REF_NO_RECURSE flag is passed, only resolves one + * level of symbolic reference. The value stored in sha1 for a symbolic + * reference will always be null_sha1 in this case, and the return + * value is the reference that the symref refers to directly. + * + * If flags is non-NULL, set the value that it points to the * combination of REF_ISPACKED (if the reference was found among the - * packed references) and REF_ISSYMREF (if the initial reference was a - * symbolic reference). + * packed references), REF_ISSYMREF (if the initial reference was a + * symbolic reference), REF_BAD_NAME (if the reference name is ill + * formed --- see RESOLVE_REF_ALLOW_BAD_NAME below), and REF_ISBROKEN + * (if the ref is malformed or has a bad name). See refs.h for more detail + * on each flag. * * If ref is not a properly-formatted, normalized reference, return * NULL. If more than MAXDEPTH recursive symbolic lookups are needed, * give up and return NULL. * - * errno is sometimes set on errors, but not always. + * RESOLVE_REF_ALLOW_BAD_NAME allows resolving refs even when their + * name is invalid according to git-check-ref-format(1). If the name + * is bad then the value stored in sha1 will be null_sha1 and the two + * flags REF_ISBROKEN and REF_BAD_NAME will be set. + * + * Even with RESOLVE_REF_ALLOW_BAD_NAME, names that escape the refs/ + * directory and do not consist of all caps and underscores cannot be + * resolved. The function returns NULL for such ref names. + * Caps and underscores refers to the special refs, such as HEAD, + * FETCH_HEAD and friends, that all live outside of the refs/ directory. */ -extern const char *resolve_ref_unsafe(const char *ref, unsigned char *sha1, int reading, int *flag); -extern char *resolve_refdup(const char *ref, unsigned char *sha1, int reading, int *flag); +#define RESOLVE_REF_READING 0x01 +#define RESOLVE_REF_NO_RECURSE 0x02 +#define RESOLVE_REF_ALLOW_BAD_NAME 0x04 +extern const char *resolve_ref_unsafe(const char *ref, int resolve_flags, unsigned char *sha1, int *flags); +extern char *resolve_refdup(const char *ref, int resolve_flags, unsigned char *sha1, int *flags); extern int dwim_ref(const char *str, int len, unsigned char *sha1, char **ref); extern int dwim_log(const char *str, int len, unsigned char *sha1, char **ref); @@@ -1085,7 -996,7 +1094,7 @@@ extern int validate_headref(const char extern int base_name_compare(const char *name1, int len1, int mode1, const char *name2, int len2, int mode2); extern int df_name_compare(const char *name1, int len1, int mode1, const char *name2, int len2, int mode2); -extern int cache_name_compare(const char *name1, int len1, const char *name2, int len2); +extern int name_compare(const char *name1, size_t len1, const char *name2, size_t len2); extern int cache_name_stage_compare(const char *name1, int len1, int stage1, const char *name2, int len2, int stage2); extern void *read_object_with_reference(const unsigned char *sha1, @@@ -1102,7 -1013,6 +1111,7 @@@ enum date_mode DATE_SHORT, DATE_LOCAL, DATE_ISO8601, + DATE_ISO8601_STRICT, DATE_RFC2822, DATE_RAW }; @@@ -1110,10 -1020,10 +1119,10 @@@ const char *show_date(unsigned long time, int timezone, enum date_mode mode); void show_date_relative(unsigned long time, int tz, const struct timeval *now, struct strbuf *timebuf); -int parse_date(const char *date, char *buf, int bufsize); +int parse_date(const char *date, struct strbuf *out); int parse_date_basic(const char *date, unsigned long *timestamp, int *offset); int parse_expiry_date(const char *date, unsigned long *timestamp); -void datestamp(char *buf, int bufsize); +void datestamp(struct strbuf *out); #define approxidate(s) approxidate_careful((s), NULL) unsigned long approxidate_careful(const char *, int *); unsigned long approxidate_relative(const char *date, const struct timeval *now); @@@ -1127,7 -1037,6 +1136,7 @@@ extern const char *git_author_info(int) extern const char *git_committer_info(int); extern const char *fmt_ident(const char *name, const char *email, const char *date_str, int); extern const char *fmt_name(const char *name, const char *email); +extern const char *ident_default_name(void); extern const char *ident_default_email(void); extern const char *git_editor(void); extern const char *git_pager(int stdout_is_tty); @@@ -1149,13 -1058,6 +1158,13 @@@ struct ident_split */ extern int split_ident_line(struct ident_split *, const char *, int); +/* + * Like show_date, but pull the timestamp and tz parameters from + * the ident_split. It will also sanity-check the values and produce + * a well-known sentinel date if they appear bogus. + */ +const char *show_ident_date(const struct ident_split *id, enum date_mode mode); + /* * Compare split idents for equality or strict ordering. Note that we * compare only the ident part of the line, ignoring any timestamp. @@@ -1166,7 -1068,6 +1175,7 @@@ extern int ident_cmp(const struct ident_split *, const struct ident_split *); struct checkout { + struct index_state *istate; const char *base_dir; int base_dir_len; unsigned force:1, @@@ -1179,16 -1080,12 +1188,16 @@@ extern int checkout_entry(struct cache_entry *ce, const struct checkout *state, char *topath); struct cache_def { - char path[PATH_MAX + 1]; - int len; + struct strbuf path; int flags; int track_flags; int prefix_len_stat_func; }; +#define CACHE_DEF_INIT { STRBUF_INIT, 0, 0, 0 } +static inline void cache_def_clear(struct cache_def *cache) +{ + strbuf_release(&cache->path); +} extern int has_symlink_leading_path(const char *name, int len); extern int threaded_has_symlink_leading_path(struct cache_def *, const char *, int); @@@ -1206,7 -1103,7 +1215,7 @@@ extern void prepare_alt_odb(void) extern void read_info_alternates(const char * relative_base, int depth); extern void add_to_alternates_file(const char *reference); typedef int alt_odb_fn(struct alternate_object_database *, void *); -extern void foreach_alt_odb(alt_odb_fn, void*); +extern int foreach_alt_odb(alt_odb_fn, void*); struct pack_window { struct pack_window *next; @@@ -1231,7 -1128,6 +1240,7 @@@ extern struct packed_git int pack_fd; unsigned pack_local:1, pack_keep:1, + freshened:1, do_not_close:1; unsigned char sha1[20]; /* something like ".git/objects/pack/xxxxx.pack" */ @@@ -1303,68 -1199,12 +1312,68 @@@ extern unsigned long unpack_object_head extern unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t); extern int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, unsigned long *); +/* + * Iterate over the files in the loose-object parts of the object + * directory "path", triggering the following callbacks: + * + * - loose_object is called for each loose object we find. + * + * - loose_cruft is called for any files that do not appear to be + * loose objects. Note that we only look in the loose object + * directories "objects/[0-9a-f]{2}/", so we will not report + * "objects/foobar" as cruft. + * + * - loose_subdir is called for each top-level hashed subdirectory + * of the object directory (e.g., "$OBJDIR/f0"). It is called + * after the objects in the directory are processed. + * + * Any callback that is NULL will be ignored. Callbacks returning non-zero + * will end the iteration. + * + * In the "buf" variant, "path" is a strbuf which will also be used as a + * scratch buffer, but restored to its original contents before + * the function returns. + */ +typedef int each_loose_object_fn(const unsigned char *sha1, + const char *path, + void *data); +typedef int each_loose_cruft_fn(const char *basename, + const char *path, + void *data); +typedef int each_loose_subdir_fn(int nr, + const char *path, + void *data); +int for_each_loose_file_in_objdir(const char *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data); +int for_each_loose_file_in_objdir_buf(struct strbuf *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data); + +/* + * Iterate over loose and packed objects in both the local + * repository and any alternates repositories (unless the + * LOCAL_ONLY flag is set). + */ +#define FOR_EACH_OBJECT_LOCAL_ONLY 0x1 +typedef int each_packed_object_fn(const unsigned char *sha1, + struct packed_git *pack, + uint32_t pos, + void *data); +extern int for_each_loose_object(each_loose_object_fn, void *, unsigned flags); +extern int for_each_packed_object(each_packed_object_fn, void *, unsigned flags); + struct object_info { /* Request */ enum object_type *typep; unsigned long *sizep; unsigned long *disk_sizep; unsigned char *delta_base_sha1; + struct strbuf *typename; /* Response */ enum { @@@ -1405,8 -1245,6 +1414,8 @@@ extern int update_server_info(int) #define CONFIG_INVALID_PATTERN 6 #define CONFIG_GENERIC_ERROR 7 +#define CONFIG_REGEX_NONE ((void *)1) + struct git_config_source { unsigned int use_stdin:1; const char *file; @@@ -1420,7 -1258,7 +1429,7 @@@ extern int git_config_from_buf(config_f const char *buf, size_t len, void *data); extern void git_config_push_parameter(const char *text); extern int git_config_from_parameters(config_fn_t fn, void *data); -extern int git_config(config_fn_t fn, void *); +extern void git_config(config_fn_t fn, void *); extern int git_config_with_options(config_fn_t fn, void *, struct git_config_source *config_source, int respect_includes); @@@ -1444,11 -1282,10 +1453,11 @@@ extern int git_config_rename_section_in extern const char *git_etc_gitconfig(void); extern int check_repository_format_version(const char *var, const char *value, void *cb); extern int git_env_bool(const char *, int); +extern unsigned long git_env_ulong(const char *, unsigned long); extern int git_config_system(void); extern int config_error_nonbool(const char *); -#if defined(__GNUC__) && ! defined(__clang__) -#define config_error_nonbool(s) (config_error_nonbool(s), -1) +#if defined(__GNUC__) +#define config_error_nonbool(s) (config_error_nonbool(s), const_error()) #endif extern const char *get_log_output_encoding(void); extern const char *get_commit_output_encoding(void); @@@ -1478,69 -1315,6 +1487,69 @@@ extern int parse_config_key(const char const char **subsection, int *subsection_len, const char **key); +struct config_set_element { + struct hashmap_entry ent; + char *key; + struct string_list value_list; +}; + +struct configset_list_item { + struct config_set_element *e; + int value_index; +}; + +/* + * the contents of the list are ordered according to their + * position in the config files and order of parsing the files. + * (i.e. key-value pair at the last position of .git/config will + * be at the last item of the list) + */ +struct configset_list { + struct configset_list_item *items; + unsigned int nr, alloc; +}; + +struct config_set { + struct hashmap config_hash; + int hash_initialized; + struct configset_list list; +}; + +extern void git_configset_init(struct config_set *cs); +extern int git_configset_add_file(struct config_set *cs, const char *filename); +extern int git_configset_get_value(struct config_set *cs, const char *key, const char **value); +extern const struct string_list *git_configset_get_value_multi(struct config_set *cs, const char *key); +extern void git_configset_clear(struct config_set *cs); +extern int git_configset_get_string_const(struct config_set *cs, const char *key, const char **dest); +extern int git_configset_get_string(struct config_set *cs, const char *key, char **dest); +extern int git_configset_get_int(struct config_set *cs, const char *key, int *dest); +extern int git_configset_get_ulong(struct config_set *cs, const char *key, unsigned long *dest); +extern int git_configset_get_bool(struct config_set *cs, const char *key, int *dest); +extern int git_configset_get_bool_or_int(struct config_set *cs, const char *key, int *is_bool, int *dest); +extern int git_configset_get_maybe_bool(struct config_set *cs, const char *key, int *dest); +extern int git_configset_get_pathname(struct config_set *cs, const char *key, const char **dest); + +extern int git_config_get_value(const char *key, const char **value); +extern const struct string_list *git_config_get_value_multi(const char *key); +extern void git_config_clear(void); +extern void git_config_iter(config_fn_t fn, void *data); +extern int git_config_get_string_const(const char *key, const char **dest); +extern int git_config_get_string(const char *key, char **dest); +extern int git_config_get_int(const char *key, int *dest); +extern int git_config_get_ulong(const char *key, unsigned long *dest); +extern int git_config_get_bool(const char *key, int *dest); +extern int git_config_get_bool_or_int(const char *key, int *is_bool, int *dest); +extern int git_config_get_maybe_bool(const char *key, int *dest); +extern int git_config_get_pathname(const char *key, const char **dest); + +struct key_value_info { + const char *filename; + int linenr; +}; + +extern NORETURN void git_die_config(const char *key, const char *err, ...) __attribute__((format(printf, 2, 3))); +extern NORETURN void git_die_config_linenr(const char *key, const char *filename, int linenr); + extern int committer_ident_sufficiently_given(void); extern int author_ident_sufficiently_given(void); @@@ -1551,15 -1325,9 +1560,15 @@@ extern const char *git_mailmap_blob /* IO helper functions */ extern void maybe_flush_or_die(FILE *, const char *); +__attribute__((format (printf, 2, 3))) +extern void fprintf_or_die(FILE *, const char *fmt, ...); + +#define COPY_READ_ERROR (-2) +#define COPY_WRITE_ERROR (-3) extern int copy_fd(int ifd, int ofd); extern int copy_file(const char *dst, const char *src, int mode); extern int copy_file_with_time(const char *dst, const char *src, int mode); + extern void write_or_die(int fd, const void *buf, size_t count); extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg); extern int write_or_whine_pipe(int fd, const void *buf, size_t count, const char *msg); @@@ -1573,8 -1341,6 +1582,8 @@@ static inline ssize_t write_str_in_full { return write_in_full(fd, str, strlen(str)); } +__attribute__((format (printf, 3, 4))) +extern int write_file(const char *path, int fatal, const char *fmt, ...); /* pager.c */ extern void setup_pager(void); @@@ -1582,7 -1348,7 +1591,7 @@@ extern const char *pager_program extern int pager_in_use(void); extern int pager_use_color; extern int term_columns(void); -extern int decimal_width(int); +extern int decimal_width(uintmax_t); extern int check_pager_config(const char *cmd); extern const char *editor_program; @@@ -1602,7 -1368,17 +1611,7 @@@ extern void *alloc_object_node(void) extern void alloc_report(void); extern unsigned int alloc_commit_index(void); -/* trace.c */ -__attribute__((format (printf, 1, 2))) -extern void trace_printf(const char *format, ...); -__attribute__((format (printf, 2, 3))) -extern void trace_argv_printf(const char **argv, const char *format, ...); -extern void trace_repo_setup(const char *prefix); -extern int trace_want(const char *key); -__attribute__((format (printf, 2, 3))) -extern void trace_printf_key(const char *key, const char *fmt, ...); -extern void trace_strbuf(const char *key, const struct strbuf *buf); - +/* pkt-line.c */ void packet_trace_identity(const char *prog); /* add */ @@@ -1644,6 -1420,7 +1653,6 @@@ extern int ws_blank_line(const char *li #define ws_tab_width(rule) ((rule) & WS_TAB_WIDTH_MASK) /* ls-files */ -int report_path_error(const char *ps_matched, const struct pathspec *pathspec, const char *prefix); void overlay_tree_on_cache(const char *tree_name, const char *prefix); char *alias_lookup(const char *alias); @@@ -1698,6 -1475,5 +1707,6 @@@ int stat_validity_check(struct stat_val void stat_validity_update(struct stat_validity *sv, int fd); int versioncmp(const char *s1, const char *s2); +void sleep_millisec(int millisec); #endif /* CACHE_H */ diff --combined sha1_file.c index 50384754e5,18d0bfb8ac..77cd81db31 --- a/sha1_file.c +++ b/sha1_file.c @@@ -8,7 -8,6 +8,7 @@@ */ #include "cache.h" #include "string-list.h" +#include "lockfile.h" #include "delta.h" #include "pack.h" #include "blob.h" @@@ -37,6 -36,9 +37,6 @@@ static inline uintmax_t sz_fmt(size_t s const unsigned char null_sha1[20]; -static const char *no_log_pack_access = "no_log_pack_access"; -static const char *log_pack_access; - /* * This is meant to hold a *small* number of objects that you would * want read_sha1_file() to be able to return, but yet you do not want @@@ -266,9 -268,9 +266,9 @@@ static struct alternate_object_databas * SHA1, an extra slash for the first level indirection, and the * terminating NUL. */ -static int link_alt_odb_entry(const char *entry, const char *relative_base, int depth) +static int link_alt_odb_entry(const char *entry, const char *relative_base, + int depth, const char *normalized_objdir) { - const char *objdir = get_object_directory(); struct alternate_object_database *ent; struct alternate_object_database *alt; int pfxlen, entlen; @@@ -319,7 -321,7 +319,7 @@@ return -1; } } - if (!strcmp(ent->base, objdir)) { + if (!strcmp_icase(ent->base, normalized_objdir)) { free(ent); return -1; } @@@ -343,7 -345,6 +343,7 @@@ static void link_alt_odb_entries(const struct string_list entries = STRING_LIST_INIT_NODUP; char *alt_copy; int i; + struct strbuf objdirbuf = STRBUF_INIT; if (depth > 5) { error("%s: ignoring alternate object stores, nesting too deep.", @@@ -351,9 -352,6 +351,9 @@@ return; } + strbuf_add_absolute_path(&objdirbuf, get_object_directory()); + normalize_path_copy(objdirbuf.buf, objdirbuf.buf); + alt_copy = xmemdupz(alt, len); string_list_split_in_place(&entries, alt_copy, sep, -1); for (i = 0; i < entries.nr; i++) { @@@ -364,12 -362,11 +364,12 @@@ error("%s: ignoring relative alternate object store %s", relative_base, entry); } else { - link_alt_odb_entry(entry, relative_base, depth); + link_alt_odb_entry(entry, relative_base, depth, objdirbuf.buf); } } string_list_clear(&entries, 0); free(alt_copy); + strbuf_release(&objdirbuf); } void read_info_alternates(const char * relative_base, int depth) @@@ -405,7 -402,7 +405,7 @@@ void add_to_alternates_file(const char { struct lock_file *lock = xcalloc(1, sizeof(struct lock_file)); int fd = hold_lock_file_for_append(lock, git_path("objects/info/alternates"), LOCK_DIE_ON_ERROR); - char *alt = mkpath("%s\n", reference); + const char *alt = mkpath("%s\n", reference); write_or_die(fd, alt, strlen(alt)); if (commit_lock_file(lock)) die("could not close alternates file"); @@@ -413,18 -410,14 +413,18 @@@ link_alt_odb_entries(alt, strlen(alt), '\n', NULL, 0); } -void foreach_alt_odb(alt_odb_fn fn, void *cb) +int foreach_alt_odb(alt_odb_fn fn, void *cb) { struct alternate_object_database *ent; + int r = 0; prepare_alt_odb(); - for (ent = alt_odb_list; ent; ent = ent->next) - if (fn(ent, cb)) - return; + for (ent = alt_odb_list; ent; ent = ent->next) { + r = fn(ent, cb); + if (r) + break; + } + return r; } void prepare_alt_odb(void) @@@ -443,53 -436,27 +443,53 @@@ read_info_alternates(get_object_directory(), 0); } -static int has_loose_object_local(const unsigned char *sha1) +static int freshen_file(const char *fn) { - return !access(sha1_file_name(sha1), F_OK); + struct utimbuf t; + t.actime = t.modtime = time(NULL); + return !utime(fn, &t); } -int has_loose_object_nonlocal(const unsigned char *sha1) +static int check_and_freshen_file(const char *fn, int freshen) +{ + if (access(fn, F_OK)) + return 0; + if (freshen && freshen_file(fn)) + return 0; + return 1; +} + +static int check_and_freshen_local(const unsigned char *sha1, int freshen) +{ + return check_and_freshen_file(sha1_file_name(sha1), freshen); +} + +static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen) { struct alternate_object_database *alt; prepare_alt_odb(); for (alt = alt_odb_list; alt; alt = alt->next) { fill_sha1_path(alt->name, sha1); - if (!access(alt->base, F_OK)) + if (check_and_freshen_file(alt->base, freshen)) return 1; } return 0; } +static int check_and_freshen(const unsigned char *sha1, int freshen) +{ + return check_and_freshen_local(sha1, freshen) || + check_and_freshen_nonlocal(sha1, freshen); +} + +int has_loose_object_nonlocal(const unsigned char *sha1) +{ + return check_and_freshen_nonlocal(sha1, 0); +} + static int has_loose_object(const unsigned char *sha1) { - return has_loose_object_local(sha1) || - has_loose_object_nonlocal(sha1); + return check_and_freshen(sha1, 0); } static unsigned int pack_used_ctr; @@@ -694,44 -661,21 +694,44 @@@ void release_pack_memory(size_t need ; /* nothing */ } -void *xmmap(void *start, size_t length, - int prot, int flags, int fd, off_t offset) +static void mmap_limit_check(size_t length) { - void *ret = mmap(start, length, prot, flags, fd, offset); + static size_t limit = 0; + if (!limit) { + limit = git_env_ulong("GIT_MMAP_LIMIT", 0); + if (!limit) + limit = SIZE_MAX; + } + if (length > limit) + die("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX, + (uintmax_t)length, (uintmax_t)limit); +} + +void *xmmap_gently(void *start, size_t length, + int prot, int flags, int fd, off_t offset) +{ + void *ret; + + mmap_limit_check(length); + ret = mmap(start, length, prot, flags, fd, offset); if (ret == MAP_FAILED) { if (!length) return NULL; release_pack_memory(length); ret = mmap(start, length, prot, flags, fd, offset); - if (ret == MAP_FAILED) - die_errno("Out of memory? mmap failed"); } return ret; } +void *xmmap(void *start, size_t length, + int prot, int flags, int fd, off_t offset) +{ + void *ret = xmmap_gently(start, length, prot, flags, fd, offset); + if (ret == MAP_FAILED) + die_errno("mmap failed"); + return ret; +} + void close_pack_windows(struct packed_git *p) { while (p->windows) { @@@ -1205,7 -1149,7 +1205,7 @@@ static void report_pack_garbage(struct if (!report_garbage) return; - sort_string_list(list); + string_list_sort(list); for (i = 0; i < list->nr; i++) { const char *path = list->items[i].string; @@@ -1234,42 -1178,48 +1234,42 @@@ static void prepare_packed_git_one(char *objdir, int local) { - /* Ensure that this buffer is large enough so that we can - append "/pack/" without clobbering the stack even if - strlen(objdir) were PATH_MAX. */ - char path[PATH_MAX + 1 + 4 + 1 + 1]; - int len; + struct strbuf path = STRBUF_INIT; + size_t dirnamelen; DIR *dir; struct dirent *de; struct string_list garbage = STRING_LIST_INIT_DUP; - sprintf(path, "%s/pack", objdir); - len = strlen(path); - dir = opendir(path); + strbuf_addstr(&path, objdir); + strbuf_addstr(&path, "/pack"); + dir = opendir(path.buf); if (!dir) { if (errno != ENOENT) error("unable to open object pack directory: %s: %s", - path, strerror(errno)); + path.buf, strerror(errno)); + strbuf_release(&path); return; } - path[len++] = '/'; + strbuf_addch(&path, '/'); + dirnamelen = path.len; while ((de = readdir(dir)) != NULL) { - int namelen = strlen(de->d_name); struct packed_git *p; - - if (len + namelen + 1 > sizeof(path)) { - if (report_garbage) { - struct strbuf sb = STRBUF_INIT; - strbuf_addf(&sb, "%.*s/%s", len - 1, path, de->d_name); - report_garbage("path too long", sb.buf); - strbuf_release(&sb); - } - continue; - } + size_t base_len; if (is_dot_or_dotdot(de->d_name)) continue; - strcpy(path + len, de->d_name); + strbuf_setlen(&path, dirnamelen); + strbuf_addstr(&path, de->d_name); - if (has_extension(de->d_name, ".idx")) { + base_len = path.len; + if (strip_suffix_mem(path.buf, &base_len, ".idx")) { /* Don't reopen a pack we already have. */ for (p = packed_git; p; p = p->next) { - if (!memcmp(path, p->pack_name, len + namelen - 4)) + size_t len; + if (strip_suffix(p->pack_name, ".pack", &len) && + len == base_len && + !memcmp(p->pack_name, path.buf, len)) break; } if (p == NULL && @@@ -1277,25 -1227,24 +1277,25 @@@ * See if it really is a valid .idx file with * corresponding .pack file that we can map. */ - (p = add_packed_git(path, len + namelen, local)) != NULL) + (p = add_packed_git(path.buf, path.len, local)) != NULL) install_packed_git(p); } if (!report_garbage) continue; - if (has_extension(de->d_name, ".idx") || - has_extension(de->d_name, ".pack") || - has_extension(de->d_name, ".bitmap") || - has_extension(de->d_name, ".keep")) - string_list_append(&garbage, path); + if (ends_with(de->d_name, ".idx") || + ends_with(de->d_name, ".pack") || + ends_with(de->d_name, ".bitmap") || + ends_with(de->d_name, ".keep")) + string_list_append(&garbage, path.buf); else - report_garbage("garbage found", path); + report_garbage("garbage found", path.buf); } closedir(dir); report_pack_garbage(&garbage); string_list_clear(&garbage, 0); + strbuf_release(&path); } static int sort_pack(const void *a_, const void *b_) @@@ -1571,40 -1520,6 +1571,40 @@@ int unpack_sha1_header(git_zstream *str return git_inflate(stream, 0); } +static int unpack_sha1_header_to_strbuf(git_zstream *stream, unsigned char *map, + unsigned long mapsize, void *buffer, + unsigned long bufsiz, struct strbuf *header) +{ + int status; + + status = unpack_sha1_header(stream, map, mapsize, buffer, bufsiz); + + /* + * Check if entire header is unpacked in the first iteration. + */ + if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer)) + return 0; + + /* + * buffer[0..bufsiz] was not large enough. Copy the partial + * result out to header, and then append the result of further + * reading the stream. + */ + strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer); + stream->next_out = buffer; + stream->avail_out = bufsiz; + + do { + status = git_inflate(stream, 0); + strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer); + if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer)) + return 0; + stream->next_out = buffer; + stream->avail_out = bufsiz; + } while (status != Z_STREAM_END); + return -1; +} + static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1) { int bytes = strlen(buffer) + 1; @@@ -1655,38 -1570,27 +1655,38 @@@ * too permissive for what we want to check. So do an anal * object header parse by hand. */ -int parse_sha1_header(const char *hdr, unsigned long *sizep) +static int parse_sha1_header_extended(const char *hdr, struct object_info *oi, + unsigned int flags) { - char type[10]; - int i; + const char *type_buf = hdr; unsigned long size; + int type, type_len = 0; /* - * The type can be at most ten bytes (including the - * terminating '\0' that we add), and is followed by + * The type can be of any size but is followed by * a space. */ - i = 0; for (;;) { char c = *hdr++; if (c == ' ') break; - type[i++] = c; - if (i >= sizeof(type)) - return -1; + type_len++; } - type[i] = 0; + + type = type_from_string_gently(type_buf, type_len, 1); + if (oi->typename) + strbuf_add(oi->typename, type_buf, type_len); + /* + * Set type to 0 if its an unknown object and + * we're obtaining the type using '--allow-unkown-type' + * option. + */ + if ((flags & LOOKUP_UNKNOWN_OBJECT) && (type < 0)) + type = 0; + else if (type < 0) + die("invalid object type"); + if (oi->typep) + *oi->typep = type; /* * The length must follow immediately, and be in canonical @@@ -1704,24 -1608,12 +1704,24 @@@ size = size * 10 + c; } } - *sizep = size; + + if (oi->sizep) + *oi->sizep = size; /* * The length must be followed by a zero byte */ - return *hdr ? -1 : type_from_string(type); + return *hdr ? -1 : type; +} + +int parse_sha1_header(const char *hdr, unsigned long *sizep) +{ + struct object_info oi; + + oi.sizep = sizep; + oi.typename = NULL; + oi.typep = NULL; + return parse_sha1_header_extended(hdr, &oi, LOOKUP_REPLACE_OBJECT); } static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1) @@@ -2034,9 -1926,7 +2034,9 @@@ static void *unpack_compressed_entry(st git_zstream stream; unsigned char *buffer, *in; - buffer = xmallocz(size); + buffer = xmallocz_gently(size); + if (!buffer) + return NULL; memset(&stream, 0, sizeof(stream)); stream.next_out = buffer; stream.avail_out = size + 1; @@@ -2196,9 -2086,27 +2196,9 @@@ static void *read_object(const unsigne static void write_pack_access_log(struct packed_git *p, off_t obj_offset) { - static FILE *log_file; - - if (!log_pack_access) - log_pack_access = getenv("GIT_TRACE_PACK_ACCESS"); - if (!log_pack_access) - log_pack_access = no_log_pack_access; - if (log_pack_access == no_log_pack_access) - return; - - if (!log_file) { - log_file = fopen(log_pack_access, "w"); - if (!log_file) { - error("cannot open pack access log '%s' for writing: %s", - log_pack_access, strerror(errno)); - log_pack_access = no_log_pack_access; - return; - } - } - fprintf(log_file, "%s %"PRIuMAX"\n", - p->pack_name, (uintmax_t)obj_offset); - fflush(log_file); + static struct trace_key pack_access = TRACE_KEY_INIT(PACK_ACCESS); + trace_printf_key(&pack_access, "%s %"PRIuMAX"\n", + p->pack_name, (uintmax_t)obj_offset); } int do_check_packed_object_crc; @@@ -2223,7 -2131,8 +2223,7 @@@ void *unpack_entry(struct packed_git *p int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC; int base_from_cache = 0; - if (log_pack_access != no_log_pack_access) - write_pack_access_log(p, obj_offset); + write_pack_access_log(p, obj_offset); /* PHASE 1: drill down to the innermost base object */ for (;;) { @@@ -2537,8 -2446,10 +2537,8 @@@ static int fill_pack_entry(const unsign * answer, as it may have been deleted since the index was * loaded! */ - if (!is_pack_valid(p)) { - warning("packfile %s cannot be accessed", p->pack_name); + if (!is_pack_valid(p)) return 0; - } e->offset = offset; e->p = p; hashcpy(e->sha1, sha1); @@@ -2586,15 -2497,13 +2586,15 @@@ struct packed_git *find_sha1_pack(cons } static int sha1_loose_object_info(const unsigned char *sha1, - struct object_info *oi) + struct object_info *oi, + int flags) { - int status; - unsigned long mapsize, size; + int status = 0; + unsigned long mapsize; void *map; git_zstream stream; char hdr[32]; + struct strbuf hdrbuf = STRBUF_INIT; if (oi->delta_base_sha1) hashclr(oi->delta_base_sha1); @@@ -2607,7 -2516,7 +2607,7 @@@ * return value implicitly indicates whether the * object even exists. */ - if (!oi->typep && !oi->sizep) { + if (!oi->typep && !oi->typename && !oi->sizep) { struct stat st; if (stat_sha1_file(sha1, &st) < 0) return -1; @@@ -2621,26 -2530,17 +2621,26 @@@ return -1; if (oi->disk_sizep) *oi->disk_sizep = mapsize; - if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) + if ((flags & LOOKUP_UNKNOWN_OBJECT)) { + if (unpack_sha1_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0) + status = error("unable to unpack %s header with --allow-unknown-type", + sha1_to_hex(sha1)); + } else if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) status = error("unable to unpack %s header", sha1_to_hex(sha1)); - else if ((status = parse_sha1_header(hdr, &size)) < 0) + if (status < 0) + ; /* Do nothing */ + else if (hdrbuf.len) { + if ((status = parse_sha1_header_extended(hdrbuf.buf, oi, flags)) < 0) + status = error("unable to parse %s header with --allow-unknown-type", + sha1_to_hex(sha1)); + } else if ((status = parse_sha1_header_extended(hdr, oi, flags)) < 0) status = error("unable to parse %s header", sha1_to_hex(sha1)); - else if (oi->sizep) - *oi->sizep = size; git_inflate_end(&stream); munmap(map, mapsize); - if (oi->typep) + if (status && oi->typep) *oi->typep = status; + strbuf_release(&hdrbuf); return 0; } @@@ -2649,7 -2549,6 +2649,7 @@@ int sha1_object_info_extended(const uns struct cached_object *co; struct pack_entry e; int rtype; + enum object_type real_type; const unsigned char *real = lookup_replace_object_extended(sha1, flags); co = find_cached_object(real); @@@ -2662,15 -2561,13 +2662,15 @@@ *(oi->disk_sizep) = 0; if (oi->delta_base_sha1) hashclr(oi->delta_base_sha1); + if (oi->typename) + strbuf_addstr(oi->typename, typename(co->type)); oi->whence = OI_CACHED; return 0; } if (!find_pack_entry(real, &e)) { /* Most likely it's a loose object. */ - if (!sha1_loose_object_info(real, oi)) { + if (!sha1_loose_object_info(real, oi, flags)) { oi->whence = OI_LOOSE; return 0; } @@@ -2681,18 -2578,9 +2681,18 @@@ return -1; } + /* + * packed_object_info() does not follow the delta chain to + * find out the real type, unless it is given oi->typep. + */ + if (oi->typename && !oi->typep) + oi->typep = &real_type; + rtype = packed_object_info(e.p, e.offset, oi); if (rtype < 0) { mark_bad_packed_object(e.p, real); + if (oi->typep == &real_type) + oi->typep = NULL; return sha1_object_info_extended(real, oi, 0); } else if (in_delta_base_cache(e.p, e.offset)) { oi->whence = OI_DBCACHED; @@@ -2703,10 -2591,6 +2703,10 @@@ oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA || rtype == OBJ_OFS_DELTA); } + if (oi->typename) + strbuf_addstr(oi->typename, typename(*oi->typep)); + if (oi->typep == &real_type) + oi->typep = NULL; return 0; } @@@ -3032,6 -2916,7 +3032,6 @@@ static int write_loose_object(const uns } /* Set it up */ - memset(&stream, 0, sizeof(stream)); git_deflate_init(&stream, zlib_compression_level); stream.next_out = compressed; stream.avail_out = sizeof(compressed); @@@ -3080,26 -2965,9 +3080,26 @@@ return move_temp_to_file(tmp_file, filename); } -int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *returnsha1) +static int freshen_loose_object(const unsigned char *sha1) +{ + return check_and_freshen(sha1, 1); +} + +static int freshen_packed_object(const unsigned char *sha1) +{ + struct pack_entry e; + if (!find_pack_entry(sha1, &e)) + return 0; + if (e.p->freshened) + return 1; + if (!freshen_file(e.p->pack_name)) + return 0; + e.p->freshened = 1; + return 1; +} + +int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1) { - unsigned char sha1[20]; char hdr[32]; int hdrlen; @@@ -3107,32 -2975,13 +3107,32 @@@ * it out into .git/objects/??/?{38} file. */ write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen); - if (returnsha1) - hashcpy(returnsha1, sha1); - if (has_sha1_file(sha1)) + if (freshen_packed_object(sha1) || freshen_loose_object(sha1)) return 0; return write_loose_object(sha1, hdr, hdrlen, buf, len, 0); } +int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type, + unsigned char *sha1, unsigned flags) +{ + char *header; + int hdrlen, status = 0; + + /* type string, SP, %lu of the length plus NUL must fit this */ + header = xmalloc(strlen(type) + 32); + write_sha1_file_prepare(buf, len, type, sha1, header, &hdrlen); + + if (!(flags & HASH_WRITE_OBJECT)) + goto cleanup; + if (freshen_packed_object(sha1) || freshen_loose_object(sha1)) + goto cleanup; + status = write_loose_object(sha1, header, hdrlen, buf, len, 0); + +cleanup: + free(header); + return status; +} + int force_object_loose(const unsigned char *sha1, time_t mtime) { void *buf; @@@ -3168,7 -3017,7 +3168,7 @@@ int has_sha1_pack(const unsigned char * return find_pack_entry(sha1, &e); } - int has_sha1_file(const unsigned char *sha1) + int has_sha1_file_with_flags(const unsigned char *sha1, int flags) { struct pack_entry e; @@@ -3176,6 -3025,8 +3176,8 @@@ return 1; if (has_loose_object(sha1)) return 1; + if (flags & HAS_SHA1_QUICK) + return 0; reprepare_packed_git(); return find_pack_entry(sha1, &e); } @@@ -3247,29 -3098,6 +3249,29 @@@ static int index_mem(unsigned char *sha return ret; } +static int index_stream_convert_blob(unsigned char *sha1, int fd, + const char *path, unsigned flags) +{ + int ret; + const int write_object = flags & HASH_WRITE_OBJECT; + struct strbuf sbuf = STRBUF_INIT; + + assert(path); + assert(would_convert_to_git_filter_fd(path)); + + convert_to_git_filter_fd(path, fd, &sbuf, + write_object ? safe_crlf : SAFE_CRLF_FALSE); + + if (write_object) + ret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB), + sha1); + else + ret = hash_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB), + sha1); + strbuf_release(&sbuf); + return ret; +} + static int index_pipe(unsigned char *sha1, int fd, enum object_type type, const char *path, unsigned flags) { @@@ -3293,7 -3121,7 +3295,7 @@@ static int index_core(unsigned char *sh int ret; if (!size) { - ret = index_mem(sha1, NULL, size, type, path, flags); + ret = index_mem(sha1, "", size, type, path, flags); } else if (size <= SMALL_FILE_SIZE) { char *buf = xmalloc(size); if (size == read_in_full(fd, buf, size)) @@@ -3335,22 -3163,15 +3337,22 @@@ int index_fd(unsigned char *sha1, int f enum object_type type, const char *path, unsigned flags) { int ret; - size_t size = xsize_t(st->st_size); - if (!S_ISREG(st->st_mode)) + /* + * Call xsize_t() only when needed to avoid potentially unnecessary + * die() for large files. + */ + if (type == OBJ_BLOB && path && would_convert_to_git_filter_fd(path)) + ret = index_stream_convert_blob(sha1, fd, path, flags); + else if (!S_ISREG(st->st_mode)) ret = index_pipe(sha1, fd, type, path, flags); - else if (size <= big_file_threshold || type != OBJ_BLOB || - (path && would_convert_to_git(path, NULL, 0, 0))) - ret = index_core(sha1, fd, size, type, path, flags); + else if (st->st_size <= big_file_threshold || type != OBJ_BLOB || + (path && would_convert_to_git(path))) + ret = index_core(sha1, fd, xsize_t(st->st_size), type, path, + flags); else - ret = index_stream(sha1, fd, size, type, path, flags); + ret = index_stream(sha1, fd, xsize_t(st->st_size), type, path, + flags); close(fd); return ret; } @@@ -3415,172 -3236,3 +3417,172 @@@ void assert_sha1_type(const unsigned ch die("%s is not a valid '%s' object", sha1_to_hex(sha1), typename(expect)); } + +static int for_each_file_in_obj_subdir(int subdir_nr, + struct strbuf *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data) +{ + size_t baselen = path->len; + DIR *dir = opendir(path->buf); + struct dirent *de; + int r = 0; + + if (!dir) { + if (errno == ENOENT) + return 0; + return error("unable to open %s: %s", path->buf, strerror(errno)); + } + + while ((de = readdir(dir))) { + if (is_dot_or_dotdot(de->d_name)) + continue; + + strbuf_setlen(path, baselen); + strbuf_addf(path, "/%s", de->d_name); + + if (strlen(de->d_name) == 38) { + char hex[41]; + unsigned char sha1[20]; + + snprintf(hex, sizeof(hex), "%02x%s", + subdir_nr, de->d_name); + if (!get_sha1_hex(hex, sha1)) { + if (obj_cb) { + r = obj_cb(sha1, path->buf, data); + if (r) + break; + } + continue; + } + } + + if (cruft_cb) { + r = cruft_cb(de->d_name, path->buf, data); + if (r) + break; + } + } + strbuf_setlen(path, baselen); + + if (!r && subdir_cb) + r = subdir_cb(subdir_nr, path->buf, data); + + closedir(dir); + return r; +} + +int for_each_loose_file_in_objdir_buf(struct strbuf *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data) +{ + size_t baselen = path->len; + int r = 0; + int i; + + for (i = 0; i < 256; i++) { + strbuf_addf(path, "/%02x", i); + r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb, + subdir_cb, data); + strbuf_setlen(path, baselen); + if (r) + break; + } + + return r; +} + +int for_each_loose_file_in_objdir(const char *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data) +{ + struct strbuf buf = STRBUF_INIT; + int r; + + strbuf_addstr(&buf, path); + r = for_each_loose_file_in_objdir_buf(&buf, obj_cb, cruft_cb, + subdir_cb, data); + strbuf_release(&buf); + + return r; +} + +struct loose_alt_odb_data { + each_loose_object_fn *cb; + void *data; +}; + +static int loose_from_alt_odb(struct alternate_object_database *alt, + void *vdata) +{ + struct loose_alt_odb_data *data = vdata; + struct strbuf buf = STRBUF_INIT; + int r; + + /* copy base not including trailing '/' */ + strbuf_add(&buf, alt->base, alt->name - alt->base - 1); + r = for_each_loose_file_in_objdir_buf(&buf, + data->cb, NULL, NULL, + data->data); + strbuf_release(&buf); + return r; +} + +int for_each_loose_object(each_loose_object_fn cb, void *data, unsigned flags) +{ + struct loose_alt_odb_data alt; + int r; + + r = for_each_loose_file_in_objdir(get_object_directory(), + cb, NULL, NULL, data); + if (r) + return r; + + if (flags & FOR_EACH_OBJECT_LOCAL_ONLY) + return 0; + + alt.cb = cb; + alt.data = data; + return foreach_alt_odb(loose_from_alt_odb, &alt); +} + +static int for_each_object_in_pack(struct packed_git *p, each_packed_object_fn cb, void *data) +{ + uint32_t i; + int r = 0; + + for (i = 0; i < p->num_objects; i++) { + const unsigned char *sha1 = nth_packed_object_sha1(p, i); + + if (!sha1) + return error("unable to get sha1 of object %u in %s", + i, p->pack_name); + + r = cb(sha1, p, i, data); + if (r) + break; + } + return r; +} + +int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags) +{ + struct packed_git *p; + int r = 0; + + prepare_packed_git(); + for (p = packed_git; p; p = p->next) { + if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local) + continue; + r = for_each_object_in_pack(p, cb, data); + if (r) + break; + } + return r; +}