From: Junio C Hamano Date: Tue, 18 Mar 2014 20:50:21 +0000 (-0700) Subject: Merge branch 'dd/use-alloc-grow' X-Git-Tag: v2.0.0-rc0~97 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/fe9122a35213827348c521a16ffd0cf2652c4ac5?ds=inline;hp=-c Merge branch 'dd/use-alloc-grow' Replace open-coded reallocation with ALLOC_GROW() macro. * dd/use-alloc-grow: sha1_file.c: use ALLOC_GROW() in pretend_sha1_file() read-cache.c: use ALLOC_GROW() in add_index_entry() builtin/mktree.c: use ALLOC_GROW() in append_to_tree() attr.c: use ALLOC_GROW() in handle_attr_line() dir.c: use ALLOC_GROW() in create_simplify() reflog-walk.c: use ALLOC_GROW() replace_object.c: use ALLOC_GROW() in register_replace_object() patch-ids.c: use ALLOC_GROW() in add_commit() diffcore-rename.c: use ALLOC_GROW() diff.c: use ALLOC_GROW() commit.c: use ALLOC_GROW() in register_commit_graft() cache-tree.c: use ALLOC_GROW() in find_subtree() bundle.c: use ALLOC_GROW() in add_to_ref_list() builtin/pack-objects.c: use ALLOC_GROW() in check_pbase_path() --- fe9122a35213827348c521a16ffd0cf2652c4ac5 diff --combined builtin/pack-objects.c index 1fb972f45a,b53bb5b25c..057c54b902 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@@ -14,12 -14,10 +14,12 @@@ #include "diff.h" #include "revision.h" #include "list-objects.h" +#include "pack-objects.h" #include "progress.h" #include "refs.h" #include "streaming.h" #include "thread-utils.h" +#include "pack-bitmap.h" static const char *pack_usage[] = { N_("git pack-objects --stdout [options...] [< ref-list | < object-list]"), @@@ -27,15 -25,42 +27,15 @@@ NULL }; -struct object_entry { - struct pack_idx_entry idx; - unsigned long size; /* uncompressed size */ - struct packed_git *in_pack; /* already in pack */ - off_t in_pack_offset; - struct object_entry *delta; /* delta base object */ - struct object_entry *delta_child; /* deltified objects who bases me */ - struct object_entry *delta_sibling; /* other deltified objects who - * uses the same base as me - */ - void *delta_data; /* cached delta (uncompressed) */ - unsigned long delta_size; /* delta data size (uncompressed) */ - unsigned long z_delta_size; /* delta data size (compressed) */ - enum object_type type; - enum object_type in_pack_type; /* could be delta */ - uint32_t hash; /* name hint hash */ - unsigned char in_pack_header_size; - unsigned preferred_base:1; /* - * we do not pack this, but is available - * to be used as the base object to delta - * objects against. - */ - unsigned no_try_delta:1; - unsigned tagged:1; /* near the very tip of refs */ - unsigned filled:1; /* assigned write-order */ -}; - /* - * Objects we are going to pack are collected in objects array (dynamically - * expanded). nr_objects & nr_alloc controls this array. They are stored - * in the order we see -- typically rev-list --objects order that gives us - * nice "minimum seek" order. + * Objects we are going to pack are collected in the `to_pack` structure. + * It contains an array (dynamically expanded) of the object data, and a map + * that can resolve SHA1s to their position in the array. */ -static struct object_entry *objects; +static struct packing_data to_pack; + static struct pack_idx_entry **written_list; -static uint32_t nr_objects, nr_alloc, nr_result, nr_written; +static uint32_t nr_result, nr_written; static int non_empty; static int reuse_delta = 1, reuse_object = 1; @@@ -58,43 -83,27 +58,43 @@@ static struct progress *progress_state static int pack_compression_level = Z_DEFAULT_COMPRESSION; static int pack_compression_seen; +static struct packed_git *reuse_packfile; +static uint32_t reuse_packfile_objects; +static off_t reuse_packfile_offset; + +static int use_bitmap_index = 1; +static int write_bitmap_index; +static uint16_t write_bitmap_options; + static unsigned long delta_cache_size = 0; static unsigned long max_delta_cache_size = 256 * 1024 * 1024; static unsigned long cache_max_small_delta_size = 1000; static unsigned long window_memory_limit = 0; -/* - * The object names in objects array are hashed with this hashtable, - * to help looking up the entry by object name. - * This hashtable is built after all the objects are seen. - */ -static int *object_ix; -static int object_ix_hashsz; -static struct object_entry *locate_object_entry(const unsigned char *sha1); - /* * stats */ static uint32_t written, written_delta; static uint32_t reused, reused_delta; +/* + * Indexed commits + */ +static struct commit **indexed_commits; +static unsigned int indexed_commits_nr; +static unsigned int indexed_commits_alloc; + +static void index_commit_for_bitmap(struct commit *commit) +{ + if (indexed_commits_nr >= indexed_commits_alloc) { + indexed_commits_alloc = (indexed_commits_alloc + 32) * 2; + indexed_commits = xrealloc(indexed_commits, + indexed_commits_alloc * sizeof(struct commit *)); + } + + indexed_commits[indexed_commits_nr++] = commit; +} static void *get_delta(struct object_entry *entry) { @@@ -544,12 -553,12 +544,12 @@@ static int mark_tagged(const char *path void *cb_data) { unsigned char peeled[20]; - struct object_entry *entry = locate_object_entry(sha1); + struct object_entry *entry = packlist_find(&to_pack, sha1, NULL); if (entry) entry->tagged = 1; if (!peel_ref(path, peeled)) { - entry = locate_object_entry(peeled); + entry = packlist_find(&to_pack, peeled, NULL); if (entry) entry->tagged = 1; } @@@ -624,10 -633,9 +624,10 @@@ static struct object_entry **compute_wr { unsigned int i, wo_end, last_untagged; - struct object_entry **wo = xmalloc(nr_objects * sizeof(*wo)); + struct object_entry **wo = xmalloc(to_pack.nr_objects * sizeof(*wo)); + struct object_entry *objects = to_pack.objects; - for (i = 0; i < nr_objects; i++) { + for (i = 0; i < to_pack.nr_objects; i++) { objects[i].tagged = 0; objects[i].filled = 0; objects[i].delta_child = NULL; @@@ -639,7 -647,7 +639,7 @@@ * Make sure delta_sibling is sorted in the original * recency order. */ - for (i = nr_objects; i > 0;) { + for (i = to_pack.nr_objects; i > 0;) { struct object_entry *e = &objects[--i]; if (!e->delta) continue; @@@ -657,7 -665,7 +657,7 @@@ * Give the objects in the original recency order until * we see a tagged tip. */ - for (i = wo_end = 0; i < nr_objects; i++) { + for (i = wo_end = 0; i < to_pack.nr_objects; i++) { if (objects[i].tagged) break; add_to_write_order(wo, &wo_end, &objects[i]); @@@ -667,7 -675,7 +667,7 @@@ /* * Then fill all the tagged tips. */ - for (; i < nr_objects; i++) { + for (; i < to_pack.nr_objects; i++) { if (objects[i].tagged) add_to_write_order(wo, &wo_end, &objects[i]); } @@@ -675,7 -683,7 +675,7 @@@ /* * And then all remaining commits and tags. */ - for (i = last_untagged; i < nr_objects; i++) { + for (i = last_untagged; i < to_pack.nr_objects; i++) { if (objects[i].type != OBJ_COMMIT && objects[i].type != OBJ_TAG) continue; @@@ -685,7 -693,7 +685,7 @@@ /* * And then all the trees. */ - for (i = last_untagged; i < nr_objects; i++) { + for (i = last_untagged; i < to_pack.nr_objects; i++) { if (objects[i].type != OBJ_TREE) continue; add_to_write_order(wo, &wo_end, &objects[i]); @@@ -694,57 -702,17 +694,57 @@@ /* * Finally all the rest in really tight order */ - for (i = last_untagged; i < nr_objects; i++) { + for (i = last_untagged; i < to_pack.nr_objects; i++) { if (!objects[i].filled) add_family_to_write_order(wo, &wo_end, &objects[i]); } - if (wo_end != nr_objects) - die("ordered %u objects, expected %"PRIu32, wo_end, nr_objects); + if (wo_end != to_pack.nr_objects) + die("ordered %u objects, expected %"PRIu32, wo_end, to_pack.nr_objects); return wo; } +static off_t write_reused_pack(struct sha1file *f) +{ + unsigned char buffer[8192]; + off_t to_write; + int fd; + + if (!is_pack_valid(reuse_packfile)) + die("packfile is invalid: %s", reuse_packfile->pack_name); + + fd = git_open_noatime(reuse_packfile->pack_name); + if (fd < 0) + die_errno("unable to open packfile for reuse: %s", + reuse_packfile->pack_name); + + if (lseek(fd, sizeof(struct pack_header), SEEK_SET) == -1) + die_errno("unable to seek in reused packfile"); + + if (reuse_packfile_offset < 0) + reuse_packfile_offset = reuse_packfile->pack_size - 20; + + to_write = reuse_packfile_offset - sizeof(struct pack_header); + + while (to_write) { + int read_pack = xread(fd, buffer, sizeof(buffer)); + + if (read_pack <= 0) + die_errno("unable to read from reused packfile"); + + if (read_pack > to_write) + read_pack = to_write; + + sha1write(f, buffer, read_pack); + to_write -= read_pack; + } + + close(fd); + written += reuse_packfile_objects; + return reuse_packfile_offset - sizeof(struct pack_header); +} + static void write_pack_file(void) { uint32_t i = 0, j; @@@ -755,8 -723,8 +755,8 @@@ struct object_entry **write_order; if (progress > pack_to_stdout) - progress_state = start_progress("Writing objects", nr_result); - written_list = xmalloc(nr_objects * sizeof(*written_list)); + progress_state = start_progress(_("Writing objects"), nr_result); + written_list = xmalloc(to_pack.nr_objects * sizeof(*written_list)); write_order = compute_write_order(); do { @@@ -769,17 -737,8 +769,17 @@@ f = create_tmp_packfile(&pack_tmp_name); offset = write_pack_header(f, nr_remaining); + + if (reuse_packfile) { + off_t packfile_size; + assert(pack_to_stdout); + + packfile_size = write_reused_pack(f); + offset += packfile_size; + } + nr_written = 0; - for (; i < nr_objects; i++) { + for (; i < to_pack.nr_objects; i++) { struct object_entry *e = write_order[i]; if (write_one(f, e, &offset) == WRITE_ONE_BREAK) break; @@@ -823,38 -782,16 +823,38 @@@ utb.modtime = --last_mtime; if (utime(pack_tmp_name, &utb) < 0) warning("failed utime() on %s: %s", - tmpname, strerror(errno)); + pack_tmp_name, strerror(errno)); } /* Enough space for "-.pack"? */ if (sizeof(tmpname) <= strlen(base_name) + 50) die("pack base name '%s' too long", base_name); snprintf(tmpname, sizeof(tmpname), "%s-", base_name); + + if (write_bitmap_index) { + bitmap_writer_set_checksum(sha1); + bitmap_writer_build_type_index(written_list, nr_written); + } + finish_tmp_packfile(tmpname, pack_tmp_name, written_list, nr_written, &pack_idx_opts, sha1); + + if (write_bitmap_index) { + char *end_of_name_prefix = strrchr(tmpname, 0); + sprintf(end_of_name_prefix, "%s.bitmap", sha1_to_hex(sha1)); + + stop_progress(&progress_state); + + bitmap_writer_show_progress(progress); + bitmap_writer_reuse_bitmaps(&to_pack); + bitmap_writer_select_commits(indexed_commits, indexed_commits_nr, -1); + bitmap_writer_build(&to_pack); + bitmap_writer_finish(written_list, nr_written, + tmpname, write_bitmap_options); + write_bitmap_index = 0; + } + free(pack_tmp_name); puts(sha1_to_hex(sha1)); } @@@ -864,7 -801,7 +864,7 @@@ written_list[j]->offset = (off_t)-1; } nr_remaining -= nr_written; - } while (nr_remaining && i < nr_objects); + } while (nr_remaining && i < to_pack.nr_objects); free(written_list); free(write_order); @@@ -874,6 -811,73 +874,6 @@@ written, nr_result); } -static int locate_object_entry_hash(const unsigned char *sha1) -{ - int i; - unsigned int ui; - memcpy(&ui, sha1, sizeof(unsigned int)); - i = ui % object_ix_hashsz; - while (0 < object_ix[i]) { - if (!hashcmp(sha1, objects[object_ix[i] - 1].idx.sha1)) - return i; - if (++i == object_ix_hashsz) - i = 0; - } - return -1 - i; -} - -static struct object_entry *locate_object_entry(const unsigned char *sha1) -{ - int i; - - if (!object_ix_hashsz) - return NULL; - - i = locate_object_entry_hash(sha1); - if (0 <= i) - return &objects[object_ix[i]-1]; - return NULL; -} - -static void rehash_objects(void) -{ - uint32_t i; - struct object_entry *oe; - - object_ix_hashsz = nr_objects * 3; - if (object_ix_hashsz < 1024) - object_ix_hashsz = 1024; - object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz); - memset(object_ix, 0, sizeof(int) * object_ix_hashsz); - for (i = 0, oe = objects; i < nr_objects; i++, oe++) { - int ix = locate_object_entry_hash(oe->idx.sha1); - if (0 <= ix) - continue; - ix = -1 - ix; - object_ix[ix] = i + 1; - } -} - -static uint32_t name_hash(const char *name) -{ - uint32_t c, hash = 0; - - if (!name) - return 0; - - /* - * This effectively just creates a sortable number from the - * last sixteen non-whitespace characters. Last characters - * count "most", so things that end in ".c" sort together. - */ - while ((c = *name++) != 0) { - if (isspace(c)) - continue; - hash = (hash >> 2) + (c << 24); - } - return hash; -} - static void setup_delta_attr_check(struct git_attr_check *check) { static struct git_attr *attr_delta; @@@ -896,69 -900,42 +896,69 @@@ static int no_try_delta(const char *pat return 0; } -static int add_object_entry(const unsigned char *sha1, enum object_type type, - const char *name, int exclude) +/* + * When adding an object, check whether we have already added it + * to our packing list. If so, we can skip. However, if we are + * being asked to excludei t, but the previous mention was to include + * it, make sure to adjust its flags and tweak our numbers accordingly. + * + * As an optimization, we pass out the index position where we would have + * found the item, since that saves us from having to look it up again a + * few lines later when we want to add the new entry. + */ +static int have_duplicate_entry(const unsigned char *sha1, + int exclude, + uint32_t *index_pos) { struct object_entry *entry; - struct packed_git *p, *found_pack = NULL; - off_t found_offset = 0; - int ix; - uint32_t hash = name_hash(name); - - ix = nr_objects ? locate_object_entry_hash(sha1) : -1; - if (ix >= 0) { - if (exclude) { - entry = objects + object_ix[ix] - 1; - if (!entry->preferred_base) - nr_result--; - entry->preferred_base = 1; - } + + entry = packlist_find(&to_pack, sha1, index_pos); + if (!entry) return 0; + + if (exclude) { + if (!entry->preferred_base) + nr_result--; + entry->preferred_base = 1; } + return 1; +} + +/* + * Check whether we want the object in the pack (e.g., we do not want + * objects found in non-local stores if the "--local" option was used). + * + * As a side effect of this check, we will find the packed version of this + * object, if any. We therefore pass out the pack information to avoid having + * to look it up again later. + */ +static int want_object_in_pack(const unsigned char *sha1, + int exclude, + struct packed_git **found_pack, + off_t *found_offset) +{ + struct packed_git *p; + if (!exclude && local && has_loose_object_nonlocal(sha1)) return 0; + *found_pack = NULL; + *found_offset = 0; + for (p = packed_git; p; p = p->next) { off_t offset = find_pack_entry_one(sha1, p); if (offset) { - if (!found_pack) { + if (!*found_pack) { if (!is_pack_valid(p)) { warning("packfile %s cannot be accessed", p->pack_name); continue; } - found_offset = offset; - found_pack = p; + *found_offset = offset; + *found_pack = p; } if (exclude) - break; + return 1; if (incremental) return 0; if (local && !p->pack_local) @@@ -968,21 -945,14 +968,21 @@@ } } - if (nr_objects >= nr_alloc) { - nr_alloc = (nr_alloc + 1024) * 3 / 2; - objects = xrealloc(objects, nr_alloc * sizeof(*entry)); - } + return 1; +} + +static void create_object_entry(const unsigned char *sha1, + enum object_type type, + uint32_t hash, + int exclude, + int no_try_delta, + uint32_t index_pos, + struct packed_git *found_pack, + off_t found_offset) +{ + struct object_entry *entry; - entry = objects + nr_objects++; - memset(entry, 0, sizeof(*entry)); - hashcpy(entry->idx.sha1, sha1); + entry = packlist_alloc(&to_pack, sha1, index_pos); entry->hash = hash; if (type) entry->type = type; @@@ -995,43 -965,16 +995,43 @@@ entry->in_pack_offset = found_offset; } - if (object_ix_hashsz * 3 <= nr_objects * 4) - rehash_objects(); - else - object_ix[-1 - ix] = nr_objects; + entry->no_try_delta = no_try_delta; +} + +static int add_object_entry(const unsigned char *sha1, enum object_type type, + const char *name, int exclude) +{ + struct packed_git *found_pack; + off_t found_offset; + uint32_t index_pos; - display_progress(progress_state, nr_objects); + if (have_duplicate_entry(sha1, exclude, &index_pos)) + return 0; + + if (!want_object_in_pack(sha1, exclude, &found_pack, &found_offset)) + return 0; - if (name && no_try_delta(name)) - entry->no_try_delta = 1; + create_object_entry(sha1, type, pack_name_hash(name), + exclude, name && no_try_delta(name), + index_pos, found_pack, found_offset); + display_progress(progress_state, to_pack.nr_objects); + return 1; +} + +static int add_object_entry_from_bitmap(const unsigned char *sha1, + enum object_type type, + int flags, uint32_t name_hash, + struct packed_git *pack, off_t offset) +{ + uint32_t index_pos; + + if (have_duplicate_entry(sha1, 0, &index_pos)) + return 0; + + create_object_entry(sha1, type, name_hash, 0, 0, index_pos, pack, offset); + + display_progress(progress_state, to_pack.nr_objects); return 1; } @@@ -1213,12 -1156,9 +1213,9 @@@ static int check_pbase_path(unsigned ha if (0 <= pos) return 1; pos = -pos - 1; - if (done_pbase_paths_alloc <= done_pbase_paths_num) { - done_pbase_paths_alloc = alloc_nr(done_pbase_paths_alloc); - done_pbase_paths = xrealloc(done_pbase_paths, - done_pbase_paths_alloc * - sizeof(unsigned)); - } + ALLOC_GROW(done_pbase_paths, + done_pbase_paths_num + 1, + done_pbase_paths_alloc); done_pbase_paths_num++; if (pos < done_pbase_paths_num) memmove(done_pbase_paths + pos + 1, @@@ -1232,7 -1172,7 +1229,7 @@@ static void add_preferred_base_object(c { struct pbase_tree *it; int cmplen; - unsigned hash = name_hash(name); + unsigned hash = pack_name_hash(name); if (!num_preferred_base || check_pbase_path(hash)) return; @@@ -1384,7 -1324,7 +1381,7 @@@ static void check_object(struct object_ break; } - if (base_ref && (base_entry = locate_object_entry(base_ref))) { + if (base_ref && (base_entry = packlist_find(&to_pack, base_ref, NULL))) { /* * If base_ref was set above that means we wish to * reuse delta data, and we even found that base @@@ -1458,12 -1398,12 +1455,12 @@@ static void get_object_details(void uint32_t i; struct object_entry **sorted_by_offset; - sorted_by_offset = xcalloc(nr_objects, sizeof(struct object_entry *)); - for (i = 0; i < nr_objects; i++) - sorted_by_offset[i] = objects + i; - qsort(sorted_by_offset, nr_objects, sizeof(*sorted_by_offset), pack_offset_sort); + sorted_by_offset = xcalloc(to_pack.nr_objects, sizeof(struct object_entry *)); + for (i = 0; i < to_pack.nr_objects; i++) + sorted_by_offset[i] = to_pack.objects + i; + qsort(sorted_by_offset, to_pack.nr_objects, sizeof(*sorted_by_offset), pack_offset_sort); - for (i = 0; i < nr_objects; i++) { + for (i = 0; i < to_pack.nr_objects; i++) { struct object_entry *entry = sorted_by_offset[i]; check_object(entry); if (big_file_threshold < entry->size) @@@ -2089,7 -2029,7 +2086,7 @@@ static int add_ref_tag(const char *path if (starts_with(path, "refs/tags/") && /* is a tag? */ !peel_ref(path, peeled) && /* peelable? */ - locate_object_entry(peeled)) /* object packed? */ + packlist_find(&to_pack, peeled, NULL)) /* object packed? */ add_object_entry(sha1, OBJ_TAG, NULL, 0); return 0; } @@@ -2112,14 -2052,14 +2109,14 @@@ static void prepare_pack(int window, in if (!pack_to_stdout) do_check_packed_object_crc = 1; - if (!nr_objects || !window || !depth) + if (!to_pack.nr_objects || !window || !depth) return; - delta_list = xmalloc(nr_objects * sizeof(*delta_list)); + delta_list = xmalloc(to_pack.nr_objects * sizeof(*delta_list)); nr_deltas = n = 0; - for (i = 0; i < nr_objects; i++) { - struct object_entry *entry = objects + i; + for (i = 0; i < to_pack.nr_objects; i++) { + struct object_entry *entry = to_pack.objects + i; if (entry->delta) /* This happens if we decided to reuse existing @@@ -2154,7 -2094,7 +2151,7 @@@ if (nr_deltas && n > 1) { unsigned nr_done = 0; if (progress) - progress_state = start_progress("Compressing objects", + progress_state = start_progress(_("Compressing objects"), nr_deltas); qsort(delta_list, n, sizeof(*delta_list), type_size_sort); ll_find_deltas(delta_list, n, window+1, depth, &nr_done); @@@ -2197,20 -2137,6 +2194,20 @@@ static int git_pack_config(const char * cache_max_small_delta_size = git_config_int(k, v); return 0; } + if (!strcmp(k, "pack.writebitmaps")) { + write_bitmap_index = git_config_bool(k, v); + return 0; + } + if (!strcmp(k, "pack.writebitmaphashcache")) { + if (git_config_bool(k, v)) + write_bitmap_options |= BITMAP_OPT_HASH_CACHE; + else + write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE; + } + if (!strcmp(k, "pack.usebitmaps")) { + use_bitmap_index = git_config_bool(k, v); + return 0; + } if (!strcmp(k, "pack.threads")) { delta_search_threads = git_config_int(k, v); if (delta_search_threads < 0) @@@ -2269,9 -2195,6 +2266,9 @@@ static void show_commit(struct commit * { add_object_entry(commit->object.sha1, OBJ_COMMIT, NULL, 0); commit->object.flags |= OBJECT_ADDED; + + if (write_bitmap_index) + index_commit_for_bitmap(commit); } static void show_object(struct object *obj, @@@ -2414,7 -2337,7 +2411,7 @@@ static void loosen_unused_packed_object for (i = 0; i < p->num_objects; i++) { sha1 = nth_packed_object_sha1(p, i); - if (!locate_object_entry(sha1) && + if (!packlist_find(&to_pack, sha1, NULL) && !has_sha1_pack_kept_or_nonlocal(sha1)) if (force_object_loose(sha1, p->mtime)) die("unable to force loose object"); @@@ -2422,29 -2345,6 +2419,29 @@@ } } +static int get_object_list_from_bitmap(struct rev_info *revs) +{ + if (prepare_bitmap_walk(revs) < 0) + return -1; + + if (!reuse_partial_packfile_from_bitmap( + &reuse_packfile, + &reuse_packfile_objects, + &reuse_packfile_offset)) { + assert(reuse_packfile_objects); + nr_result += reuse_packfile_objects; + + if (progress) { + fprintf(stderr, "Reusing existing pack: %d, done.\n", + reuse_packfile_objects); + fflush(stderr); + } + } + + traverse_bitmap_commit_list(&add_object_entry_from_bitmap); + return 0; +} + static void get_object_list(int ac, const char **av) { struct rev_info revs; @@@ -2464,7 -2364,6 +2461,7 @@@ if (*line == '-') { if (!strcmp(line, "--not")) { flags ^= UNINTERESTING; + write_bitmap_index = 0; continue; } die("not a rev '%s'", line); @@@ -2473,9 -2372,6 +2470,9 @@@ die("bad revision '%s'", line); } + if (use_bitmap_index && !get_object_list_from_bitmap(&revs)) + return; + if (prepare_revision_walk(&revs)) die("revision walk setup failed"); mark_edges_uninteresting(&revs, show_edge); @@@ -2605,14 -2501,10 +2602,14 @@@ int cmd_pack_objects(int argc, const ch N_("pack compression level")), OPT_SET_INT(0, "keep-true-parents", &grafts_replace_parents, N_("do not hide commits by grafts"), 0), + OPT_BOOL(0, "use-bitmap-index", &use_bitmap_index, + N_("use a bitmap index if available to speed up counting objects")), + OPT_BOOL(0, "write-bitmap-index", &write_bitmap_index, + N_("write a bitmap index together with the pack index")), OPT_END(), }; - read_replace_refs = 0; + check_replace_refs = 0; reset_pack_idx_option(&pack_idx_opts); git_config(git_pack_config, NULL); @@@ -2675,19 -2567,13 +2672,19 @@@ if (keep_unreachable && unpack_unreachable) die("--keep-unreachable and --unpack-unreachable are incompatible."); + if (!use_internal_rev_list || !pack_to_stdout || is_repository_shallow()) + use_bitmap_index = 0; + + if (pack_to_stdout || !rev_list_all) + write_bitmap_index = 0; + if (progress && all_progress_implied) progress = 2; prepare_packed_git(); if (progress) - progress_state = start_progress("Counting objects", 0); + progress_state = start_progress(_("Counting objects"), 0); if (!use_internal_rev_list) read_object_list_from_stdin(); else { diff --combined commit.c index a263947383,e0043144b1..0f28902bc3 --- a/commit.c +++ b/commit.c @@@ -10,7 -10,6 +10,7 @@@ #include "mergesort.h" #include "commit-slab.h" #include "prio-queue.h" +#include "sha1-lookup.h" static struct commit_extra_header *read_commit_extra_header_lines(const char *buf, size_t len, const char **); @@@ -115,16 -114,23 +115,16 @@@ static unsigned long parse_commit_date( static struct commit_graft **commit_graft; static int commit_graft_alloc, commit_graft_nr; +static const unsigned char *commit_graft_sha1_access(size_t index, void *table) +{ + struct commit_graft **commit_graft_table = table; + return commit_graft_table[index]->sha1; +} + static int commit_graft_pos(const unsigned char *sha1) { - int lo, hi; - lo = 0; - hi = commit_graft_nr; - while (lo < hi) { - int mi = (lo + hi) / 2; - struct commit_graft *graft = commit_graft[mi]; - int cmp = hashcmp(sha1, graft->sha1); - if (!cmp) - return mi; - if (cmp < 0) - hi = mi; - else - lo = mi + 1; - } - return -lo - 1; + return sha1_pos(sha1, commit_graft, commit_graft_nr, + commit_graft_sha1_access); } int register_commit_graft(struct commit_graft *graft, int ignore_dups) @@@ -141,12 -147,8 +141,8 @@@ return 1; } pos = -pos - 1; - if (commit_graft_alloc <= ++commit_graft_nr) { - commit_graft_alloc = alloc_nr(commit_graft_alloc); - commit_graft = xrealloc(commit_graft, - sizeof(*commit_graft) * - commit_graft_alloc); - } + ALLOC_GROW(commit_graft, commit_graft_nr + 1, commit_graft_alloc); + commit_graft_nr++; if (pos < commit_graft_nr) memmove(commit_graft + pos + 1, commit_graft + pos, @@@ -542,7 -544,7 +538,7 @@@ define_commit_slab(author_date_slab, un static void record_author_date(struct author_date_slab *author_date, struct commit *commit) { - const char *buf, *line_end; + const char *buf, *line_end, *ident_line; char *buffer = NULL; struct ident_split ident; char *date_end; @@@ -560,14 -562,14 +556,14 @@@ buf; buf = line_end + 1) { line_end = strchrnul(buf, '\n'); - if (!starts_with(buf, "author ")) { + ident_line = skip_prefix(buf, "author "); + if (!ident_line) { if (!line_end[0] || line_end[1] == '\n') return; /* end of header */ continue; } if (split_ident_line(&ident, - buf + strlen("author "), - line_end - (buf + strlen("author "))) || + ident_line, line_end - ident_line) || !ident.date_begin || !ident.date_end) goto fail_exit; /* malformed "author" line */ break; @@@ -1187,8 -1189,10 +1183,8 @@@ static void parse_gpg_output(struct sig for (i = 0; i < ARRAY_SIZE(sigcheck_gpg_status); i++) { const char *found, *next; - if (starts_with(buf, sigcheck_gpg_status[i].check + 1)) { - /* At the very beginning of the buffer */ - found = buf + strlen(sigcheck_gpg_status[i].check + 1); - } else { + found = skip_prefix(buf, sigcheck_gpg_status[i].check + 1); + if (!found) { found = strstr(buf, sigcheck_gpg_status[i].check); if (!found) continue; diff --combined diff.c index e9a8874d06,f5f0fd1e7f..9ae8116fca --- a/diff.c +++ b/diff.c @@@ -1361,11 -1361,7 +1361,7 @@@ static struct diffstat_file *diffstat_a { struct diffstat_file *x; x = xcalloc(sizeof (*x), 1); - if (diffstat->nr == diffstat->alloc) { - diffstat->alloc = alloc_nr(diffstat->alloc); - diffstat->files = xrealloc(diffstat->files, - diffstat->alloc * sizeof(x)); - } + ALLOC_GROW(diffstat->files, diffstat->nr + 1, diffstat->alloc); diffstat->files[diffstat->nr++] = x; if (name_b) { x->from_name = xstrdup(name_a); @@@ -2845,9 -2841,8 +2841,9 @@@ static struct diff_tempfile *prepare_te remove_tempfile_installed = 1; } - if (!one->sha1_valid || - reuse_worktree_file(name, one->sha1, 1)) { + if (!S_ISGITLINK(one->mode) && + (!one->sha1_valid || + reuse_worktree_file(name, one->sha1, 1))) { struct stat st; if (lstat(name, &st) < 0) { if (errno == ENOENT) @@@ -3600,6 -3595,14 +3596,6 @@@ static int parse_diff_filter_opt(const return 0; } -/* Used only by "diff-files" and "diff --no-index" */ -void handle_deprecated_show_diff_q(struct diff_options *opt) -{ - warning("'diff -q' and 'diff-files -q' are deprecated."); - warning("Use 'diff --diff-filter=d' instead to ignore deleted filepairs."); - parse_diff_filter_opt("d", opt); -} - static void enable_patch_output(int *fmt) { *fmt &= ~DIFF_FORMAT_NO_OUTPUT; *fmt |= DIFF_FORMAT_PATCH; @@@ -3958,11 -3961,7 +3954,7 @@@ struct diff_queue_struct diff_queued_di void diff_q(struct diff_queue_struct *queue, struct diff_filepair *dp) { - if (queue->alloc <= queue->nr) { - queue->alloc = alloc_nr(queue->alloc); - queue->queue = xrealloc(queue->queue, - sizeof(dp) * queue->alloc); - } + ALLOC_GROW(queue->queue, queue->nr + 1, queue->alloc); queue->queue[queue->nr++] = dp; } @@@ -4690,38 -4689,6 +4682,38 @@@ static int diff_filespec_is_identical(s return !memcmp(one->data, two->data, one->size); } +static int diff_filespec_check_stat_unmatch(struct diff_filepair *p) +{ + if (p->done_skip_stat_unmatch) + return p->skip_stat_unmatch_result; + + p->done_skip_stat_unmatch = 1; + p->skip_stat_unmatch_result = 0; + /* + * 1. Entries that come from stat info dirtiness + * always have both sides (iow, not create/delete), + * one side of the object name is unknown, with + * the same mode and size. Keep the ones that + * do not match these criteria. They have real + * differences. + * + * 2. At this point, the file is known to be modified, + * with the same mode and size, and the object + * name of one side is unknown. Need to inspect + * the identical contents. + */ + if (!DIFF_FILE_VALID(p->one) || /* (1) */ + !DIFF_FILE_VALID(p->two) || + (p->one->sha1_valid && p->two->sha1_valid) || + (p->one->mode != p->two->mode) || + diff_populate_filespec(p->one, 1) || + diff_populate_filespec(p->two, 1) || + (p->one->size != p->two->size) || + !diff_filespec_is_identical(p->one, p->two)) /* (2) */ + p->skip_stat_unmatch_result = 1; + return p->skip_stat_unmatch_result; +} + static void diffcore_skip_stat_unmatch(struct diff_options *diffopt) { int i; @@@ -4732,7 -4699,27 +4724,7 @@@ for (i = 0; i < q->nr; i++) { struct diff_filepair *p = q->queue[i]; - /* - * 1. Entries that come from stat info dirtiness - * always have both sides (iow, not create/delete), - * one side of the object name is unknown, with - * the same mode and size. Keep the ones that - * do not match these criteria. They have real - * differences. - * - * 2. At this point, the file is known to be modified, - * with the same mode and size, and the object - * name of one side is unknown. Need to inspect - * the identical contents. - */ - if (!DIFF_FILE_VALID(p->one) || /* (1) */ - !DIFF_FILE_VALID(p->two) || - (p->one->sha1_valid && p->two->sha1_valid) || - (p->one->mode != p->two->mode) || - diff_populate_filespec(p->one, 1) || - diff_populate_filespec(p->two, 1) || - (p->one->size != p->two->size) || - !diff_filespec_is_identical(p->one, p->two)) /* (2) */ + if (diff_filespec_check_stat_unmatch(p)) diff_q(&outq, p); else { /* @@@ -4895,7 -4882,6 +4887,7 @@@ void diff_change(struct diff_options *o unsigned old_dirty_submodule, unsigned new_dirty_submodule) { struct diff_filespec *one, *two; + struct diff_filepair *p; if (S_ISGITLINK(old_mode) && S_ISGITLINK(new_mode) && is_submodule_ignored(concatpath, options)) @@@ -4922,16 -4908,10 +4914,16 @@@ fill_filespec(two, new_sha1, new_sha1_valid, new_mode); one->dirty_submodule = old_dirty_submodule; two->dirty_submodule = new_dirty_submodule; + p = diff_queue(&diff_queued_diff, one, two); - diff_queue(&diff_queued_diff, one, two); - if (!DIFF_OPT_TST(options, DIFF_FROM_CONTENTS)) - DIFF_OPT_SET(options, HAS_CHANGES); + if (DIFF_OPT_TST(options, DIFF_FROM_CONTENTS)) + return; + + if (DIFF_OPT_TST(options, QUICK) && options->skip_stat_unmatch && + !diff_filespec_check_stat_unmatch(p)) + return; + + DIFF_OPT_SET(options, HAS_CHANGES); } struct diff_filepair *diff_unmerge(struct diff_options *options, const char *path) diff --combined diffcore-rename.c index 39effecd84,f54d5bf479..749a35d2c2 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@@ -4,7 -4,7 +4,7 @@@ #include "cache.h" #include "diff.h" #include "diffcore.h" -#include "hash.h" +#include "hashmap.h" #include "progress.h" /* Table of rename/copy destinations */ @@@ -38,11 -38,7 +38,7 @@@ static struct diff_rename_dst *locate_r if (!insert_ok) return NULL; /* insert to make it at "first" */ - if (rename_dst_alloc <= rename_dst_nr) { - rename_dst_alloc = alloc_nr(rename_dst_alloc); - rename_dst = xrealloc(rename_dst, - rename_dst_alloc * sizeof(*rename_dst)); - } + ALLOC_GROW(rename_dst, rename_dst_nr + 1, rename_dst_alloc); rename_dst_nr++; if (first < rename_dst_nr) memmove(rename_dst + first + 1, rename_dst + first, @@@ -82,11 -78,7 +78,7 @@@ static struct diff_rename_src *register } /* insert to make it at "first" */ - if (rename_src_alloc <= rename_src_nr) { - rename_src_alloc = alloc_nr(rename_src_alloc); - rename_src = xrealloc(rename_src, - rename_src_alloc * sizeof(*rename_src)); - } + ALLOC_GROW(rename_src, rename_src_nr + 1, rename_src_alloc); rename_src_nr++; if (first < rename_src_nr) memmove(rename_src + first + 1, rename_src + first, @@@ -243,82 -235,137 +235,82 @@@ static int score_compare(const void *a_ } struct file_similarity { - int src_dst, index; + struct hashmap_entry entry; + int index; struct diff_filespec *filespec; - struct file_similarity *next; }; -static int find_identical_files(struct file_similarity *src, - struct file_similarity *dst, +static unsigned int hash_filespec(struct diff_filespec *filespec) +{ + unsigned int hash; + if (!filespec->sha1_valid) { + if (diff_populate_filespec(filespec, 0)) + return 0; + hash_sha1_file(filespec->data, filespec->size, "blob", filespec->sha1); + } + memcpy(&hash, filespec->sha1, sizeof(hash)); + return hash; +} + +static int find_identical_files(struct hashmap *srcs, + int dst_index, struct diff_options *options) { int renames = 0; + struct diff_filespec *target = rename_dst[dst_index].two; + struct file_similarity *p, *best, dst; + int i = 100, best_score = -1; + /* - * Walk over all the destinations ... + * Find the best source match for specified destination. */ - do { - struct diff_filespec *target = dst->filespec; - struct file_similarity *p, *best; - int i = 100, best_score = -1; - - /* - * .. to find the best source match - */ - best = NULL; - for (p = src; p; p = p->next) { - int score; - struct diff_filespec *source = p->filespec; - - /* False hash collision? */ - if (hashcmp(source->sha1, target->sha1)) - continue; - /* Non-regular files? If so, the modes must match! */ - if (!S_ISREG(source->mode) || !S_ISREG(target->mode)) { - if (source->mode != target->mode) - continue; - } - /* Give higher scores to sources that haven't been used already */ - score = !source->rename_used; - if (source->rename_used && options->detect_rename != DIFF_DETECT_COPY) + best = NULL; + hashmap_entry_init(&dst, hash_filespec(target)); + for (p = hashmap_get(srcs, &dst, NULL); p; p = hashmap_get_next(srcs, p)) { + int score; + struct diff_filespec *source = p->filespec; + + /* False hash collision? */ + if (hashcmp(source->sha1, target->sha1)) + continue; + /* Non-regular files? If so, the modes must match! */ + if (!S_ISREG(source->mode) || !S_ISREG(target->mode)) { + if (source->mode != target->mode) continue; - score += basename_same(source, target); - if (score > best_score) { - best = p; - best_score = score; - if (score == 2) - break; - } - - /* Too many identical alternatives? Pick one */ - if (!--i) - break; } - if (best) { - record_rename_pair(dst->index, best->index, MAX_SCORE); - renames++; + /* Give higher scores to sources that haven't been used already */ + score = !source->rename_used; + if (source->rename_used && options->detect_rename != DIFF_DETECT_COPY) + continue; + score += basename_same(source, target); + if (score > best_score) { + best = p; + best_score = score; + if (score == 2) + break; } - } while ((dst = dst->next) != NULL); - return renames; -} -static void free_similarity_list(struct file_similarity *p) -{ - while (p) { - struct file_similarity *entry = p; - p = p->next; - free(entry); + /* Too many identical alternatives? Pick one */ + if (!--i) + break; } -} - -static int find_same_files(void *ptr, void *data) -{ - int ret; - struct file_similarity *p = ptr; - struct file_similarity *src = NULL, *dst = NULL; - struct diff_options *options = data; - - /* Split the hash list up into sources and destinations */ - do { - struct file_similarity *entry = p; - p = p->next; - if (entry->src_dst < 0) { - entry->next = src; - src = entry; - } else { - entry->next = dst; - dst = entry; - } - } while (p); - - /* - * If we have both sources *and* destinations, see if - * we can match them up - */ - ret = (src && dst) ? find_identical_files(src, dst, options) : 0; - - /* Free the hashes and return the number of renames found */ - free_similarity_list(src); - free_similarity_list(dst); - return ret; -} - -static unsigned int hash_filespec(struct diff_filespec *filespec) -{ - unsigned int hash; - if (!filespec->sha1_valid) { - if (diff_populate_filespec(filespec, 0)) - return 0; - hash_sha1_file(filespec->data, filespec->size, "blob", filespec->sha1); + if (best) { + record_rename_pair(dst_index, best->index, MAX_SCORE); + renames++; } - memcpy(&hash, filespec->sha1, sizeof(hash)); - return hash; + return renames; } -static void insert_file_table(struct hash_table *table, int src_dst, int index, struct diff_filespec *filespec) +static void insert_file_table(struct hashmap *table, int index, struct diff_filespec *filespec) { - void **pos; - unsigned int hash; struct file_similarity *entry = xmalloc(sizeof(*entry)); - entry->src_dst = src_dst; entry->index = index; entry->filespec = filespec; - entry->next = NULL; - hash = hash_filespec(filespec); - pos = insert_hash(hash, entry, table); - - /* We already had an entry there? */ - if (pos) { - entry->next = *pos; - *pos = entry; - } + hashmap_entry_init(entry, hash_filespec(filespec)); + hashmap_add(table, entry); } /* @@@ -330,22 -377,24 +322,22 @@@ */ static int find_exact_renames(struct diff_options *options) { - int i; - struct hash_table file_table; + int i, renames = 0; + struct hashmap file_table; - init_hash(&file_table); - preallocate_hash(&file_table, rename_src_nr + rename_dst_nr); + /* Add all sources to the hash table */ + hashmap_init(&file_table, NULL, rename_src_nr); for (i = 0; i < rename_src_nr; i++) - insert_file_table(&file_table, -1, i, rename_src[i].p->one); + insert_file_table(&file_table, i, rename_src[i].p->one); + /* Walk the destinations and find best source match */ for (i = 0; i < rename_dst_nr; i++) - insert_file_table(&file_table, 1, i, rename_dst[i].two); + renames += find_identical_files(&file_table, i, options); - /* Find the renames */ - i = for_each_hash(&file_table, find_same_files, options); + /* Free the hash data structure and entries */ + hashmap_free(&file_table, 1); - /* .. and free the hash data structure */ - free_hash(&file_table); - - return i; + return renames; } #define NUM_CANDIDATE_PER_DST 4 @@@ -522,7 -571,7 +514,7 @@@ void diffcore_rename(struct diff_option if (options->show_rename_progress) { progress = start_progress_delay( - "Performing inexact rename detection", + _("Performing inexact rename detection"), rename_dst_nr * rename_src_nr, 50, 1); } diff --combined dir.c index 70eefdd350,f6c647006b..99f53033ba --- a/dir.c +++ b/dir.c @@@ -49,9 -49,7 +49,9 @@@ int strncmp_icase(const char *a, const int fnmatch_icase(const char *pattern, const char *string, int flags) { - return fnmatch(pattern, string, flags | (ignore_case ? FNM_CASEFOLD : 0)); + return wildmatch(pattern, string, + flags | (ignore_case ? WM_CASEFOLD : 0), + NULL); } inline int git_fnmatch(const struct pathspec_item *item, @@@ -60,7 -58,7 +60,7 @@@ { if (prefix > 0) { if (ps_strncmp(item, pattern, string, prefix)) - return FNM_NOMATCH; + return WM_NOMATCH; pattern += prefix; string += prefix; } @@@ -78,9 -76,8 +78,9 @@@ NULL); else /* wildmatch has not learned no FNM_PATHNAME mode yet */ - return fnmatch(pattern, string, - item->magic & PATHSPEC_ICASE ? FNM_CASEFOLD : 0); + return wildmatch(pattern, string, + item->magic & PATHSPEC_ICASE ? WM_CASEFOLD : 0, + NULL); } static int fnmatch_icase_mem(const char *pattern, int patternlen, @@@ -198,9 -195,6 +198,9 @@@ int within_depth(const char *name, int return 1; } +#define DO_MATCH_EXCLUDE 1 +#define DO_MATCH_DIRECTORY 2 + /* * Does 'match' match the given name? * A match is found if @@@ -214,7 -208,7 +214,7 @@@ * It returns 0 when there is no match. */ static int match_pathspec_item(const struct pathspec_item *item, int prefix, - const char *name, int namelen) + const char *name, int namelen, unsigned flags) { /* name/namelen has prefix cut off by caller */ const char *match = item->match + prefix; @@@ -224,7 -218,7 +224,7 @@@ * The normal call pattern is: * 1. prefix = common_prefix_len(ps); * 2. prune something, or fill_directory - * 3. match_pathspec_depth() + * 3. match_pathspec() * * 'prefix' at #1 may be shorter than the command's prefix and * it's ok for #2 to match extra files. Those extras will be @@@ -263,11 -257,7 +263,11 @@@ if (match[matchlen-1] == '/' || name[matchlen] == '/') return MATCHED_RECURSIVELY; - } + } else if ((flags & DO_MATCH_DIRECTORY) && + match[matchlen - 1] == '/' && + namelen == matchlen - 1 && + !ps_strncmp(item, match, name, namelen)) + return MATCHED_EXACTLY; if (item->nowildcard_len < item->len && !git_fnmatch(item, match, name, @@@ -292,12 -282,12 +292,12 @@@ * pathspec did not match any names, which could indicate that the * user mistyped the nth pathspec. */ -static int match_pathspec_depth_1(const struct pathspec *ps, - const char *name, int namelen, - int prefix, char *seen, - int exclude) +static int do_match_pathspec(const struct pathspec *ps, + const char *name, int namelen, + int prefix, char *seen, + unsigned flags) { - int i, retval = 0; + int i, retval = 0, exclude = flags & DO_MATCH_EXCLUDE; GUARD_PATHSPEC(ps, PATHSPEC_FROMTOP | @@@ -337,8 -327,7 +337,8 @@@ */ if (seen && ps->items[i].magic & PATHSPEC_EXCLUDE) seen[i] = MATCHED_FNMATCH; - how = match_pathspec_item(ps->items+i, prefix, name, namelen); + how = match_pathspec_item(ps->items+i, prefix, name, + namelen, flags); if (ps->recursive && (ps->magic & PATHSPEC_MAXDEPTH) && ps->max_depth != -1 && @@@ -361,19 -350,15 +361,19 @@@ return retval; } -int match_pathspec_depth(const struct pathspec *ps, - const char *name, int namelen, - int prefix, char *seen) +int match_pathspec(const struct pathspec *ps, + const char *name, int namelen, + int prefix, char *seen, int is_dir) { int positive, negative; - positive = match_pathspec_depth_1(ps, name, namelen, prefix, seen, 0); + unsigned flags = is_dir ? DO_MATCH_DIRECTORY : 0; + positive = do_match_pathspec(ps, name, namelen, + prefix, seen, flags); if (!(ps->magic & PATHSPEC_EXCLUDE) || !positive) return positive; - negative = match_pathspec_depth_1(ps, name, namelen, prefix, seen, 1); + negative = do_match_pathspec(ps, name, namelen, + prefix, seen, + flags | DO_MATCH_EXCLUDE); return negative ? 0 : positive; } @@@ -506,25 -491,6 +506,25 @@@ void clear_exclude_list(struct exclude_ el->filebuf = NULL; } +static void trim_trailing_spaces(char *buf) +{ + int i, last_space = -1, nr_spaces, len = strlen(buf); + for (i = 0; i < len; i++) + if (buf[i] == '\\') + i++; + else if (buf[i] == ' ') { + if (last_space == -1) { + last_space = i; + nr_spaces = 1; + } else + nr_spaces++; + } else + last_space = -1; + + if (last_space != -1 && last_space + nr_spaces == len) + buf[last_space] = '\0'; +} + int add_excludes_from_file_to_list(const char *fname, const char *base, int baselen, @@@ -576,7 -542,6 +576,7 @@@ if (buf[i] == '\n') { if (entry != buf + i && entry[0] != '#') { buf[i - (i && buf[i-1] == '\r')] = 0; + trim_trailing_spaces(entry); add_exclude(entry, base, baselen, el, lineno); } lineno++; @@@ -1364,10 -1329,7 +1364,7 @@@ static struct path_simplify *create_sim for (nr = 0 ; ; nr++) { const char *match; - if (nr >= alloc) { - alloc = alloc_nr(alloc); - simplify = xrealloc(simplify, alloc * sizeof(*simplify)); - } + ALLOC_GROW(simplify, nr + 1, alloc); match = *pathspec++; if (!match) break; diff --combined read-cache.c index f23c44a81a,40dbec92ac..ba13353b37 --- a/read-cache.c +++ b/read-cache.c @@@ -15,8 -15,7 +15,8 @@@ #include "strbuf.h" #include "varint.h" -static struct cache_entry *refresh_cache_entry(struct cache_entry *ce, int really); +static struct cache_entry *refresh_cache_entry(struct cache_entry *ce, + unsigned int options); /* Mask for the name length in ce_flags in the on-disk index */ @@@ -48,7 -47,6 +48,7 @@@ static void replace_index_entry(struct struct cache_entry *old = istate->cache[nr]; remove_name_hash(istate, old); + free(old); set_index_entry(istate, nr, ce); istate->cache_changed = 1; } @@@ -60,7 -58,7 +60,7 @@@ void rename_index_entry_at(struct index new = xmalloc(cache_entry_size(namelen)); copy_cache_entry(new, old); - new->ce_flags &= ~CE_STATE_MASK; + new->ce_flags &= ~CE_HASHED; new->ce_namelen = namelen; memcpy(new->name, new_name, namelen + 1); @@@ -480,7 -478,6 +480,7 @@@ int remove_index_entry_at(struct index_ record_resolve_undo(istate, ce); remove_name_hash(istate, ce); + free(ce); istate->cache_changed = 1; istate->cache_nr--; if (pos >= istate->cache_nr) @@@ -502,10 -499,8 +502,10 @@@ void remove_marked_cache_entries(struc unsigned int i, j; for (i = j = 0; i < istate->cache_nr; i++) { - if (ce_array[i]->ce_flags & CE_REMOVE) + if (ce_array[i]->ce_flags & CE_REMOVE) { remove_name_hash(istate, ce_array[i]); + free(ce_array[i]); + } else ce_array[j++] = ce_array[i]; } @@@ -584,7 -579,7 +584,7 @@@ static struct cache_entry *create_alias return new; } -static void record_intent_to_add(struct cache_entry *ce) +void set_object_name_for_intent_to_add_entry(struct cache_entry *ce) { unsigned char sha1[20]; if (write_sha1_file("", 0, blob_type, sha1)) @@@ -670,7 -665,7 +670,7 @@@ int add_to_index(struct index_state *is if (index_path(ce->sha1, path, st, HASH_WRITE_OBJECT)) return error("unable to index file %s", path); } else - record_intent_to_add(ce); + set_object_name_for_intent_to_add_entry(ce); if (ignore_case && alias && different_name(ce, alias)) ce = create_alias_ce(ce, alias); @@@ -701,7 -696,7 +701,7 @@@ int add_file_to_index(struct index_stat struct cache_entry *make_cache_entry(unsigned int mode, const unsigned char *sha1, const char *path, int stage, - int refresh) + unsigned int refresh_options) { int size, len; struct cache_entry *ce; @@@ -721,7 -716,10 +721,7 @@@ ce->ce_namelen = len; ce->ce_mode = create_ce_mode(mode); - if (refresh) - return refresh_cache_entry(ce, 0); - - return ce; + return refresh_cache_entry(ce, refresh_options); } int ce_same_name(const struct cache_entry *a, const struct cache_entry *b) @@@ -730,6 -728,11 +730,6 @@@ return ce_namelen(b) == len && !memcmp(a->name, b->name, len); } -int ce_path_match(const struct cache_entry *ce, const struct pathspec *pathspec) -{ - return match_pathspec_depth(pathspec, ce->name, ce_namelen(ce), 0, NULL); -} - /* * We fundamentally don't like some paths: we don't want * dot or dot-dot anywhere, and for obvious reasons don't @@@ -990,11 -993,7 +990,7 @@@ int add_index_entry(struct index_state } /* Make sure the array is big enough .. */ - if (istate->cache_nr == istate->cache_alloc) { - istate->cache_alloc = alloc_nr(istate->cache_alloc); - istate->cache = xrealloc(istate->cache, - istate->cache_alloc * sizeof(*istate->cache)); - } + ALLOC_GROW(istate->cache, istate->cache_nr + 1, istate->cache_alloc); /* Add it in.. */ istate->cache_nr++; @@@ -1026,12 -1025,10 +1022,12 @@@ static struct cache_entry *refresh_cach struct stat st; struct cache_entry *updated; int changed, size; + int refresh = options & CE_MATCH_REFRESH; int ignore_valid = options & CE_MATCH_IGNORE_VALID; int ignore_skip_worktree = options & CE_MATCH_IGNORE_SKIP_WORKTREE; + int ignore_missing = options & CE_MATCH_IGNORE_MISSING; - if (ce_uptodate(ce)) + if (!refresh || ce_uptodate(ce)) return ce; /* @@@ -1049,8 -1046,6 +1045,8 @@@ } if (lstat(ce->name, &st) < 0) { + if (ignore_missing && errno == ENOENT) + return ce; if (err) *err = errno; return NULL; @@@ -1128,9 -1123,7 +1124,9 @@@ int refresh_index(struct index_state *i int ignore_submodules = (flags & REFRESH_IGNORE_SUBMODULES) != 0; int first = 1; int in_porcelain = (flags & REFRESH_IN_PORCELAIN); - unsigned int options = really ? CE_MATCH_IGNORE_VALID : 0; + unsigned int options = (CE_MATCH_REFRESH | + (really ? CE_MATCH_IGNORE_VALID : 0) | + (not_new ? CE_MATCH_IGNORE_MISSING : 0)); const char *modified_fmt; const char *deleted_fmt; const char *typechange_fmt; @@@ -1152,7 -1145,8 +1148,7 @@@ if (ignore_submodules && S_ISGITLINK(ce->ce_mode)) continue; - if (pathspec && - !match_pathspec_depth(pathspec, ce->name, ce_namelen(ce), 0, seen)) + if (pathspec && !ce_path_match(ce, pathspec, seen)) filtered = 1; if (ce_stage(ce)) { @@@ -1178,6 -1172,8 +1174,6 @@@ if (!new) { const char *fmt; - if (not_new && cache_errno == ENOENT) - continue; if (really && cache_errno == EINVAL) { /* If we are doing --really-refresh that * means the index is not valid anymore. @@@ -1207,10 -1203,9 +1203,10 @@@ return has_errors; } -static struct cache_entry *refresh_cache_entry(struct cache_entry *ce, int really) +static struct cache_entry *refresh_cache_entry(struct cache_entry *ce, + unsigned int options) { - return refresh_cache_ent(&the_index, ce, really, NULL, NULL); + return refresh_cache_ent(&the_index, ce, options, NULL, NULL); } @@@ -1220,42 -1215,6 +1216,42 @@@ #define INDEX_FORMAT_DEFAULT 3 +static int index_format_config(const char *var, const char *value, void *cb) +{ + unsigned int *version = cb; + if (!strcmp(var, "index.version")) { + *version = git_config_int(var, value); + return 0; + } + return 1; +} + +static unsigned int get_index_format_default(void) +{ + char *envversion = getenv("GIT_INDEX_VERSION"); + char *endp; + unsigned int version = INDEX_FORMAT_DEFAULT; + + if (!envversion) { + git_config(index_format_config, &version); + if (version < INDEX_FORMAT_LB || INDEX_FORMAT_UB < version) { + warning(_("index.version set, but the value is invalid.\n" + "Using version %i"), INDEX_FORMAT_DEFAULT); + return INDEX_FORMAT_DEFAULT; + } + return version; + } + + version = strtoul(envversion, &endp, 10); + if (*endp || + version < INDEX_FORMAT_LB || INDEX_FORMAT_UB < version) { + warning(_("GIT_INDEX_VERSION set, but the value is invalid.\n" + "Using version %i"), INDEX_FORMAT_DEFAULT); + version = INDEX_FORMAT_DEFAULT; + } + return version; +} + /* * dev/ino/uid/gid/size are also just tracked to the low 32 bits * Again - this is just a (very strong in practice) heuristic that @@@ -1350,6 -1309,26 +1346,6 @@@ int read_index(struct index_state *ista return read_index_from(istate, get_index_file()); } -#ifndef NEEDS_ALIGNED_ACCESS -#define ntoh_s(var) ntohs(var) -#define ntoh_l(var) ntohl(var) -#else -static inline uint16_t ntoh_s_force_align(void *p) -{ - uint16_t x; - memcpy(&x, p, sizeof(x)); - return ntohs(x); -} -static inline uint32_t ntoh_l_force_align(void *p) -{ - uint32_t x; - memcpy(&x, p, sizeof(x)); - return ntohl(x); -} -#define ntoh_s(var) ntoh_s_force_align(&(var)) -#define ntoh_l(var) ntoh_l_force_align(&(var)) -#endif - static struct cache_entry *cache_entry_from_ondisk(struct ondisk_cache_entry *ondisk, unsigned int flags, const char *name, @@@ -1357,16 -1336,16 +1353,16 @@@ { struct cache_entry *ce = xmalloc(cache_entry_size(len)); - ce->ce_stat_data.sd_ctime.sec = ntoh_l(ondisk->ctime.sec); - ce->ce_stat_data.sd_mtime.sec = ntoh_l(ondisk->mtime.sec); - ce->ce_stat_data.sd_ctime.nsec = ntoh_l(ondisk->ctime.nsec); - ce->ce_stat_data.sd_mtime.nsec = ntoh_l(ondisk->mtime.nsec); - ce->ce_stat_data.sd_dev = ntoh_l(ondisk->dev); - ce->ce_stat_data.sd_ino = ntoh_l(ondisk->ino); - ce->ce_mode = ntoh_l(ondisk->mode); - ce->ce_stat_data.sd_uid = ntoh_l(ondisk->uid); - ce->ce_stat_data.sd_gid = ntoh_l(ondisk->gid); - ce->ce_stat_data.sd_size = ntoh_l(ondisk->size); + ce->ce_stat_data.sd_ctime.sec = get_be32(&ondisk->ctime.sec); + ce->ce_stat_data.sd_mtime.sec = get_be32(&ondisk->mtime.sec); + ce->ce_stat_data.sd_ctime.nsec = get_be32(&ondisk->ctime.nsec); + ce->ce_stat_data.sd_mtime.nsec = get_be32(&ondisk->mtime.nsec); + ce->ce_stat_data.sd_dev = get_be32(&ondisk->dev); + ce->ce_stat_data.sd_ino = get_be32(&ondisk->ino); + ce->ce_mode = get_be32(&ondisk->mode); + ce->ce_stat_data.sd_uid = get_be32(&ondisk->uid); + ce->ce_stat_data.sd_gid = get_be32(&ondisk->gid); + ce->ce_stat_data.sd_size = get_be32(&ondisk->size); ce->ce_flags = flags & ~CE_NAMEMASK; ce->ce_namelen = len; hashcpy(ce->sha1, ondisk->sha1); @@@ -1406,14 -1385,14 +1402,14 @@@ static struct cache_entry *create_from_ unsigned int flags; /* On-disk flags are just 16 bits */ - flags = ntoh_s(ondisk->flags); + flags = get_be16(&ondisk->flags); len = flags & CE_NAMEMASK; if (flags & CE_EXTENDED) { struct ondisk_cache_entry_extended *ondisk2; int extended_flags; ondisk2 = (struct ondisk_cache_entry_extended *)ondisk; - extended_flags = ntoh_s(ondisk2->flags2) << 16; + extended_flags = get_be16(&ondisk2->flags2) << 16; /* We do not yet understand any bit out of CE_EXTENDED_FLAGS */ if (extended_flags & ~CE_EXTENDED_FLAGS) die("Unknown index entry format %08x", extended_flags); @@@ -1812,7 -1791,7 +1808,7 @@@ int write_index(struct index_state *ist } if (!istate->version) - istate->version = INDEX_FORMAT_DEFAULT; + istate->version = get_index_format_default(); /* demote version 3 to version 2 when the latter suffices */ if (istate->version == 3 || istate->version == 2) @@@ -1911,7 -1890,7 +1907,7 @@@ int read_index_unmerged(struct index_st new_ce->ce_mode = ce->ce_mode; if (add_index_entry(istate, new_ce, 0)) return error("%s: cannot drop to stage #0", - ce->name); + new_ce->name); i = index_name_pos(istate, new_ce->name, len); } return unmerged; diff --combined replace_object.c index 4ee4c8d104,843deef599..0ab2dc1374 --- a/replace_object.c +++ b/replace_object.c @@@ -3,13 -3,8 +3,13 @@@ #include "refs.h" #include "commit.h" +/* + * An array of replacements. The array is kept sorted by the original + * sha1. + */ static struct replace_object { - unsigned char sha1[2][20]; + unsigned char original[20]; + unsigned char replacement[20]; } **replace_object; static int replace_object_alloc, replace_object_nr; @@@ -17,7 -12,7 +17,7 @@@ static const unsigned char *replace_sha1_access(size_t index, void *table) { struct replace_object **replace = table; - return replace[index]->sha1[0]; + return replace[index]->original; } static int replace_object_pos(const unsigned char *sha1) @@@ -29,7 -24,7 +29,7 @@@ static int register_replace_object(struct replace_object *replace, int ignore_dups) { - int pos = replace_object_pos(replace->sha1[0]); + int pos = replace_object_pos(replace->original); if (0 <= pos) { if (ignore_dups) @@@ -41,12 -36,8 +41,8 @@@ return 1; } pos = -pos - 1; - if (replace_object_alloc <= ++replace_object_nr) { - replace_object_alloc = alloc_nr(replace_object_alloc); - replace_object = xrealloc(replace_object, - sizeof(*replace_object) * - replace_object_alloc); - } + ALLOC_GROW(replace_object, replace_object_nr + 1, replace_object_alloc); + replace_object_nr++; if (pos < replace_object_nr) memmove(replace_object + pos + 1, replace_object + pos, @@@ -65,14 -56,14 +61,14 @@@ static int register_replace_ref(const c const char *hash = slash ? slash + 1 : refname; struct replace_object *repl_obj = xmalloc(sizeof(*repl_obj)); - if (strlen(hash) != 40 || get_sha1_hex(hash, repl_obj->sha1[0])) { + if (strlen(hash) != 40 || get_sha1_hex(hash, repl_obj->original)) { free(repl_obj); warning("bad replace ref name: %s", refname); return 0; } /* Copy sha1 from the read ref */ - hashcpy(repl_obj->sha1[1], sha1); + hashcpy(repl_obj->replacement, sha1); /* Register new object */ if (register_replace_object(repl_obj, 1)) @@@ -91,19 -82,12 +87,19 @@@ static void prepare_replace_object(void for_each_replace_ref(register_replace_ref, NULL); replace_object_prepared = 1; if (!replace_object_nr) - read_replace_refs = 0; + check_replace_refs = 0; } /* We allow "recursive" replacement. Only within reason, though */ #define MAXREPLACEDEPTH 5 +/* + * If a replacement for object sha1 has been set up, return the + * replacement object's name (replaced recursively, if necessary). + * The return value is either sha1 or a pointer to a + * permanently-allocated value. This function always respects replace + * references, regardless of the value of check_replace_refs. + */ const unsigned char *do_lookup_replace_object(const unsigned char *sha1) { int pos, depth = MAXREPLACEDEPTH; @@@ -119,7 -103,7 +115,7 @@@ pos = replace_object_pos(cur); if (0 <= pos) - cur = replace_object[pos]->sha1[1]; + cur = replace_object[pos]->replacement; } while (0 <= pos); return cur; diff --combined sha1_file.c index 18b2378d8a,2af06f8423..400aa463a4 --- a/sha1_file.c +++ b/sha1_file.c @@@ -60,12 -60,6 +60,12 @@@ static struct cached_object empty_tree 0 }; +/* + * A pointer to the last packed_git in which an object was found. + * When an object is sought, we look in this packfile first, because + * objects that are looked up at similar times are often in the same + * packfile as one another. + */ static struct packed_git *last_found_pack; static struct cached_object *find_cached_object(const unsigned char *sha1) @@@ -184,7 -178,17 +184,7 @@@ static void fill_sha1_path(char *pathbu } } -/* - * NOTE! This returns a statically allocated buffer, so you have to be - * careful about using it. Do an "xstrdup()" if you need to save the - * filename. - * - * Also note that this returns the location for creating. Reading - * SHA1 file can happen from any alternate directory listed in the - * DB_ENVIRONMENT environment variable if it is not found in - * the primary object database. - */ -char *sha1_file_name(const unsigned char *sha1) +const char *sha1_file_name(const unsigned char *sha1) { static char buf[PATH_MAX]; const char *objdir; @@@ -204,11 -208,6 +204,11 @@@ return buf; } +/* + * Return the name of the pack or index file with the specified sha1 + * in its filename. *base and *name are scratch space that must be + * provided by the caller. which should be "pack" or "idx". + */ static char *sha1_get_pack_name(const unsigned char *sha1, char **name, char **base, const char *which) { @@@ -253,6 -252,8 +253,6 @@@ char *sha1_pack_index_name(const unsign struct alternate_object_database *alt_odb_list; static struct alternate_object_database **alt_odb_tail; -static int git_open_noatime(const char *name); - /* * Prepare alternate object database registry. * @@@ -437,7 -438,8 +437,7 @@@ void prepare_alt_odb(void static int has_loose_object_local(const unsigned char *sha1) { - char *name = sha1_file_name(sha1); - return !access(name, F_OK); + return !access(sha1_file_name(sha1), F_OK); } int has_loose_object_nonlocal(const unsigned char *sha1) @@@ -489,12 -491,7 +489,12 @@@ void pack_report(void sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped)); } -static int check_packed_git_idx(const char *path, struct packed_git *p) +/* + * Open and mmap the index file at path, perform a couple of + * consistency checks, then record its information to p. Return 0 on + * success. + */ +static int check_packed_git_idx(const char *path, struct packed_git *p) { void *idx_map; struct pack_idx_header *hdr; @@@ -1235,7 -1232,6 +1235,7 @@@ static void prepare_packed_git_one(cha if (has_extension(de->d_name, ".idx") || has_extension(de->d_name, ".pack") || + has_extension(de->d_name, ".bitmap") || has_extension(de->d_name, ".keep")) string_list_append(&garbage, path); else @@@ -1320,6 -1316,7 +1320,6 @@@ void prepare_packed_git(void void reprepare_packed_git(void) { - discard_revindex(); prepare_packed_git_run_once = 0; prepare_packed_git(); } @@@ -1396,7 -1393,7 +1396,7 @@@ int check_sha1_signature(const unsigne return hashcmp(sha1, real_sha1) ? -1 : 0; } -static int git_open_noatime(const char *name) +int git_open_noatime(const char *name) { static int sha1_file_open_flag = O_NOATIME; @@@ -1417,15 -1414,17 +1417,15 @@@ static int stat_sha1_file(const unsigned char *sha1, struct stat *st) { - char *name = sha1_file_name(sha1); struct alternate_object_database *alt; - if (!lstat(name, st)) + if (!lstat(sha1_file_name(sha1), st)) return 0; prepare_alt_odb(); errno = ENOENT; for (alt = alt_odb_list; alt; alt = alt->next) { - name = alt->name; - fill_sha1_path(name, sha1); + fill_sha1_path(alt->name, sha1); if (!lstat(alt->base, st)) return 0; } @@@ -1436,16 -1435,18 +1436,16 @@@ static int open_sha1_file(const unsigned char *sha1) { int fd; - char *name = sha1_file_name(sha1); struct alternate_object_database *alt; - fd = git_open_noatime(name); + fd = git_open_noatime(sha1_file_name(sha1)); if (fd >= 0) return fd; prepare_alt_odb(); errno = ENOENT; for (alt = alt_odb_list; alt; alt = alt->next) { - name = alt->name; - fill_sha1_path(name, sha1); + fill_sha1_path(alt->name, sha1); fd = git_open_noatime(alt->base); if (fd >= 0) return fd; @@@ -2288,10 -2289,6 +2288,10 @@@ void *unpack_entry(struct packed_git *p *final_size = size; unuse_pack(&w_curs); + + if (delta_stack != small_delta_stack) + free(delta_stack); + return data; } @@@ -2451,10 -2448,6 +2451,10 @@@ static int fill_pack_entry(const unsign return 1; } +/* + * Iff a pack file contains the object named by sha1, return true and + * store its location to e. + */ static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e) { struct packed_git *p; @@@ -2467,13 -2460,11 +2467,13 @@@ return 1; for (p = packed_git; p; p = p->next) { - if (p == last_found_pack || !fill_pack_entry(sha1, e, p)) - continue; + if (p == last_found_pack) + continue; /* we already checked this one */ - last_found_pack = p; - return 1; + if (fill_pack_entry(sha1, e, p)) { + last_found_pack = p; + return 1; + } } return 0; } @@@ -2635,12 -2626,7 +2635,7 @@@ int pretend_sha1_file(void *buf, unsign hash_sha1_file(buf, len, typename(type), sha1); if (has_sha1_file(sha1) || find_cached_object(sha1)) return 0; - if (cached_object_alloc <= cached_object_nr) { - cached_object_alloc = alloc_nr(cached_object_alloc); - cached_objects = xrealloc(cached_objects, - sizeof(*cached_objects) * - cached_object_alloc); - } + ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc); co = &cached_objects[cached_object_nr++]; co->size = len; co->type = type; @@@ -2688,6 -2674,7 +2683,6 @@@ void *read_sha1_file_extended(const uns unsigned flag) { void *data; - char *path; const struct packed_git *p; const unsigned char *repl = lookup_replace_object_extended(sha1, flag); @@@ -2705,8 -2692,7 +2700,8 @@@ sha1_to_hex(repl), sha1_to_hex(sha1)); if (has_loose_object(repl)) { - path = sha1_file_name(sha1); + const char *path = sha1_file_name(sha1); + die("loose object %s (stored in %s) is corrupt", sha1_to_hex(repl), path); } @@@ -2904,9 -2890,10 +2899,9 @@@ static int write_loose_object(const uns git_zstream stream; git_SHA_CTX c; unsigned char parano_sha1[20]; - char *filename; static char tmp_file[PATH_MAX]; + const char *filename = sha1_file_name(sha1); - filename = sha1_file_name(sha1); fd = create_tmpfile(tmp_file, sizeof(tmp_file), filename); if (fd < 0) { if (errno == EACCES)