From: Junio C Hamano Date: Tue, 9 Jul 2019 22:25:43 +0000 (-0700) Subject: Merge branch 'jk/oidhash' X-Git-Tag: v2.23.0-rc0~66 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/a7db4c193d98f701eda0328dbf69b36f28dd22d3?ds=inline;hp=-c Merge branch 'jk/oidhash' Code clean-up to remove hardcoded SHA-1 hash from many places. * jk/oidhash: hashmap: convert sha1hash() to oidhash() hash.h: move object_id definition from cache.h khash: rename oid helper functions khash: drop sha1-specific map types pack-bitmap: convert khash_sha1 maps into kh_oid_map delta-islands: convert island_marks khash to use oids khash: rename kh_oid_t to kh_oid_set khash: drop broken oid_map typedef object: convert create_object() to use object_id object: convert internal hash_obj() to object_id object: convert lookup_object() to use object_id object: convert lookup_unknown_object() to use object_id pack-objects: convert locate_object_entry_hash() to object_id pack-objects: convert packlist_find() to use object_id pack-bitmap-write: convert some helpers to use object_id upload-pack: rename a "sha1" variable to "oid" describe: fix accidental oid/hash type-punning --- a7db4c193d98f701eda0328dbf69b36f28dd22d3 diff --combined builtin/pack-objects.c index 698c901523,3e8467aa23..000dc4b872 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@@ -606,12 -606,12 +606,12 @@@ static int mark_tagged(const char *path void *cb_data) { struct object_id peeled; - struct object_entry *entry = packlist_find(&to_pack, oid->hash, NULL); + struct object_entry *entry = packlist_find(&to_pack, oid, NULL); if (entry) entry->tagged = 1; if (!peel_ref(path, &peeled)) { - entry = packlist_find(&to_pack, peeled.hash, NULL); + entry = packlist_find(&to_pack, &peeled, NULL); if (entry) entry->tagged = 1; } @@@ -996,7 -996,7 +996,7 @@@ static int have_duplicate_entry(const s { struct object_entry *entry; - entry = packlist_find(&to_pack, oid->hash, index_pos); + entry = packlist_find(&to_pack, oid, index_pos); if (!entry) return 0; @@@ -1494,11 -1494,13 +1494,13 @@@ static int can_reuse_delta(const unsign if (!base_sha1) return 0; + oidread(&base_oid, base_sha1); + /* * First see if we're already sending the base (or it's explicitly in * our "excluded" list). */ - base = packlist_find(&to_pack, base_sha1, NULL); + base = packlist_find(&to_pack, &base_oid, NULL); if (base) { if (!in_same_island(&delta->idx.oid, &base->idx.oid)) return 0; @@@ -1511,7 -1513,6 +1513,6 @@@ * even if it was buried too deep in history to make it into the * packing list. */ - oidread(&base_oid, base_sha1); if (thin && bitmap_has_oid_in_uninteresting(bitmap_git, &base_oid)) { if (use_delta_islands) { if (!in_same_island(&delta->idx.oid, &base_oid)) @@@ -2571,7 -2572,7 +2572,7 @@@ static void add_tag_chain(const struct * it was included via bitmaps, we would not have parsed it * previously). */ - if (packlist_find(&to_pack, oid->hash, NULL)) + if (packlist_find(&to_pack, oid, NULL)) return; tag = lookup_tag(the_repository, oid); @@@ -2595,7 -2596,7 +2596,7 @@@ static int add_ref_tag(const char *path if (starts_with(path, "refs/tags/") && /* is a tag? */ !peel_ref(path, &peeled) && /* peelable? */ - packlist_find(&to_pack, peeled.hash, NULL)) /* object packed? */ + packlist_find(&to_pack, &peeled, NULL)) /* object packed? */ add_tag_chain(oid); return 0; } @@@ -2795,7 -2796,7 +2796,7 @@@ static void show_object(struct object * for (p = strchr(name, '/'); p; p = strchr(p + 1, '/')) depth++; - ent = packlist_find(&to_pack, obj->oid.hash, NULL); + ent = packlist_find(&to_pack, &obj->oid, NULL); if (ent && depth > oe_tree_depth(&to_pack, ent)) oe_set_tree_depth(&to_pack, ent, depth); } @@@ -2922,7 -2923,7 +2923,7 @@@ static void add_objects_in_unpacked_pac for (i = 0; i < p->num_objects; i++) { nth_packed_object_oid(&oid, p, i); - o = lookup_unknown_object(oid.hash); + o = lookup_unknown_object(&oid); if (!(o->flags & OBJECT_ADDED)) mark_in_pack_object(o, p, &in_pack); o->flags |= OBJECT_ADDED; @@@ -3026,7 -3027,7 +3027,7 @@@ static void loosen_unused_packed_object for (i = 0; i < p->num_objects; i++) { nth_packed_object_oid(&oid, p, i); - if (!packlist_find(&to_pack, oid.hash, NULL) && + if (!packlist_find(&to_pack, &oid, NULL) && !has_sha1_pack_kept_or_nonlocal(&oid) && !loosened_object_can_be_discarded(&oid, p->mtime)) if (force_object_loose(&oid, p->mtime)) @@@ -3134,7 -3135,7 +3135,7 @@@ static void get_object_list(int ac, con return; if (use_delta_islands) - load_delta_islands(the_repository); + load_delta_islands(the_repository, progress); if (prepare_revision_walk(&revs)) die(_("revision walk setup failed")); diff --combined commit-graph.c index 1752341098,5a62131d68..8cc1d1d6c3 --- a/commit-graph.c +++ b/commit-graph.c @@@ -361,10 -361,10 +361,10 @@@ int generation_numbers_enabled(struct r return !!first_generation; } -void close_commit_graph(struct repository *r) +void close_commit_graph(struct raw_object_store *o) { - free_commit_graph(r->objects->commit_graph); - r->objects->commit_graph = NULL; + free_commit_graph(o->commit_graph); + o->commit_graph = NULL; } static int bsearch_graph(struct commit_graph *g, struct object_id *oid, uint32_t *pos) @@@ -525,38 -525,14 +525,38 @@@ struct tree *get_commit_tree_in_graph(s return get_commit_tree_in_graph_one(r, r->objects->commit_graph, c); } +struct packed_commit_list { + struct commit **list; + int nr; + int alloc; +}; + +struct packed_oid_list { + struct object_id *list; + int nr; + int alloc; +}; + +struct write_commit_graph_context { + struct repository *r; + const char *obj_dir; + char *graph_name; + struct packed_oid_list oids; + struct packed_commit_list commits; + int num_extra_edges; + unsigned long approx_nr_objects; + struct progress *progress; + int progress_done; + uint64_t progress_cnt; + unsigned append:1, + report_progress:1; +}; + static void write_graph_chunk_fanout(struct hashfile *f, - struct commit **commits, - int nr_commits, - struct progress *progress, - uint64_t *progress_cnt) + struct write_commit_graph_context *ctx) { int i, count = 0; - struct commit **list = commits; + struct commit **list = ctx->commits.list; /* * Write the first-level table (the list is sorted, @@@ -564,10 -540,10 +564,10 @@@ * having to do eight extra binary search iterations). */ for (i = 0; i < 256; i++) { - while (count < nr_commits) { + while (count < ctx->commits.nr) { if ((*list)->object.oid.hash[0] != i) break; - display_progress(progress, ++*progress_cnt); + display_progress(ctx->progress, ++ctx->progress_cnt); count++; list++; } @@@ -577,12 -553,14 +577,12 @@@ } static void write_graph_chunk_oids(struct hashfile *f, int hash_len, - struct commit **commits, int nr_commits, - struct progress *progress, - uint64_t *progress_cnt) + struct write_commit_graph_context *ctx) { - struct commit **list = commits; + struct commit **list = ctx->commits.list; int count; - for (count = 0; count < nr_commits; count++, list++) { - display_progress(progress, ++*progress_cnt); + for (count = 0; count < ctx->commits.nr; count++, list++) { + display_progress(ctx->progress, ++ctx->progress_cnt); hashwrite(f, (*list)->object.oid.hash, (int)hash_len); } } @@@ -594,17 -572,19 +594,17 @@@ static const unsigned char *commit_to_s } static void write_graph_chunk_data(struct hashfile *f, int hash_len, - struct commit **commits, int nr_commits, - struct progress *progress, - uint64_t *progress_cnt) + struct write_commit_graph_context *ctx) { - struct commit **list = commits; - struct commit **last = commits + nr_commits; + struct commit **list = ctx->commits.list; + struct commit **last = ctx->commits.list + ctx->commits.nr; uint32_t num_extra_edges = 0; while (list < last) { struct commit_list *parent; int edge_value; uint32_t packedDate[2]; - display_progress(progress, ++*progress_cnt); + display_progress(ctx->progress, ++ctx->progress_cnt); parse_commit_no_graph(*list); hashwrite(f, get_commit_tree_oid(*list)->hash, hash_len); @@@ -615,8 -595,8 +615,8 @@@ edge_value = GRAPH_PARENT_NONE; else { edge_value = sha1_pos(parent->item->object.oid.hash, - commits, - nr_commits, + ctx->commits.list, + ctx->commits.nr, commit_to_sha1); if (edge_value < 0) @@@ -636,8 -616,8 +636,8 @@@ edge_value = GRAPH_EXTRA_EDGES_NEEDED | num_extra_edges; else { edge_value = sha1_pos(parent->item->object.oid.hash, - commits, - nr_commits, + ctx->commits.list, + ctx->commits.nr, commit_to_sha1); if (edge_value < 0) BUG("missing parent %s for commit %s", @@@ -669,16 -649,19 +669,16 @@@ } static void write_graph_chunk_extra_edges(struct hashfile *f, - struct commit **commits, - int nr_commits, - struct progress *progress, - uint64_t *progress_cnt) + struct write_commit_graph_context *ctx) { - struct commit **list = commits; - struct commit **last = commits + nr_commits; + struct commit **list = ctx->commits.list; + struct commit **last = ctx->commits.list + ctx->commits.nr; struct commit_list *parent; while (list < last) { int num_parents = 0; - display_progress(progress, ++*progress_cnt); + display_progress(ctx->progress, ++ctx->progress_cnt); for (parent = (*list)->parents; num_parents < 3 && parent; parent = parent->next) @@@ -692,8 -675,8 +692,8 @@@ /* Since num_parents > 2, this initializer is safe. */ for (parent = (*list)->parents->next; parent; parent = parent->next) { int edge_value = sha1_pos(parent->item->object.oid.hash, - commits, - nr_commits, + ctx->commits.list, + ctx->commits.nr, commit_to_sha1); if (edge_value < 0) @@@ -717,111 -700,125 +717,111 @@@ static int commit_compare(const void *_ return oidcmp(a, b); } -struct packed_commit_list { - struct commit **list; - int nr; - int alloc; -}; - -struct packed_oid_list { - struct object_id *list; - int nr; - int alloc; - struct progress *progress; - int progress_done; -}; - static int add_packed_commits(const struct object_id *oid, struct packed_git *pack, uint32_t pos, void *data) { - struct packed_oid_list *list = (struct packed_oid_list*)data; + struct write_commit_graph_context *ctx = (struct write_commit_graph_context*)data; enum object_type type; off_t offset = nth_packed_object_offset(pack, pos); struct object_info oi = OBJECT_INFO_INIT; - if (list->progress) - display_progress(list->progress, ++list->progress_done); + if (ctx->progress) + display_progress(ctx->progress, ++ctx->progress_done); oi.typep = &type; - if (packed_object_info(the_repository, pack, offset, &oi) < 0) + if (packed_object_info(ctx->r, pack, offset, &oi) < 0) die(_("unable to get type of object %s"), oid_to_hex(oid)); if (type != OBJ_COMMIT) return 0; - ALLOC_GROW(list->list, list->nr + 1, list->alloc); - oidcpy(&(list->list[list->nr]), oid); - list->nr++; + ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc); + oidcpy(&(ctx->oids.list[ctx->oids.nr]), oid); + ctx->oids.nr++; return 0; } -static void add_missing_parents(struct packed_oid_list *oids, struct commit *commit) +static void add_missing_parents(struct write_commit_graph_context *ctx, struct commit *commit) { struct commit_list *parent; for (parent = commit->parents; parent; parent = parent->next) { if (!(parent->item->object.flags & UNINTERESTING)) { - ALLOC_GROW(oids->list, oids->nr + 1, oids->alloc); - oidcpy(&oids->list[oids->nr], &(parent->item->object.oid)); - oids->nr++; + ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc); + oidcpy(&ctx->oids.list[ctx->oids.nr], &(parent->item->object.oid)); + ctx->oids.nr++; parent->item->object.flags |= UNINTERESTING; } } } -static void close_reachable(struct packed_oid_list *oids, int report_progress) +static void close_reachable(struct write_commit_graph_context *ctx) { int i; struct commit *commit; - struct progress *progress = NULL; - if (report_progress) - progress = start_delayed_progress( - _("Loading known commits in commit graph"), oids->nr); - for (i = 0; i < oids->nr; i++) { - display_progress(progress, i + 1); - commit = lookup_commit(the_repository, &oids->list[i]); + if (ctx->report_progress) + ctx->progress = start_delayed_progress( + _("Loading known commits in commit graph"), + ctx->oids.nr); + for (i = 0; i < ctx->oids.nr; i++) { + display_progress(ctx->progress, i + 1); + commit = lookup_commit(ctx->r, &ctx->oids.list[i]); if (commit) commit->object.flags |= UNINTERESTING; } - stop_progress(&progress); + stop_progress(&ctx->progress); /* - * As this loop runs, oids->nr may grow, but not more + * As this loop runs, ctx->oids.nr may grow, but not more * than the number of missing commits in the reachable * closure. */ - if (report_progress) - progress = start_delayed_progress( - _("Expanding reachable commits in commit graph"), oids->nr); - for (i = 0; i < oids->nr; i++) { - display_progress(progress, i + 1); - commit = lookup_commit(the_repository, &oids->list[i]); + if (ctx->report_progress) + ctx->progress = start_delayed_progress( + _("Expanding reachable commits in commit graph"), + ctx->oids.nr); + for (i = 0; i < ctx->oids.nr; i++) { + display_progress(ctx->progress, i + 1); + commit = lookup_commit(ctx->r, &ctx->oids.list[i]); if (commit && !parse_commit_no_graph(commit)) - add_missing_parents(oids, commit); + add_missing_parents(ctx, commit); } - stop_progress(&progress); + stop_progress(&ctx->progress); - if (report_progress) - progress = start_delayed_progress( - _("Clearing commit marks in commit graph"), oids->nr); - for (i = 0; i < oids->nr; i++) { - display_progress(progress, i + 1); - commit = lookup_commit(the_repository, &oids->list[i]); + if (ctx->report_progress) + ctx->progress = start_delayed_progress( + _("Clearing commit marks in commit graph"), + ctx->oids.nr); + for (i = 0; i < ctx->oids.nr; i++) { + display_progress(ctx->progress, i + 1); + commit = lookup_commit(ctx->r, &ctx->oids.list[i]); if (commit) commit->object.flags &= ~UNINTERESTING; } - stop_progress(&progress); + stop_progress(&ctx->progress); } -static void compute_generation_numbers(struct packed_commit_list* commits, - int report_progress) +static void compute_generation_numbers(struct write_commit_graph_context *ctx) { int i; struct commit_list *list = NULL; - struct progress *progress = NULL; - if (report_progress) - progress = start_progress( - _("Computing commit graph generation numbers"), - commits->nr); - for (i = 0; i < commits->nr; i++) { - display_progress(progress, i + 1); - if (commits->list[i]->generation != GENERATION_NUMBER_INFINITY && - commits->list[i]->generation != GENERATION_NUMBER_ZERO) + if (ctx->report_progress) + ctx->progress = start_progress( + _("Computing commit graph generation numbers"), + ctx->commits.nr); + for (i = 0; i < ctx->commits.nr; i++) { + display_progress(ctx->progress, i + 1); + if (ctx->commits.list[i]->generation != GENERATION_NUMBER_INFINITY && + ctx->commits.list[i]->generation != GENERATION_NUMBER_ZERO) continue; - commit_list_insert(commits->list[i], &list); + commit_list_insert(ctx->commits.list[i], &list); while (list) { struct commit *current = list->item; struct commit_list *parent; @@@ -848,7 -845,7 +848,7 @@@ } } } - stop_progress(&progress); + stop_progress(&ctx->progress); } static int add_ref_to_list(const char *refname, @@@ -861,187 -858,207 +861,187 @@@ return 0; } -void write_commit_graph_reachable(const char *obj_dir, int append, - int report_progress) +int write_commit_graph_reachable(const char *obj_dir, unsigned int flags) { struct string_list list = STRING_LIST_INIT_DUP; + int result; for_each_ref(add_ref_to_list, &list); - write_commit_graph(obj_dir, NULL, &list, append, report_progress); + result = write_commit_graph(obj_dir, NULL, &list, + flags); string_list_clear(&list, 0); + return result; } -void write_commit_graph(const char *obj_dir, - struct string_list *pack_indexes, - struct string_list *commit_hex, - int append, int report_progress) +static int fill_oids_from_packs(struct write_commit_graph_context *ctx, + struct string_list *pack_indexes) { - struct packed_oid_list oids; - struct packed_commit_list commits; - struct hashfile *f; - uint32_t i, count_distinct = 0; - char *graph_name; - struct lock_file lk = LOCK_INIT; - uint32_t chunk_ids[5]; - uint64_t chunk_offsets[5]; - int num_chunks; - int num_extra_edges; - struct commit_list *parent; - struct progress *progress = NULL; - const unsigned hashsz = the_hash_algo->rawsz; - uint64_t progress_cnt = 0; + uint32_t i; struct strbuf progress_title = STRBUF_INIT; - unsigned long approx_nr_objects; - - if (!commit_graph_compatible(the_repository)) - return; - - oids.nr = 0; - approx_nr_objects = approximate_object_count(); - oids.alloc = approx_nr_objects / 32; - oids.progress = NULL; - oids.progress_done = 0; + struct strbuf packname = STRBUF_INIT; + int dirlen; - if (append) { - prepare_commit_graph_one(the_repository, obj_dir); - if (the_repository->objects->commit_graph) - oids.alloc += the_repository->objects->commit_graph->num_commits; + strbuf_addf(&packname, "%s/pack/", ctx->obj_dir); + dirlen = packname.len; + if (ctx->report_progress) { + strbuf_addf(&progress_title, + Q_("Finding commits for commit graph in %d pack", + "Finding commits for commit graph in %d packs", + pack_indexes->nr), + pack_indexes->nr); + ctx->progress = start_delayed_progress(progress_title.buf, 0); + ctx->progress_done = 0; } - - if (oids.alloc < 1024) - oids.alloc = 1024; - ALLOC_ARRAY(oids.list, oids.alloc); - - if (append && the_repository->objects->commit_graph) { - struct commit_graph *commit_graph = - the_repository->objects->commit_graph; - for (i = 0; i < commit_graph->num_commits; i++) { - const unsigned char *hash = commit_graph->chunk_oid_lookup + - commit_graph->hash_len * i; - hashcpy(oids.list[oids.nr++].hash, hash); + for (i = 0; i < pack_indexes->nr; i++) { + struct packed_git *p; + strbuf_setlen(&packname, dirlen); + strbuf_addstr(&packname, pack_indexes->items[i].string); + p = add_packed_git(packname.buf, packname.len, 1); + if (!p) { + error(_("error adding pack %s"), packname.buf); + return -1; } + if (open_pack_index(p)) { + error(_("error opening index for %s"), packname.buf); + return -1; + } + for_each_object_in_pack(p, add_packed_commits, ctx, + FOR_EACH_OBJECT_PACK_ORDER); + close_pack(p); + free(p); } - if (pack_indexes) { - struct strbuf packname = STRBUF_INIT; - int dirlen; - strbuf_addf(&packname, "%s/pack/", obj_dir); - dirlen = packname.len; - if (report_progress) { - strbuf_addf(&progress_title, - Q_("Finding commits for commit graph in %d pack", - "Finding commits for commit graph in %d packs", - pack_indexes->nr), - pack_indexes->nr); - oids.progress = start_delayed_progress(progress_title.buf, 0); - oids.progress_done = 0; - } - for (i = 0; i < pack_indexes->nr; i++) { - struct packed_git *p; - strbuf_setlen(&packname, dirlen); - strbuf_addstr(&packname, pack_indexes->items[i].string); - p = add_packed_git(packname.buf, packname.len, 1); - if (!p) - die(_("error adding pack %s"), packname.buf); - if (open_pack_index(p)) - die(_("error opening index for %s"), packname.buf); - for_each_object_in_pack(p, add_packed_commits, &oids, - FOR_EACH_OBJECT_PACK_ORDER); - close_pack(p); - free(p); - } - stop_progress(&oids.progress); - strbuf_reset(&progress_title); - strbuf_release(&packname); + stop_progress(&ctx->progress); + strbuf_reset(&progress_title); + strbuf_release(&packname); + + return 0; +} + +static void fill_oids_from_commit_hex(struct write_commit_graph_context *ctx, + struct string_list *commit_hex) +{ + uint32_t i; + struct strbuf progress_title = STRBUF_INIT; + + if (ctx->report_progress) { + strbuf_addf(&progress_title, + Q_("Finding commits for commit graph from %d ref", + "Finding commits for commit graph from %d refs", + commit_hex->nr), + commit_hex->nr); + ctx->progress = start_delayed_progress( + progress_title.buf, + commit_hex->nr); } + for (i = 0; i < commit_hex->nr; i++) { + const char *end; + struct object_id oid; + struct commit *result; + + display_progress(ctx->progress, i + 1); + if (commit_hex->items[i].string && + parse_oid_hex(commit_hex->items[i].string, &oid, &end)) + continue; - if (commit_hex) { - if (report_progress) { - strbuf_addf(&progress_title, - Q_("Finding commits for commit graph from %d ref", - "Finding commits for commit graph from %d refs", - commit_hex->nr), - commit_hex->nr); - progress = start_delayed_progress(progress_title.buf, - commit_hex->nr); - } - for (i = 0; i < commit_hex->nr; i++) { - const char *end; - struct object_id oid; - struct commit *result; - - display_progress(progress, i + 1); - if (commit_hex->items[i].string && - parse_oid_hex(commit_hex->items[i].string, &oid, &end)) - continue; - - result = lookup_commit_reference_gently(the_repository, &oid, 1); - - if (result) { - ALLOC_GROW(oids.list, oids.nr + 1, oids.alloc); - oidcpy(&oids.list[oids.nr], &(result->object.oid)); - oids.nr++; - } + result = lookup_commit_reference_gently(ctx->r, &oid, 1); + + if (result) { + ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc); + oidcpy(&ctx->oids.list[ctx->oids.nr], &(result->object.oid)); + ctx->oids.nr++; } - stop_progress(&progress); - strbuf_reset(&progress_title); } + stop_progress(&ctx->progress); + strbuf_release(&progress_title); +} - if (!pack_indexes && !commit_hex) { - if (report_progress) - oids.progress = start_delayed_progress( - _("Finding commits for commit graph among packed objects"), - approx_nr_objects); - for_each_packed_object(add_packed_commits, &oids, - FOR_EACH_OBJECT_PACK_ORDER); - if (oids.progress_done < approx_nr_objects) - display_progress(oids.progress, approx_nr_objects); - stop_progress(&oids.progress); - } +static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx) +{ + if (ctx->report_progress) + ctx->progress = start_delayed_progress( + _("Finding commits for commit graph among packed objects"), + ctx->approx_nr_objects); + for_each_packed_object(add_packed_commits, ctx, + FOR_EACH_OBJECT_PACK_ORDER); + if (ctx->progress_done < ctx->approx_nr_objects) + display_progress(ctx->progress, ctx->approx_nr_objects); + stop_progress(&ctx->progress); +} - close_reachable(&oids, report_progress); +static uint32_t count_distinct_commits(struct write_commit_graph_context *ctx) +{ + uint32_t i, count_distinct = 1; - if (report_progress) - progress = start_delayed_progress( + if (ctx->report_progress) + ctx->progress = start_delayed_progress( _("Counting distinct commits in commit graph"), - oids.nr); - display_progress(progress, 0); /* TODO: Measure QSORT() progress */ - QSORT(oids.list, oids.nr, commit_compare); - count_distinct = 1; - for (i = 1; i < oids.nr; i++) { - display_progress(progress, i + 1); - if (!oideq(&oids.list[i - 1], &oids.list[i])) + ctx->oids.nr); + display_progress(ctx->progress, 0); /* TODO: Measure QSORT() progress */ + QSORT(ctx->oids.list, ctx->oids.nr, commit_compare); + + for (i = 1; i < ctx->oids.nr; i++) { + display_progress(ctx->progress, i + 1); + if (!oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i])) count_distinct++; } - stop_progress(&progress); + stop_progress(&ctx->progress); - if (count_distinct >= GRAPH_EDGE_LAST_MASK) - die(_("the commit graph format cannot write %d commits"), count_distinct); + return count_distinct; +} - commits.nr = 0; - commits.alloc = count_distinct; - ALLOC_ARRAY(commits.list, commits.alloc); +static void copy_oids_to_commits(struct write_commit_graph_context *ctx) +{ + uint32_t i; + struct commit_list *parent; - num_extra_edges = 0; - if (report_progress) - progress = start_delayed_progress( + ctx->num_extra_edges = 0; + if (ctx->report_progress) + ctx->progress = start_delayed_progress( _("Finding extra edges in commit graph"), - oids.nr); - for (i = 0; i < oids.nr; i++) { + ctx->oids.nr); + for (i = 0; i < ctx->oids.nr; i++) { int num_parents = 0; - display_progress(progress, i + 1); - if (i > 0 && oideq(&oids.list[i - 1], &oids.list[i])) + display_progress(ctx->progress, i + 1); + if (i > 0 && oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i])) continue; - commits.list[commits.nr] = lookup_commit(the_repository, &oids.list[i]); - parse_commit_no_graph(commits.list[commits.nr]); + ctx->commits.list[ctx->commits.nr] = lookup_commit(ctx->r, &ctx->oids.list[i]); + parse_commit_no_graph(ctx->commits.list[ctx->commits.nr]); - for (parent = commits.list[commits.nr]->parents; + for (parent = ctx->commits.list[ctx->commits.nr]->parents; parent; parent = parent->next) num_parents++; if (num_parents > 2) - num_extra_edges += num_parents - 1; + ctx->num_extra_edges += num_parents - 1; - commits.nr++; + ctx->commits.nr++; } - num_chunks = num_extra_edges ? 4 : 3; - stop_progress(&progress); - - if (commits.nr >= GRAPH_EDGE_LAST_MASK) - die(_("too many commits to write graph")); - - compute_generation_numbers(&commits, report_progress); + stop_progress(&ctx->progress); +} - graph_name = get_commit_graph_filename(obj_dir); - if (safe_create_leading_directories(graph_name)) { - UNLEAK(graph_name); - die_errno(_("unable to create leading directories of %s"), - graph_name); +static int write_commit_graph_file(struct write_commit_graph_context *ctx) +{ + uint32_t i; + struct hashfile *f; + struct lock_file lk = LOCK_INIT; + uint32_t chunk_ids[5]; + uint64_t chunk_offsets[5]; + const unsigned hashsz = the_hash_algo->rawsz; + struct strbuf progress_title = STRBUF_INIT; + int num_chunks = ctx->num_extra_edges ? 4 : 3; + + ctx->graph_name = get_commit_graph_filename(ctx->obj_dir); + if (safe_create_leading_directories(ctx->graph_name)) { + UNLEAK(ctx->graph_name); + error(_("unable to create leading directories of %s"), + ctx->graph_name); + return -1; } - hold_lock_file_for_update(&lk, graph_name, LOCK_DIE_ON_ERROR); + hold_lock_file_for_update(&lk, ctx->graph_name, LOCK_DIE_ON_ERROR); f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf); hashwrite_be32(f, GRAPH_SIGNATURE); @@@ -1054,7 -1071,7 +1054,7 @@@ chunk_ids[0] = GRAPH_CHUNKID_OIDFANOUT; chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP; chunk_ids[2] = GRAPH_CHUNKID_DATA; - if (num_extra_edges) + if (ctx->num_extra_edges) chunk_ids[3] = GRAPH_CHUNKID_EXTRAEDGES; else chunk_ids[3] = 0; @@@ -1062,9 -1079,9 +1062,9 @@@ chunk_offsets[0] = 8 + (num_chunks + 1) * GRAPH_CHUNKLOOKUP_WIDTH; chunk_offsets[1] = chunk_offsets[0] + GRAPH_FANOUT_SIZE; - chunk_offsets[2] = chunk_offsets[1] + hashsz * commits.nr; - chunk_offsets[3] = chunk_offsets[2] + (hashsz + 16) * commits.nr; - chunk_offsets[4] = chunk_offsets[3] + 4 * num_extra_edges; + chunk_offsets[2] = chunk_offsets[1] + hashsz * ctx->commits.nr; + chunk_offsets[3] = chunk_offsets[2] + (hashsz + 16) * ctx->commits.nr; + chunk_offsets[4] = chunk_offsets[3] + 4 * ctx->num_extra_edges; for (i = 0; i <= num_chunks; i++) { uint32_t chunk_write[3]; @@@ -1075,113 -1092,31 +1075,113 @@@ hashwrite(f, chunk_write, 12); } - if (report_progress) { + if (ctx->report_progress) { strbuf_addf(&progress_title, Q_("Writing out commit graph in %d pass", "Writing out commit graph in %d passes", num_chunks), num_chunks); - progress = start_delayed_progress( + ctx->progress = start_delayed_progress( progress_title.buf, - num_chunks * commits.nr); + num_chunks * ctx->commits.nr); } - write_graph_chunk_fanout(f, commits.list, commits.nr, progress, &progress_cnt); - write_graph_chunk_oids(f, hashsz, commits.list, commits.nr, progress, &progress_cnt); - write_graph_chunk_data(f, hashsz, commits.list, commits.nr, progress, &progress_cnt); - if (num_extra_edges) - write_graph_chunk_extra_edges(f, commits.list, commits.nr, progress, &progress_cnt); - stop_progress(&progress); + write_graph_chunk_fanout(f, ctx); + write_graph_chunk_oids(f, hashsz, ctx); + write_graph_chunk_data(f, hashsz, ctx); + if (ctx->num_extra_edges) + write_graph_chunk_extra_edges(f, ctx); + stop_progress(&ctx->progress); strbuf_release(&progress_title); - close_commit_graph(the_repository); + close_commit_graph(ctx->r->objects); finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC); commit_lock_file(&lk); - free(graph_name); - free(commits.list); - free(oids.list); + return 0; +} + +int write_commit_graph(const char *obj_dir, + struct string_list *pack_indexes, + struct string_list *commit_hex, + unsigned int flags) +{ + struct write_commit_graph_context *ctx; + uint32_t i, count_distinct = 0; + int res = 0; + + if (!commit_graph_compatible(the_repository)) + return 0; + + ctx = xcalloc(1, sizeof(struct write_commit_graph_context)); + ctx->r = the_repository; + ctx->obj_dir = obj_dir; + ctx->append = flags & COMMIT_GRAPH_APPEND ? 1 : 0; + ctx->report_progress = flags & COMMIT_GRAPH_PROGRESS ? 1 : 0; + + ctx->approx_nr_objects = approximate_object_count(); + ctx->oids.alloc = ctx->approx_nr_objects / 32; + + if (ctx->append) { + prepare_commit_graph_one(ctx->r, ctx->obj_dir); + if (ctx->r->objects->commit_graph) + ctx->oids.alloc += ctx->r->objects->commit_graph->num_commits; + } + + if (ctx->oids.alloc < 1024) + ctx->oids.alloc = 1024; + ALLOC_ARRAY(ctx->oids.list, ctx->oids.alloc); + + if (ctx->append && ctx->r->objects->commit_graph) { + struct commit_graph *g = ctx->r->objects->commit_graph; + for (i = 0; i < g->num_commits; i++) { + const unsigned char *hash = g->chunk_oid_lookup + g->hash_len * i; + hashcpy(ctx->oids.list[ctx->oids.nr++].hash, hash); + } + } + + if (pack_indexes) { + if ((res = fill_oids_from_packs(ctx, pack_indexes))) + goto cleanup; + } + + if (commit_hex) + fill_oids_from_commit_hex(ctx, commit_hex); + + if (!pack_indexes && !commit_hex) + fill_oids_from_all_packs(ctx); + + close_reachable(ctx); + + count_distinct = count_distinct_commits(ctx); + + if (count_distinct >= GRAPH_EDGE_LAST_MASK) { + error(_("the commit graph format cannot write %d commits"), count_distinct); + res = -1; + goto cleanup; + } + + ctx->commits.alloc = count_distinct; + ALLOC_ARRAY(ctx->commits.list, ctx->commits.alloc); + + copy_oids_to_commits(ctx); + + if (ctx->commits.nr >= GRAPH_EDGE_LAST_MASK) { + error(_("too many commits to write graph")); + res = -1; + goto cleanup; + } + + compute_generation_numbers(ctx); + + res = write_commit_graph_file(ctx); + +cleanup: + free(ctx->graph_name); + free(ctx->commits.list); + free(ctx->oids.list); + free(ctx); + + return res; } #define VERIFY_COMMIT_GRAPH_ERROR_HASH 2 @@@ -1279,7 -1214,7 +1279,7 @@@ int verify_commit_graph(struct reposito hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i); graph_commit = lookup_commit(r, &cur_oid); - odb_commit = (struct commit *)create_object(r, cur_oid.hash, alloc_commit_node(r)); + odb_commit = (struct commit *)create_object(r, &cur_oid, alloc_commit_node(r)); if (parse_commit_internal(odb_commit, 0, 0)) { graph_report(_("failed to parse commit %s from object database for commit-graph"), oid_to_hex(&cur_oid)); diff --combined commit.c index 26ce0770f6,b71ac195d4..a98de16e3d --- a/commit.c +++ b/commit.c @@@ -57,10 -57,9 +57,9 @@@ struct commit *lookup_commit_or_die(con struct commit *lookup_commit(struct repository *r, const struct object_id *oid) { - struct object *obj = lookup_object(r, oid->hash); + struct object *obj = lookup_object(r, oid); if (!obj) - return create_object(r, oid->hash, - alloc_commit_node(r)); + return create_object(r, oid, alloc_commit_node(r)); return object_as_type(r, obj, OBJ_COMMIT, 0); } @@@ -449,7 -448,7 +448,7 @@@ int parse_commit_buffer(struct reposito item->date = parse_commit_date(bufptr, tail); if (check_graph) - load_commit_graph_info(the_repository, item); + load_commit_graph_info(r, item); return 0; } diff --combined delta-islands.c index b959f6c380,88d102298c..09dbd3cf72 --- a/delta-islands.c +++ b/delta-islands.c @@@ -22,7 -22,7 +22,7 @@@ KHASH_INIT(str, const char *, void *, 1, kh_str_hash_func, kh_str_hash_equal) - static khash_sha1 *island_marks; + static kh_oid_map_t *island_marks; static unsigned island_counter; static unsigned island_counter_core; @@@ -105,7 -105,7 +105,7 @@@ int in_same_island(const struct object_ * If we don't have a bitmap for the target, we can delta it * against anything -- it's not an important object */ - trg_pos = kh_get_sha1(island_marks, trg_oid->hash); + trg_pos = kh_get_oid_map(island_marks, *trg_oid); if (trg_pos >= kh_end(island_marks)) return 1; @@@ -113,7 -113,7 +113,7 @@@ * if the source (our delta base) doesn't have a bitmap, * we don't want to base any deltas on it! */ - src_pos = kh_get_sha1(island_marks, src_oid->hash); + src_pos = kh_get_oid_map(island_marks, *src_oid); if (src_pos >= kh_end(island_marks)) return 0; @@@ -129,11 -129,11 +129,11 @@@ int island_delta_cmp(const struct objec if (!island_marks) return 0; - a_pos = kh_get_sha1(island_marks, a->hash); + a_pos = kh_get_oid_map(island_marks, *a); if (a_pos < kh_end(island_marks)) a_bitmap = kh_value(island_marks, a_pos); - b_pos = kh_get_sha1(island_marks, b->hash); + b_pos = kh_get_oid_map(island_marks, *b); if (b_pos < kh_end(island_marks)) b_bitmap = kh_value(island_marks, b_pos); @@@ -154,7 -154,7 +154,7 @@@ static struct island_bitmap *create_or_ khiter_t pos; int hash_ret; - pos = kh_put_sha1(island_marks, obj->oid.hash, &hash_ret); + pos = kh_put_oid_map(island_marks, obj->oid, &hash_ret); if (hash_ret) kh_value(island_marks, pos) = island_bitmap_new(NULL); @@@ -167,7 -167,7 +167,7 @@@ static void set_island_marks(struct obj khiter_t pos; int hash_ret; - pos = kh_put_sha1(island_marks, obj->oid.hash, &hash_ret); + pos = kh_put_oid_map(island_marks, obj->oid, &hash_ret); if (hash_ret) { /* * We don't have one yet; make a copy-on-write of the @@@ -279,7 -279,7 +279,7 @@@ void resolve_tree_islands(struct reposi struct name_entry entry; khiter_t pos; - pos = kh_get_sha1(island_marks, ent->idx.oid.hash); + pos = kh_get_oid_map(island_marks, ent->idx.oid); if (pos >= kh_end(island_marks)) continue; @@@ -296,7 -296,7 +296,7 @@@ if (S_ISGITLINK(entry.mode)) continue; - obj = lookup_object(r, entry.oid.hash); + obj = lookup_object(r, &entry.oid); if (!obj) continue; @@@ -454,22 -454,21 +454,22 @@@ static void deduplicate_islands(struct free(list); } -void load_delta_islands(struct repository *r) +void load_delta_islands(struct repository *r, int progress) { - island_marks = kh_init_sha1(); + island_marks = kh_init_oid_map(); remote_islands = kh_init_str(); git_config(island_config_callback, NULL); for_each_ref(find_island_for_ref, NULL); deduplicate_islands(r); - fprintf(stderr, _("Marked %d islands, done.\n"), island_counter); + if (progress) + fprintf(stderr, _("Marked %d islands, done.\n"), island_counter); } void propagate_island_marks(struct commit *commit) { - khiter_t pos = kh_get_sha1(island_marks, commit->object.oid.hash); + khiter_t pos = kh_get_oid_map(island_marks, commit->object.oid); if (pos < kh_end(island_marks)) { struct commit_list *p; @@@ -491,7 -490,7 +491,7 @@@ int compute_pack_layers(struct packing_ for (i = 0; i < to_pack->nr_objects; ++i) { struct object_entry *entry = &to_pack->objects[i]; - khiter_t pos = kh_get_sha1(island_marks, entry->idx.oid.hash); + khiter_t pos = kh_get_oid_map(island_marks, entry->idx.oid); oe_set_layer(to_pack, entry, 1); diff --combined diffcore-rename.c index 6af92d5eba,1e50d491c1..9624864858 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@@ -23,7 -23,7 +23,7 @@@ static int find_rename_dst(struct diff_ first = 0; last = rename_dst_nr; while (last > first) { - int next = (last + first) >> 1; + int next = first + ((last - first) >> 1); struct diff_rename_dst *dst = &(rename_dst[next]); int cmp = strcmp(two->path, dst->two->path); if (!cmp) @@@ -83,7 -83,7 +83,7 @@@ static struct diff_rename_src *register first = 0; last = rename_src_nr; while (last > first) { - int next = (last + first) >> 1; + int next = first + ((last - first) >> 1); struct diff_rename_src *src = &(rename_src[next]); int cmp = strcmp(one->path, src->p->one->path); if (!cmp) @@@ -266,7 -266,7 +266,7 @@@ static unsigned int hash_filespec(struc hash_object_file(filespec->data, filespec->size, "blob", &filespec->oid); } - return sha1hash(filespec->oid.hash); + return oidhash(&filespec->oid); } static int find_identical_files(struct hashmap *srcs, diff --combined object.c index cf1a2b7086,94db02214a..07bdd5b26e --- a/object.c +++ b/object.c @@@ -59,9 -59,9 +59,9 @@@ int type_from_string_gently(const char * the specified sha1. n must be a power of 2. Please note that the * return value is *not* consistent across computer architectures. */ - static unsigned int hash_obj(const unsigned char *sha1, unsigned int n) + static unsigned int hash_obj(const struct object_id *oid, unsigned int n) { - return sha1hash(sha1) & (n - 1); + return oidhash(oid) & (n - 1); } /* @@@ -71,7 -71,7 +71,7 @@@ */ static void insert_obj_hash(struct object *obj, struct object **hash, unsigned int size) { - unsigned int j = hash_obj(obj->oid.hash, size); + unsigned int j = hash_obj(&obj->oid, size); while (hash[j]) { j++; @@@ -85,7 -85,7 +85,7 @@@ * Look up the record for the given sha1 in the hash map stored in * obj_hash. Return NULL if it was not found. */ - struct object *lookup_object(struct repository *r, const unsigned char *sha1) + struct object *lookup_object(struct repository *r, const struct object_id *oid) { unsigned int i, first; struct object *obj; @@@ -93,9 -93,9 +93,9 @@@ if (!r->parsed_objects->obj_hash) return NULL; - first = i = hash_obj(sha1, r->parsed_objects->obj_hash_size); + first = i = hash_obj(oid, r->parsed_objects->obj_hash_size); while ((obj = r->parsed_objects->obj_hash[i]) != NULL) { - if (hasheq(sha1, obj->oid.hash)) + if (oideq(oid, &obj->oid)) break; i++; if (i == r->parsed_objects->obj_hash_size) @@@ -141,13 -141,13 +141,13 @@@ static void grow_object_hash(struct rep r->parsed_objects->obj_hash_size = new_hash_size; } - void *create_object(struct repository *r, const unsigned char *sha1, void *o) + void *create_object(struct repository *r, const struct object_id *oid, void *o) { struct object *obj = o; obj->parsed = 0; obj->flags = 0; - hashcpy(obj->oid.hash, sha1); + oidcpy(&obj->oid, oid); if (r->parsed_objects->obj_hash_size - 1 <= r->parsed_objects->nr_objs * 2) grow_object_hash(r); @@@ -178,11 -178,11 +178,11 @@@ void *object_as_type(struct repository } } - struct object *lookup_unknown_object(const unsigned char *sha1) + struct object *lookup_unknown_object(const struct object_id *oid) { - struct object *obj = lookup_object(the_repository, sha1); + struct object *obj = lookup_object(the_repository, oid); if (!obj) - obj = create_object(the_repository, sha1, + obj = create_object(the_repository, oid, alloc_object_node(the_repository)); return obj; } @@@ -256,7 -256,7 +256,7 @@@ struct object *parse_object(struct repo void *buffer; struct object *obj; - obj = lookup_object(r, oid->hash); + obj = lookup_object(r, oid); if (obj && obj->parsed) return obj; @@@ -268,7 -268,7 +268,7 @@@ return NULL; } parse_blob_buffer(lookup_blob(r, oid), NULL, 0); - return lookup_object(r, oid->hash); + return lookup_object(r, oid); } buffer = repo_read_object_file(r, oid, &type, &size); @@@ -517,7 -517,7 +517,7 @@@ void raw_object_store_clear(struct raw_ o->loaded_alternates = 0; INIT_LIST_HEAD(&o->packed_git_mru); - close_all_packs(o); + close_object_store(o); o->packed_git = NULL; } diff --combined upload-pack.c index b2a9f368ec,a0f170b5b5..222cd3ad89 --- a/upload-pack.c +++ b/upload-pack.c @@@ -528,13 -528,13 +528,13 @@@ static int get_reachable_list(struct ob return -1; while ((i = read_in_full(cmd.out, namebuf, hexsz + 1)) == hexsz + 1) { - struct object_id sha1; + struct object_id oid; const char *p; - if (parse_oid_hex(namebuf, &sha1, &p) || *p != '\n') + if (parse_oid_hex(namebuf, &oid, &p) || *p != '\n') break; - o = lookup_object(the_repository, sha1.hash); + o = lookup_object(the_repository, &oid); if (o && o->type == OBJ_COMMIT) { o->flags &= ~TMP_MARK; } @@@ -722,7 -722,7 +722,7 @@@ static void deepen_by_rev_list(struct p { struct commit_list *result; - close_commit_graph(the_repository); + close_commit_graph(the_repository->objects); result = get_shallow_commits_by_rev_list(ac, av, SHALLOW, NOT_SHALLOW); send_shallow(writer, result); free_commit_list(result); @@@ -960,7 -960,7 +960,7 @@@ static void receive_needs(struct packet static int mark_our_ref(const char *refname, const char *refname_full, const struct object_id *oid) { - struct object *o = lookup_unknown_object(oid->hash); + struct object *o = lookup_unknown_object(oid); if (ref_is_hidden(refname, refname_full)) { o->flags |= HIDDEN_REF;