From: Junio C Hamano Date: Tue, 16 Oct 2018 07:15:58 +0000 (+0900) Subject: Merge branch 'ab/commit-graph-progress' X-Git-Tag: v2.20.0-rc0~194 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/36d767d02e12c1ff991511b7bd8c1d023340f2ca?hp=-c Merge branch 'ab/commit-graph-progress' Generation of (experimental) commit-graph files have so far been fairly silent, even though it takes noticeable amount of time in a meaningfully large repository. The users will now see progress output. * ab/commit-graph-progress: gc: fix regression in 7b0f229222 impacting --quiet commit-graph verify: add progress output commit-graph write: add progress output --- 36d767d02e12c1ff991511b7bd8c1d023340f2ca diff --combined builtin/commit.c index b57d8e4b82,0d9828e29e..3f33ae6043 --- a/builtin/commit.c +++ b/builtin/commit.c @@@ -33,8 -33,6 +33,8 @@@ #include "sequencer.h" #include "mailmap.h" #include "help.h" +#include "commit-reach.h" +#include "commit-graph.h" static const char * const builtin_commit_usage[] = { N_("git commit [] [--] ..."), @@@ -1653,9 -1651,6 +1653,9 @@@ int cmd_commit(int argc, const char **a "new_index file. Check that disk is not full and quota is\n" "not exceeded, and then \"git reset HEAD\" to recover.")); + if (git_env_bool(GIT_TEST_COMMIT_GRAPH, 0)) - write_commit_graph_reachable(get_object_directory(), 0); ++ write_commit_graph_reachable(get_object_directory(), 0, 0); + rerere(0); run_command_v_opt(argv_gc_auto, RUN_GIT_CMD); run_commit_hook(use_editor, get_index_file(), "post-commit", NULL); diff --combined builtin/gc.c index 2b592260e9,91ffb1a803..6591ddbe83 --- a/builtin/gc.c +++ b/builtin/gc.c @@@ -183,7 -183,7 +183,7 @@@ static struct packed_git *find_base_pac { struct packed_git *p, *base = NULL; - for (p = get_packed_git(the_repository); p; p = p->next) { + for (p = get_all_packs(the_repository); p; p = p->next) { if (!p->pack_local) continue; if (limit) { @@@ -208,7 -208,7 +208,7 @@@ static int too_many_packs(void if (gc_auto_pack_limit <= 0) return 0; - for (cnt = 0, p = get_packed_git(the_repository); p; p = p->next) { + for (cnt = 0, p = get_all_packs(the_repository); p; p = p->next) { if (!p->pack_local) continue; if (p->pack_keep) @@@ -646,7 -646,8 +646,8 @@@ int cmd_gc(int argc, const char **argv clean_pack_garbage(); if (gc_write_commit_graph) - write_commit_graph_reachable(get_object_directory(), 0); + write_commit_graph_reachable(get_object_directory(), 0, + !quiet && !daemonized); if (auto_gc && too_many_loose_objects()) warning(_("There are too many unreachable loose objects; " diff --combined commit-graph.c index 7f4519ec3b,2a24eb8b5a..5908bd4e34 --- a/commit-graph.c +++ b/commit-graph.c @@@ -13,6 -13,7 +13,7 @@@ #include "commit-graph.h" #include "object-store.h" #include "alloc.h" + #include "progress.h" #define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */ #define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */ @@@ -213,9 -214,8 +214,9 @@@ static int prepare_commit_graph(struct return !!r->objects->commit_graph; r->objects->commit_graph_attempted = 1; - if (repo_config_get_bool(r, "core.commitgraph", &config_value) || - !config_value) + if (!git_env_bool(GIT_TEST_COMMIT_GRAPH, 0) && + (repo_config_get_bool(r, "core.commitgraph", &config_value) || + !config_value)) /* * This repository is not configured to use commit graphs, so * do not load one. (But report commit_graph_attempted anyway @@@ -234,24 -234,6 +235,24 @@@ return !!r->objects->commit_graph; } +int generation_numbers_enabled(struct repository *r) +{ + uint32_t first_generation; + struct commit_graph *g; + if (!prepare_commit_graph(r)) + return 0; + + g = r->objects->commit_graph; + + if (!g->num_commits) + return 0; + + first_generation = get_be32(g->chunk_commit_data + + g->hash_len + 8) >> 2; + + return !!first_generation; +} + static void close_commit_graph(void) { free_commit_graph(the_repository->objects->commit_graph); @@@ -567,6 -549,8 +568,8 @@@ struct packed_oid_list struct object_id *list; int nr; int alloc; + struct progress *progress; + int progress_done; }; static int add_packed_commits(const struct object_id *oid, @@@ -579,6 -563,9 +582,9 @@@ off_t offset = nth_packed_object_offset(pack, pos); struct object_info oi = OBJECT_INFO_INIT; + if (list->progress) + display_progress(list->progress, ++list->progress_done); + oi.typep = &type; if (packed_object_info(the_repository, pack, offset, &oi) < 0) die(_("unable to get type of object %s"), oid_to_hex(oid)); @@@ -606,12 -593,18 +612,18 @@@ static void add_missing_parents(struct } } - static void close_reachable(struct packed_oid_list *oids) + static void close_reachable(struct packed_oid_list *oids, int report_progress) { int i; struct commit *commit; + struct progress *progress = NULL; + int j = 0; + if (report_progress) + progress = start_delayed_progress( + _("Annotating commits in commit graph"), 0); for (i = 0; i < oids->nr; i++) { + display_progress(progress, ++j); commit = lookup_commit(the_repository, &oids->list[i]); if (commit) commit->object.flags |= UNINTERESTING; @@@ -623,6 -616,7 +635,7 @@@ * closure. */ for (i = 0; i < oids->nr; i++) { + display_progress(progress, ++j); commit = lookup_commit(the_repository, &oids->list[i]); if (commit && !parse_commit(commit)) @@@ -630,19 -624,28 +643,28 @@@ } for (i = 0; i < oids->nr; i++) { + display_progress(progress, ++j); commit = lookup_commit(the_repository, &oids->list[i]); if (commit) commit->object.flags &= ~UNINTERESTING; } + stop_progress(&progress); } - static void compute_generation_numbers(struct packed_commit_list* commits) + static void compute_generation_numbers(struct packed_commit_list* commits, + int report_progress) { int i; struct commit_list *list = NULL; + struct progress *progress = NULL; + if (report_progress) + progress = start_progress( + _("Computing commit graph generation numbers"), + commits->nr); for (i = 0; i < commits->nr; i++) { + display_progress(progress, i + 1); if (commits->list[i]->generation != GENERATION_NUMBER_INFINITY && commits->list[i]->generation != GENERATION_NUMBER_ZERO) continue; @@@ -674,6 -677,7 +696,7 @@@ } } } + stop_progress(&progress); } static int add_ref_to_list(const char *refname, @@@ -686,19 -690,20 +709,20 @@@ return 0; } - void write_commit_graph_reachable(const char *obj_dir, int append) + void write_commit_graph_reachable(const char *obj_dir, int append, + int report_progress) { struct string_list list; string_list_init(&list, 1); for_each_ref(add_ref_to_list, &list); - write_commit_graph(obj_dir, NULL, &list, append); + write_commit_graph(obj_dir, NULL, &list, append, report_progress); } void write_commit_graph(const char *obj_dir, struct string_list *pack_indexes, struct string_list *commit_hex, - int append) + int append, int report_progress) { struct packed_oid_list oids; struct packed_commit_list commits; @@@ -711,9 -716,12 +735,12 @@@ int num_chunks; int num_extra_edges; struct commit_list *parent; + struct progress *progress = NULL; oids.nr = 0; oids.alloc = approximate_object_count() / 4; + oids.progress = NULL; + oids.progress_done = 0; if (append) { prepare_commit_graph_one(the_repository, obj_dir); @@@ -740,6 -748,11 +767,11 @@@ int dirlen; strbuf_addf(&packname, "%s/pack/", obj_dir); dirlen = packname.len; + if (report_progress) { + oids.progress = start_delayed_progress( + _("Finding commits for commit graph"), 0); + oids.progress_done = 0; + } for (i = 0; i < pack_indexes->nr; i++) { struct packed_git *p; strbuf_setlen(&packname, dirlen); @@@ -752,15 -765,21 +784,21 @@@ for_each_object_in_pack(p, add_packed_commits, &oids, 0); close_pack(p); } + stop_progress(&oids.progress); strbuf_release(&packname); } if (commit_hex) { + if (report_progress) + progress = start_delayed_progress( + _("Finding commits for commit graph"), + commit_hex->nr); for (i = 0; i < commit_hex->nr; i++) { const char *end; struct object_id oid; struct commit *result; + display_progress(progress, i + 1); if (commit_hex->items[i].string && parse_oid_hex(commit_hex->items[i].string, &oid, &end)) continue; @@@ -773,18 -792,24 +811,24 @@@ oids.nr++; } } + stop_progress(&progress); } - if (!pack_indexes && !commit_hex) + if (!pack_indexes && !commit_hex) { + if (report_progress) + oids.progress = start_delayed_progress( + _("Finding commits for commit graph"), 0); for_each_packed_object(add_packed_commits, &oids, 0); + stop_progress(&oids.progress); + } - close_reachable(&oids); + close_reachable(&oids, report_progress); QSORT(oids.list, oids.nr, commit_compare); count_distinct = 1; for (i = 1; i < oids.nr; i++) { - if (oidcmp(&oids.list[i-1], &oids.list[i])) + if (!oideq(&oids.list[i - 1], &oids.list[i])) count_distinct++; } @@@ -798,7 -823,7 +842,7 @@@ num_extra_edges = 0; for (i = 0; i < oids.nr; i++) { int num_parents = 0; - if (i > 0 && !oidcmp(&oids.list[i-1], &oids.list[i])) + if (i > 0 && oideq(&oids.list[i - 1], &oids.list[i])) continue; commits.list[commits.nr] = lookup_commit(the_repository, &oids.list[i]); @@@ -818,7 -843,7 +862,7 @@@ if (commits.nr >= GRAPH_PARENT_MISSING) die(_("too many commits to write graph")); - compute_generation_numbers(&commits); + compute_generation_numbers(&commits, report_progress); graph_name = get_commit_graph_filename(obj_dir); if (safe_create_leading_directories(graph_name)) @@@ -897,6 -922,7 +941,7 @@@ int verify_commit_graph(struct reposito int generation_zero = 0; struct hashfile *f; int devnull; + struct progress *progress = NULL; if (!g) { graph_report("no commit-graph file loaded"); @@@ -919,7 -945,7 +964,7 @@@ f = hashfd(devnull, NULL); hashwrite(f, g->data, g->data_len - g->hash_len); finalize_hashfile(f, checksum.hash, CSUM_CLOSE); - if (hashcmp(checksum.hash, g->data + g->data_len - g->hash_len)) { + if (!hasheq(checksum.hash, g->data + g->data_len - g->hash_len)) { graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt")); verify_commit_graph_error = VERIFY_COMMIT_GRAPH_ERROR_HASH; } @@@ -964,11 -990,14 +1009,14 @@@ if (verify_commit_graph_error & ~VERIFY_COMMIT_GRAPH_ERROR_HASH) return verify_commit_graph_error; + progress = start_progress(_("Verifying commits in commit graph"), + g->num_commits); for (i = 0; i < g->num_commits; i++) { struct commit *graph_commit, *odb_commit; struct commit_list *graph_parents, *odb_parents; uint32_t max_generation = 0; + display_progress(progress, i + 1); hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i); graph_commit = lookup_commit(r, &cur_oid); @@@ -979,7 -1008,7 +1027,7 @@@ continue; } - if (oidcmp(&get_commit_tree_in_graph_one(g, graph_commit)->object.oid, + if (!oideq(&get_commit_tree_in_graph_one(g, graph_commit)->object.oid, get_commit_tree_oid(odb_commit))) graph_report("root tree OID for commit %s in commit-graph is %s != %s", oid_to_hex(&cur_oid), @@@ -996,7 -1025,7 +1044,7 @@@ break; } - if (oidcmp(&graph_parents->item->object.oid, &odb_parents->item->object.oid)) + if (!oideq(&graph_parents->item->object.oid, &odb_parents->item->object.oid)) graph_report("commit-graph parent for %s is %s != %s", oid_to_hex(&cur_oid), oid_to_hex(&graph_parents->item->object.oid), @@@ -1045,6 -1074,7 +1093,7 @@@ graph_commit->date, odb_commit->date); } + stop_progress(&progress); return verify_commit_graph_error; } diff --combined commit-graph.h index b050476765,f50712a973..5678a8f4ca --- a/commit-graph.h +++ b/commit-graph.h @@@ -6,8 -6,6 +6,8 @@@ #include "string-list.h" #include "cache.h" +#define GIT_TEST_COMMIT_GRAPH "GIT_TEST_COMMIT_GRAPH" + struct commit; char *get_commit_graph_filename(const char *obj_dir); @@@ -54,17 -52,12 +54,18 @@@ struct commit_graph struct commit_graph *load_commit_graph_one(const char *graph_file); +/* + * Return 1 if and only if the repository has a commit-graph + * file and generation numbers are computed in that file. + */ +int generation_numbers_enabled(struct repository *r); + - void write_commit_graph_reachable(const char *obj_dir, int append); + void write_commit_graph_reachable(const char *obj_dir, int append, + int report_progress); void write_commit_graph(const char *obj_dir, struct string_list *pack_indexes, struct string_list *commit_hex, - int append); + int append, int report_progress); int verify_commit_graph(struct repository *r, struct commit_graph *g);