From: Junio C Hamano Date: Tue, 17 Jul 2018 22:46:19 +0000 (-0700) Subject: Merge branch 'ds/commit-graph-fsck' into jt/commit-graph-per-object-store X-Git-Tag: v2.19.0-rc0~89^2~6 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/8295296458bfa5e371dccae0a0e0a4b9a56f9b40 Merge branch 'ds/commit-graph-fsck' into jt/commit-graph-per-object-store * ds/commit-graph-fsck: (23 commits) coccinelle: update commit.cocci commit-graph: update design document gc: automatically write commit-graph files commit-graph: add '--reachable' option commit-graph: use string-list API for input fsck: verify commit-graph commit-graph: verify contents match checksum commit-graph: test for corrupted octopus edge commit-graph: verify commit date commit-graph: verify generation number commit-graph: verify parent list commit-graph: verify root tree OIDs commit-graph: verify objects exist commit-graph: verify corrupt OID fanout and lookup commit-graph: verify required chunks are present commit-graph: verify catches corrupt signature commit-graph: add 'verify' subcommand commit-graph: load a root tree from specific graph commit: force commit to parse from object database commit-graph: parse commit from chosen graph ... --- 8295296458bfa5e371dccae0a0e0a4b9a56f9b40 diff --cc commit-graph.c index 7f907b4bfb,212232e752..41a0133ff7 --- a/commit-graph.c +++ b/commit-graph.c @@@ -241,8 -244,12 +244,12 @@@ static struct commit_list **insert_pare { struct commit *c; struct object_id oid; + + if (pos >= g->num_commits) + die("invalid parent position %"PRIu64, pos); + hashcpy(oid.hash, g->chunk_oid_lookup + g->hash_len * pos); - c = lookup_commit(&oid); + c = lookup_commit(the_repository, &oid); if (!c) die("could not find commit %s", oid_to_hex(&oid)); c->graph_pos = pos; @@@ -698,10 -740,11 +740,11 @@@ void write_commit_graph(const char *obj struct object_id oid; struct commit *result; - if (commit_hex[i] && parse_oid_hex(commit_hex[i], &oid, &end)) + if (commit_hex->items[i].string && + parse_oid_hex(commit_hex->items[i].string, &oid, &end)) continue; - result = lookup_commit_reference_gently(&oid, 1); + result = lookup_commit_reference_gently(the_repository, &oid, 1); if (result) { ALLOC_GROW(oids.list, oids.nr + 1, oids.alloc); @@@ -808,3 -851,179 +851,179 @@@ oids.alloc = 0; oids.nr = 0; } + + #define VERIFY_COMMIT_GRAPH_ERROR_HASH 2 + static int verify_commit_graph_error; + + static void graph_report(const char *fmt, ...) + { + va_list ap; + + verify_commit_graph_error = 1; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); + va_end(ap); + } + + #define GENERATION_ZERO_EXISTS 1 + #define GENERATION_NUMBER_EXISTS 2 + + int verify_commit_graph(struct repository *r, struct commit_graph *g) + { + uint32_t i, cur_fanout_pos = 0; + struct object_id prev_oid, cur_oid, checksum; + int generation_zero = 0; + struct hashfile *f; + int devnull; + + if (!g) { + graph_report("no commit-graph file loaded"); + return 1; + } + + verify_commit_graph_error = 0; + + if (!g->chunk_oid_fanout) + graph_report("commit-graph is missing the OID Fanout chunk"); + if (!g->chunk_oid_lookup) + graph_report("commit-graph is missing the OID Lookup chunk"); + if (!g->chunk_commit_data) + graph_report("commit-graph is missing the Commit Data chunk"); + + if (verify_commit_graph_error) + return verify_commit_graph_error; + + devnull = open("/dev/null", O_WRONLY); + f = hashfd(devnull, NULL); + hashwrite(f, g->data, g->data_len - g->hash_len); + finalize_hashfile(f, checksum.hash, CSUM_CLOSE); + if (hashcmp(checksum.hash, g->data + g->data_len - g->hash_len)) { + graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt")); + verify_commit_graph_error = VERIFY_COMMIT_GRAPH_ERROR_HASH; + } + + for (i = 0; i < g->num_commits; i++) { + struct commit *graph_commit; + + hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i); + + if (i && oidcmp(&prev_oid, &cur_oid) >= 0) + graph_report("commit-graph has incorrect OID order: %s then %s", + oid_to_hex(&prev_oid), + oid_to_hex(&cur_oid)); + + oidcpy(&prev_oid, &cur_oid); + + while (cur_oid.hash[0] > cur_fanout_pos) { + uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos); + + if (i != fanout_value) + graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u", + cur_fanout_pos, fanout_value, i); + cur_fanout_pos++; + } + - graph_commit = lookup_commit(&cur_oid); ++ graph_commit = lookup_commit(r, &cur_oid); + if (!parse_commit_in_graph_one(g, graph_commit)) + graph_report("failed to parse %s from commit-graph", + oid_to_hex(&cur_oid)); + } + + while (cur_fanout_pos < 256) { + uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos); + + if (g->num_commits != fanout_value) + graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u", + cur_fanout_pos, fanout_value, i); + + cur_fanout_pos++; + } + + if (verify_commit_graph_error & ~VERIFY_COMMIT_GRAPH_ERROR_HASH) + return verify_commit_graph_error; + + for (i = 0; i < g->num_commits; i++) { + struct commit *graph_commit, *odb_commit; + struct commit_list *graph_parents, *odb_parents; + uint32_t max_generation = 0; + + hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i); + - graph_commit = lookup_commit(&cur_oid); ++ graph_commit = lookup_commit(r, &cur_oid); + odb_commit = (struct commit *)create_object(r, cur_oid.hash, alloc_commit_node(r)); + if (parse_commit_internal(odb_commit, 0, 0)) { + graph_report("failed to parse %s from object database", + oid_to_hex(&cur_oid)); + continue; + } + + if (oidcmp(&get_commit_tree_in_graph_one(g, graph_commit)->object.oid, + get_commit_tree_oid(odb_commit))) + graph_report("root tree OID for commit %s in commit-graph is %s != %s", + oid_to_hex(&cur_oid), + oid_to_hex(get_commit_tree_oid(graph_commit)), + oid_to_hex(get_commit_tree_oid(odb_commit))); + + graph_parents = graph_commit->parents; + odb_parents = odb_commit->parents; + + while (graph_parents) { + if (odb_parents == NULL) { + graph_report("commit-graph parent list for commit %s is too long", + oid_to_hex(&cur_oid)); + break; + } + + if (oidcmp(&graph_parents->item->object.oid, &odb_parents->item->object.oid)) + graph_report("commit-graph parent for %s is %s != %s", + oid_to_hex(&cur_oid), + oid_to_hex(&graph_parents->item->object.oid), + oid_to_hex(&odb_parents->item->object.oid)); + + if (graph_parents->item->generation > max_generation) + max_generation = graph_parents->item->generation; + + graph_parents = graph_parents->next; + odb_parents = odb_parents->next; + } + + if (odb_parents != NULL) + graph_report("commit-graph parent list for commit %s terminates early", + oid_to_hex(&cur_oid)); + + if (!graph_commit->generation) { + if (generation_zero == GENERATION_NUMBER_EXISTS) + graph_report("commit-graph has generation number zero for commit %s, but non-zero elsewhere", + oid_to_hex(&cur_oid)); + generation_zero = GENERATION_ZERO_EXISTS; + } else if (generation_zero == GENERATION_ZERO_EXISTS) + graph_report("commit-graph has non-zero generation number for commit %s, but zero elsewhere", + oid_to_hex(&cur_oid)); + + if (generation_zero == GENERATION_ZERO_EXISTS) + continue; + + /* + * If one of our parents has generation GENERATION_NUMBER_MAX, then + * our generation is also GENERATION_NUMBER_MAX. Decrement to avoid + * extra logic in the following condition. + */ + if (max_generation == GENERATION_NUMBER_MAX) + max_generation--; + + if (graph_commit->generation != max_generation + 1) + graph_report("commit-graph generation for commit %s is %u != %u", + oid_to_hex(&cur_oid), + graph_commit->generation, + max_generation + 1); + + if (graph_commit->date != odb_commit->date) + graph_report("commit date for commit %s in commit-graph is %"PRItime" != %"PRItime, + oid_to_hex(&cur_oid), + graph_commit->date, + odb_commit->date); + } + + return verify_commit_graph_error; + } diff --cc commit.c index b88ced5b02,598cf21cee..c0a83d2644 --- a/commit.c +++ b/commit.c @@@ -466,9 -441,10 +466,10 @@@ int parse_commit_internal(struct commi return error("Object %s not a commit", oid_to_hex(&item->object.oid)); } + - ret = parse_commit_buffer(item, buffer, size, 0); + ret = parse_commit_buffer(the_repository, item, buffer, size, 0); if (save_commit_buffer && !ret) { - set_commit_buffer(item, buffer, size); + set_commit_buffer(the_repository, item, buffer, size); return 0; } free(buffer); diff --cc commit.h index 8b2cf9692d,7e0f273720..da0db36eba --- a/commit.h +++ b/commit.h @@@ -78,7 -76,8 +78,8 @@@ struct commit *lookup_commit_reference_ */ struct commit *lookup_commit_or_die(const struct object_id *oid, const char *ref_name); -int parse_commit_buffer(struct commit *item, const void *buffer, unsigned long size, int check_graph); +int parse_commit_buffer(struct repository *r, struct commit *item, const void *buffer, unsigned long size, int check_graph); + int parse_commit_internal(struct commit *item, int quiet_on_missing, int use_commit_graph); int parse_commit_gently(struct commit *item, int quiet_on_missing); static inline int parse_commit(struct commit *item) {