From: Junio C Hamano Date: Tue, 5 Feb 2019 22:26:14 +0000 (-0800) Subject: Merge branch 'ab/commit-graph-write-optim' X-Git-Tag: v2.21.0-rc0~45 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/04d67b6ab222060546af328207ad48ae8ad58fe9?ds=inline;hp=-c Merge branch 'ab/commit-graph-write-optim' The codepath to write out commit-graph has been optimized by following the usual pattern of visiting objects in in-pack order. * ab/commit-graph-write-optim: commit-graph write: use pack order when finding commits --- 04d67b6ab222060546af328207ad48ae8ad58fe9 diff --combined commit-graph.c index 18bd2b6df7,981faf0465..8d52727def --- a/commit-graph.c +++ b/commit-graph.c @@@ -23,12 -23,18 +23,12 @@@ #define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */ #define GRAPH_CHUNKID_LARGEEDGES 0x45444745 /* "EDGE" */ -#define GRAPH_DATA_WIDTH 36 +#define GRAPH_DATA_WIDTH (the_hash_algo->rawsz + 16) #define GRAPH_VERSION_1 0x1 #define GRAPH_VERSION GRAPH_VERSION_1 -#define GRAPH_OID_VERSION_SHA1 1 -#define GRAPH_OID_LEN_SHA1 GIT_SHA1_RAWSZ -#define GRAPH_OID_VERSION GRAPH_OID_VERSION_SHA1 -#define GRAPH_OID_LEN GRAPH_OID_LEN_SHA1 - #define GRAPH_OCTOPUS_EDGES_NEEDED 0x80000000 -#define GRAPH_PARENT_MISSING 0x7fffffff #define GRAPH_EDGE_LAST_MASK 0x7fffffff #define GRAPH_PARENT_NONE 0x70000000 @@@ -38,18 -44,13 +38,18 @@@ #define GRAPH_FANOUT_SIZE (4 * 256) #define GRAPH_CHUNKLOOKUP_WIDTH 12 #define GRAPH_MIN_SIZE (GRAPH_HEADER_SIZE + 4 * GRAPH_CHUNKLOOKUP_WIDTH \ - + GRAPH_FANOUT_SIZE + GRAPH_OID_LEN) + + GRAPH_FANOUT_SIZE + the_hash_algo->rawsz) char *get_commit_graph_filename(const char *obj_dir) { return xstrfmt("%s/info/commit-graph", obj_dir); } +static uint8_t oid_version(void) +{ + return 1; +} + static struct commit_graph *alloc_commit_graph(void) { struct commit_graph *g = xcalloc(1, sizeof(*g)); @@@ -83,10 -84,16 +83,10 @@@ static int commit_graph_compatible(stru struct commit_graph *load_commit_graph_one(const char *graph_file) { void *graph_map; - const unsigned char *data, *chunk_lookup; size_t graph_size; struct stat st; - uint32_t i; - struct commit_graph *graph; + struct commit_graph *ret; int fd = git_open(graph_file); - uint64_t last_chunk_offset; - uint32_t last_chunk_id; - uint32_t graph_signature; - unsigned char graph_version, hash_version; if (fd < 0) return NULL; @@@ -101,60 -108,32 +101,60 @@@ die(_("graph file %s is too small"), graph_file); } graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0); + ret = parse_commit_graph(graph_map, fd, graph_size); + + if (!ret) { + munmap(graph_map, graph_size); + close(fd); + exit(1); + } + + return ret; +} + +struct commit_graph *parse_commit_graph(void *graph_map, int fd, + size_t graph_size) +{ + const unsigned char *data, *chunk_lookup; + uint32_t i; + struct commit_graph *graph; + uint64_t last_chunk_offset; + uint32_t last_chunk_id; + uint32_t graph_signature; + unsigned char graph_version, hash_version; + + if (!graph_map) + return NULL; + + if (graph_size < GRAPH_MIN_SIZE) + return NULL; + data = (const unsigned char *)graph_map; graph_signature = get_be32(data); if (graph_signature != GRAPH_SIGNATURE) { error(_("graph signature %X does not match signature %X"), graph_signature, GRAPH_SIGNATURE); - goto cleanup_fail; + return NULL; } graph_version = *(unsigned char*)(data + 4); if (graph_version != GRAPH_VERSION) { error(_("graph version %X does not match version %X"), graph_version, GRAPH_VERSION); - goto cleanup_fail; + return NULL; } hash_version = *(unsigned char*)(data + 5); - if (hash_version != GRAPH_OID_VERSION) { + if (hash_version != oid_version()) { error(_("hash version %X does not match version %X"), - hash_version, GRAPH_OID_VERSION); - goto cleanup_fail; + hash_version, oid_version()); + return NULL; } graph = alloc_commit_graph(); - graph->hash_len = GRAPH_OID_LEN; + graph->hash_len = the_hash_algo->rawsz; graph->num_chunks = *(unsigned char*)(data + 6); graph->graph_fd = fd; graph->data = graph_map; @@@ -164,27 -143,16 +164,27 @@@ last_chunk_offset = 8; chunk_lookup = data + 8; for (i = 0; i < graph->num_chunks; i++) { - uint32_t chunk_id = get_be32(chunk_lookup + 0); - uint64_t chunk_offset = get_be64(chunk_lookup + 4); + uint32_t chunk_id; + uint64_t chunk_offset; int chunk_repeated = 0; + if (data + graph_size - chunk_lookup < + GRAPH_CHUNKLOOKUP_WIDTH) { + error(_("chunk lookup table entry missing; graph file may be incomplete")); + free(graph); + return NULL; + } + + chunk_id = get_be32(chunk_lookup + 0); + chunk_offset = get_be64(chunk_lookup + 4); + chunk_lookup += GRAPH_CHUNKLOOKUP_WIDTH; - if (chunk_offset > graph_size - GIT_MAX_RAWSZ) { + if (chunk_offset > graph_size - the_hash_algo->rawsz) { error(_("improper chunk offset %08x%08x"), (uint32_t)(chunk_offset >> 32), (uint32_t)chunk_offset); - goto cleanup_fail; + free(graph); + return NULL; } switch (chunk_id) { @@@ -219,8 -187,7 +219,8 @@@ if (chunk_repeated) { error(_("chunk id %08x appears multiple times"), chunk_id); - goto cleanup_fail; + free(graph); + return NULL; } if (last_chunk_id == GRAPH_CHUNKID_OIDLOOKUP) @@@ -234,6 -201,11 +234,6 @@@ } return graph; - -cleanup_fail: - munmap(graph_map, graph_size); - close(fd); - exit(1); } static void prepare_commit_graph_one(struct repository *r, const char *obj_dir) @@@ -317,8 -289,7 +317,8 @@@ static int bsearch_graph(struct commit_ g->chunk_oid_lookup, g->hash_len, pos); } -static struct commit_list **insert_parent_or_die(struct commit_graph *g, +static struct commit_list **insert_parent_or_die(struct repository *r, + struct commit_graph *g, uint64_t pos, struct commit_list **pptr) { @@@ -329,7 -300,7 +329,7 @@@ die("invalid parent position %"PRIu64, pos); hashcpy(oid.hash, g->chunk_oid_lookup + g->hash_len * pos); - c = lookup_commit(the_repository, &oid); + c = lookup_commit(r, &oid); if (!c) die(_("could not find commit %s"), oid_to_hex(&oid)); c->graph_pos = pos; @@@ -343,9 -314,7 +343,9 @@@ static void fill_commit_graph_info(stru item->generation = get_be32(commit_data + g->hash_len + 8) >> 2; } -static int fill_commit_in_graph(struct commit *item, struct commit_graph *g, uint32_t pos) +static int fill_commit_in_graph(struct repository *r, + struct commit *item, + struct commit_graph *g, uint32_t pos) { uint32_t edge_value; uint32_t *parent_data_ptr; @@@ -369,13 -338,13 +369,13 @@@ edge_value = get_be32(commit_data + g->hash_len); if (edge_value == GRAPH_PARENT_NONE) return 1; - pptr = insert_parent_or_die(g, edge_value, pptr); + pptr = insert_parent_or_die(r, g, edge_value, pptr); edge_value = get_be32(commit_data + g->hash_len + 4); if (edge_value == GRAPH_PARENT_NONE) return 1; if (!(edge_value & GRAPH_OCTOPUS_EDGES_NEEDED)) { - pptr = insert_parent_or_die(g, edge_value, pptr); + pptr = insert_parent_or_die(r, g, edge_value, pptr); return 1; } @@@ -383,7 -352,7 +383,7 @@@ 4 * (uint64_t)(edge_value & GRAPH_EDGE_LAST_MASK)); do { edge_value = get_be32(parent_data_ptr); - pptr = insert_parent_or_die(g, + pptr = insert_parent_or_die(r, g, edge_value & GRAPH_EDGE_LAST_MASK, pptr); parent_data_ptr++; @@@ -402,9 -371,7 +402,9 @@@ static int find_commit_in_graph(struct } } -static int parse_commit_in_graph_one(struct commit_graph *g, struct commit *item) +static int parse_commit_in_graph_one(struct repository *r, + struct commit_graph *g, + struct commit *item) { uint32_t pos; @@@ -412,7 -379,7 +412,7 @@@ return 1; if (find_commit_in_graph(item, g, &pos)) - return fill_commit_in_graph(item, g, pos); + return fill_commit_in_graph(r, item, g, pos); return 0; } @@@ -421,7 -388,7 +421,7 @@@ int parse_commit_in_graph(struct reposi { if (!prepare_commit_graph(r)) return 0; - return parse_commit_in_graph_one(r->objects->commit_graph, item); + return parse_commit_in_graph_one(r, r->objects->commit_graph, item); } void load_commit_graph_info(struct repository *r, struct commit *item) @@@ -433,22 -400,19 +433,22 @@@ fill_commit_graph_info(item, r->objects->commit_graph, pos); } -static struct tree *load_tree_for_commit(struct commit_graph *g, struct commit *c) +static struct tree *load_tree_for_commit(struct repository *r, + struct commit_graph *g, + struct commit *c) { struct object_id oid; const unsigned char *commit_data = g->chunk_commit_data + GRAPH_DATA_WIDTH * (c->graph_pos); hashcpy(oid.hash, commit_data); - c->maybe_tree = lookup_tree(the_repository, &oid); + c->maybe_tree = lookup_tree(r, &oid); return c->maybe_tree; } -static struct tree *get_commit_tree_in_graph_one(struct commit_graph *g, +static struct tree *get_commit_tree_in_graph_one(struct repository *r, + struct commit_graph *g, const struct commit *c) { if (c->maybe_tree) @@@ -456,12 -420,12 +456,12 @@@ if (c->graph_pos == COMMIT_NOT_FROM_GRAPH) BUG("get_commit_tree_in_graph_one called from non-commit-graph commit"); - return load_tree_for_commit(g, (struct commit *)c); + return load_tree_for_commit(r, g, (struct commit *)c); } struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit *c) { - return get_commit_tree_in_graph_one(r->objects->commit_graph, c); + return get_commit_tree_in_graph_one(r, r->objects->commit_graph, c); } static void write_graph_chunk_fanout(struct hashfile *f, @@@ -529,9 -493,7 +529,9 @@@ static void write_graph_chunk_data(stru commit_to_sha1); if (edge_value < 0) - edge_value = GRAPH_PARENT_MISSING; + BUG("missing parent %s for commit %s", + oid_to_hex(&parent->item->object.oid), + oid_to_hex(&(*list)->object.oid)); } hashwrite_be32(f, edge_value); @@@ -549,9 -511,7 +549,9 @@@ nr_commits, commit_to_sha1); if (edge_value < 0) - edge_value = GRAPH_PARENT_MISSING; + BUG("missing parent %s for commit %s", + oid_to_hex(&parent->item->object.oid), + oid_to_hex(&(*list)->object.oid)); } hashwrite_be32(f, edge_value); @@@ -604,9 -564,7 +604,9 @@@ static void write_graph_chunk_large_edg commit_to_sha1); if (edge_value < 0) - edge_value = GRAPH_PARENT_MISSING; + BUG("missing parent %s for commit %s", + oid_to_hex(&parent->item->object.oid), + oid_to_hex(&(*list)->object.oid)); else if (!parent->next) edge_value |= GRAPH_LAST_EDGE; @@@ -810,7 -768,6 +810,7 @@@ void write_commit_graph(const char *obj int num_extra_edges; struct commit_list *parent; struct progress *progress = NULL; + const unsigned hashsz = the_hash_algo->rawsz; if (!commit_graph_compatible(the_repository)) return; @@@ -859,7 -816,8 +859,8 @@@ die(_("error adding pack %s"), packname.buf); if (open_pack_index(p)) die(_("error opening index for %s"), packname.buf); - for_each_object_in_pack(p, add_packed_commits, &oids, 0); + for_each_object_in_pack(p, add_packed_commits, &oids, + FOR_EACH_OBJECT_PACK_ORDER); close_pack(p); free(p); } @@@ -897,7 -855,8 +898,8 @@@ if (report_progress) oids.progress = start_delayed_progress( _("Finding commits for commit graph"), 0); - for_each_packed_object(add_packed_commits, &oids, 0); + for_each_packed_object(add_packed_commits, &oids, + FOR_EACH_OBJECT_PACK_ORDER); stop_progress(&oids.progress); } @@@ -911,7 -870,7 +913,7 @@@ count_distinct++; } - if (count_distinct >= GRAPH_PARENT_MISSING) + if (count_distinct >= GRAPH_EDGE_LAST_MASK) die(_("the commit graph format cannot write %d commits"), count_distinct); commits.nr = 0; @@@ -938,7 -897,7 +940,7 @@@ } num_chunks = num_extra_edges ? 4 : 3; - if (commits.nr >= GRAPH_PARENT_MISSING) + if (commits.nr >= GRAPH_EDGE_LAST_MASK) die(_("too many commits to write graph")); compute_generation_numbers(&commits, report_progress); @@@ -956,7 -915,7 +958,7 @@@ hashwrite_be32(f, GRAPH_SIGNATURE); hashwrite_u8(f, GRAPH_VERSION); - hashwrite_u8(f, GRAPH_OID_VERSION); + hashwrite_u8(f, oid_version()); hashwrite_u8(f, num_chunks); hashwrite_u8(f, 0); /* unused padding byte */ @@@ -971,8 -930,8 +973,8 @@@ chunk_offsets[0] = 8 + (num_chunks + 1) * GRAPH_CHUNKLOOKUP_WIDTH; chunk_offsets[1] = chunk_offsets[0] + GRAPH_FANOUT_SIZE; - chunk_offsets[2] = chunk_offsets[1] + GRAPH_OID_LEN * commits.nr; - chunk_offsets[3] = chunk_offsets[2] + (GRAPH_OID_LEN + 16) * commits.nr; + chunk_offsets[2] = chunk_offsets[1] + hashsz * commits.nr; + chunk_offsets[3] = chunk_offsets[2] + (hashsz + 16) * commits.nr; chunk_offsets[4] = chunk_offsets[3] + 4 * num_extra_edges; for (i = 0; i <= num_chunks; i++) { @@@ -985,8 -944,8 +987,8 @@@ } write_graph_chunk_fanout(f, commits.list, commits.nr); - write_graph_chunk_oids(f, GRAPH_OID_LEN, commits.list, commits.nr); - write_graph_chunk_data(f, GRAPH_OID_LEN, commits.list, commits.nr); + write_graph_chunk_oids(f, hashsz, commits.list, commits.nr); + write_graph_chunk_data(f, hashsz, commits.list, commits.nr); write_graph_chunk_large_edges(f, commits.list, commits.nr); close_commit_graph(the_repository); @@@ -1072,7 -1031,7 +1074,7 @@@ int verify_commit_graph(struct reposito } graph_commit = lookup_commit(r, &cur_oid); - if (!parse_commit_in_graph_one(g, graph_commit)) + if (!parse_commit_in_graph_one(r, g, graph_commit)) graph_report("failed to parse %s from commit-graph", oid_to_hex(&cur_oid)); } @@@ -1108,7 -1067,7 +1110,7 @@@ continue; } - if (!oideq(&get_commit_tree_in_graph_one(g, graph_commit)->object.oid, + if (!oideq(&get_commit_tree_in_graph_one(r, g, graph_commit)->object.oid, get_commit_tree_oid(odb_commit))) graph_report("root tree OID for commit %s in commit-graph is %s != %s", oid_to_hex(&cur_oid),