Add initial support for many promisor remotes
[gitweb.git] / commit-graph.c
index 3f9c03afd57ef1b45a1aa65f648f4cf153aaad96..7c5e54875fdacdf77235a077c9928f7d3bf0d001 100644 (file)
 #define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
 #define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
 #define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
-#define GRAPH_CHUNKID_LARGEEDGES 0x45444745 /* "EDGE" */
+#define GRAPH_CHUNKID_EXTRAEDGES 0x45444745 /* "EDGE" */
 
 #define GRAPH_DATA_WIDTH (the_hash_algo->rawsz + 16)
 
 #define GRAPH_VERSION_1 0x1
 #define GRAPH_VERSION GRAPH_VERSION_1
 
-#define GRAPH_OCTOPUS_EDGES_NEEDED 0x80000000
+#define GRAPH_EXTRA_EDGES_NEEDED 0x80000000
 #define GRAPH_EDGE_LAST_MASK 0x7fffffff
 #define GRAPH_PARENT_NONE 0x70000000
 
@@ -80,54 +80,110 @@ static int commit_graph_compatible(struct repository *r)
        return 1;
 }
 
-struct commit_graph *load_commit_graph_one(const char *graph_file)
+int open_commit_graph(const char *graph_file, int *fd, struct stat *st)
+{
+       *fd = git_open(graph_file);
+       if (*fd < 0)
+               return 0;
+       if (fstat(*fd, st)) {
+               close(*fd);
+               return 0;
+       }
+       return 1;
+}
+
+struct commit_graph *load_commit_graph_one_fd_st(int fd, struct stat *st)
 {
        void *graph_map;
-       const unsigned char *data, *chunk_lookup;
        size_t graph_size;
-       struct stat st;
+       struct commit_graph *ret;
+
+       graph_size = xsize_t(st->st_size);
+
+       if (graph_size < GRAPH_MIN_SIZE) {
+               close(fd);
+               error(_("commit-graph file is too small"));
+               return NULL;
+       }
+       graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0);
+       ret = parse_commit_graph(graph_map, fd, graph_size);
+
+       if (!ret) {
+               munmap(graph_map, graph_size);
+               close(fd);
+       }
+
+       return ret;
+}
+
+static int verify_commit_graph_lite(struct commit_graph *g)
+{
+       /*
+        * Basic validation shared between parse_commit_graph()
+        * which'll be called every time the graph is used, and the
+        * much more expensive verify_commit_graph() used by
+        * "commit-graph verify".
+        *
+        * There should only be very basic checks here to ensure that
+        * we don't e.g. segfault in fill_commit_in_graph(), but
+        * because this is a very hot codepath nothing that e.g. loops
+        * over g->num_commits, or runs a checksum on the commit-graph
+        * itself.
+        */
+       if (!g->chunk_oid_fanout) {
+               error("commit-graph is missing the OID Fanout chunk");
+               return 1;
+       }
+       if (!g->chunk_oid_lookup) {
+               error("commit-graph is missing the OID Lookup chunk");
+               return 1;
+       }
+       if (!g->chunk_commit_data) {
+               error("commit-graph is missing the Commit Data chunk");
+               return 1;
+       }
+
+       return 0;
+}
+
+struct commit_graph *parse_commit_graph(void *graph_map, int fd,
+                                       size_t graph_size)
+{
+       const unsigned char *data, *chunk_lookup;
        uint32_t i;
        struct commit_graph *graph;
-       int fd = git_open(graph_file);
        uint64_t last_chunk_offset;
        uint32_t last_chunk_id;
        uint32_t graph_signature;
        unsigned char graph_version, hash_version;
 
-       if (fd < 0)
+       if (!graph_map)
                return NULL;
-       if (fstat(fd, &st)) {
-               close(fd);
+
+       if (graph_size < GRAPH_MIN_SIZE)
                return NULL;
-       }
-       graph_size = xsize_t(st.st_size);
 
-       if (graph_size < GRAPH_MIN_SIZE) {
-               close(fd);
-               die(_("graph file %s is too small"), graph_file);
-       }
-       graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0);
        data = (const unsigned char *)graph_map;
 
        graph_signature = get_be32(data);
        if (graph_signature != GRAPH_SIGNATURE) {
-               error(_("graph signature %X does not match signature %X"),
+               error(_("commit-graph signature %X does not match signature %X"),
                      graph_signature, GRAPH_SIGNATURE);
-               goto cleanup_fail;
+               return NULL;
        }
 
        graph_version = *(unsigned char*)(data + 4);
        if (graph_version != GRAPH_VERSION) {
-               error(_("graph version %X does not match version %X"),
+               error(_("commit-graph version %X does not match version %X"),
                      graph_version, GRAPH_VERSION);
-               goto cleanup_fail;
+               return NULL;
        }
 
        hash_version = *(unsigned char*)(data + 5);
        if (hash_version != oid_version()) {
-               error(_("hash version %X does not match version %X"),
+               error(_("commit-graph hash version %X does not match version %X"),
                      hash_version, oid_version());
-               goto cleanup_fail;
+               return NULL;
        }
 
        graph = alloc_commit_graph();
@@ -142,16 +198,27 @@ struct commit_graph *load_commit_graph_one(const char *graph_file)
        last_chunk_offset = 8;
        chunk_lookup = data + 8;
        for (i = 0; i < graph->num_chunks; i++) {
-               uint32_t chunk_id = get_be32(chunk_lookup + 0);
-               uint64_t chunk_offset = get_be64(chunk_lookup + 4);
+               uint32_t chunk_id;
+               uint64_t chunk_offset;
                int chunk_repeated = 0;
 
+               if (data + graph_size - chunk_lookup <
+                   GRAPH_CHUNKLOOKUP_WIDTH) {
+                       error(_("commit-graph chunk lookup table entry missing; file may be incomplete"));
+                       free(graph);
+                       return NULL;
+               }
+
+               chunk_id = get_be32(chunk_lookup + 0);
+               chunk_offset = get_be64(chunk_lookup + 4);
+
                chunk_lookup += GRAPH_CHUNKLOOKUP_WIDTH;
 
                if (chunk_offset > graph_size - the_hash_algo->rawsz) {
-                       error(_("improper chunk offset %08x%08x"), (uint32_t)(chunk_offset >> 32),
+                       error(_("commit-graph improper chunk offset %08x%08x"), (uint32_t)(chunk_offset >> 32),
                              (uint32_t)chunk_offset);
-                       goto cleanup_fail;
+                       free(graph);
+                       return NULL;
                }
 
                switch (chunk_id) {
@@ -176,17 +243,18 @@ struct commit_graph *load_commit_graph_one(const char *graph_file)
                                graph->chunk_commit_data = data + chunk_offset;
                        break;
 
-               case GRAPH_CHUNKID_LARGEEDGES:
-                       if (graph->chunk_large_edges)
+               case GRAPH_CHUNKID_EXTRAEDGES:
+                       if (graph->chunk_extra_edges)
                                chunk_repeated = 1;
                        else
-                               graph->chunk_large_edges = data + chunk_offset;
+                               graph->chunk_extra_edges = data + chunk_offset;
                        break;
                }
 
                if (chunk_repeated) {
-                       error(_("chunk id %08x appears multiple times"), chunk_id);
-                       goto cleanup_fail;
+                       error(_("commit-graph chunk id %08x appears multiple times"), chunk_id);
+                       free(graph);
+                       return NULL;
                }
 
                if (last_chunk_id == GRAPH_CHUNKID_OIDLOOKUP)
@@ -199,12 +267,25 @@ struct commit_graph *load_commit_graph_one(const char *graph_file)
                last_chunk_offset = chunk_offset;
        }
 
+       if (verify_commit_graph_lite(graph)) {
+               free(graph);
+               return NULL;
+       }
+
        return graph;
+}
 
-cleanup_fail:
-       munmap(graph_map, graph_size);
-       close(fd);
-       exit(1);
+static struct commit_graph *load_commit_graph_one(const char *graph_file)
+{
+
+       struct stat st;
+       int fd;
+       int open_ok = open_commit_graph(graph_file, &fd, &st);
+
+       if (!open_ok)
+               return NULL;
+
+       return load_commit_graph_one_fd_st(fd, &st);
 }
 
 static void prepare_commit_graph_one(struct repository *r, const char *obj_dir)
@@ -232,6 +313,10 @@ static int prepare_commit_graph(struct repository *r)
        struct object_directory *odb;
        int config_value;
 
+       if (git_env_bool(GIT_TEST_COMMIT_GRAPH_DIE_ON_LOAD, 0))
+               die("dying as requested by the '%s' variable on commit-graph load!",
+                   GIT_TEST_COMMIT_GRAPH_DIE_ON_LOAD);
+
        if (r->objects->commit_graph_attempted)
                return !!r->objects->commit_graph;
        r->objects->commit_graph_attempted = 1;
@@ -288,7 +373,8 @@ static int bsearch_graph(struct commit_graph *g, struct object_id *oid, uint32_t
                            g->chunk_oid_lookup, g->hash_len, pos);
 }
 
-static struct commit_list **insert_parent_or_die(struct commit_graph *g,
+static struct commit_list **insert_parent_or_die(struct repository *r,
+                                                struct commit_graph *g,
                                                 uint64_t pos,
                                                 struct commit_list **pptr)
 {
@@ -299,7 +385,7 @@ static struct commit_list **insert_parent_or_die(struct commit_graph *g,
                die("invalid parent position %"PRIu64, pos);
 
        hashcpy(oid.hash, g->chunk_oid_lookup + g->hash_len * pos);
-       c = lookup_commit(the_repository, &oid);
+       c = lookup_commit(r, &oid);
        if (!c)
                die(_("could not find commit %s"), oid_to_hex(&oid));
        c->graph_pos = pos;
@@ -313,7 +399,14 @@ static void fill_commit_graph_info(struct commit *item, struct commit_graph *g,
        item->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
 }
 
-static int fill_commit_in_graph(struct commit *item, struct commit_graph *g, uint32_t pos)
+static inline void set_commit_tree(struct commit *c, struct tree *t)
+{
+       c->maybe_tree = t;
+}
+
+static int fill_commit_in_graph(struct repository *r,
+                               struct commit *item,
+                               struct commit_graph *g, uint32_t pos)
 {
        uint32_t edge_value;
        uint32_t *parent_data_ptr;
@@ -324,7 +417,7 @@ static int fill_commit_in_graph(struct commit *item, struct commit_graph *g, uin
        item->object.parsed = 1;
        item->graph_pos = pos;
 
-       item->maybe_tree = NULL;
+       set_commit_tree(item, NULL);
 
        date_high = get_be32(commit_data + g->hash_len + 8) & 0x3;
        date_low = get_be32(commit_data + g->hash_len + 12);
@@ -337,21 +430,21 @@ static int fill_commit_in_graph(struct commit *item, struct commit_graph *g, uin
        edge_value = get_be32(commit_data + g->hash_len);
        if (edge_value == GRAPH_PARENT_NONE)
                return 1;
-       pptr = insert_parent_or_die(g, edge_value, pptr);
+       pptr = insert_parent_or_die(r, g, edge_value, pptr);
 
        edge_value = get_be32(commit_data + g->hash_len + 4);
        if (edge_value == GRAPH_PARENT_NONE)
                return 1;
-       if (!(edge_value & GRAPH_OCTOPUS_EDGES_NEEDED)) {
-               pptr = insert_parent_or_die(g, edge_value, pptr);
+       if (!(edge_value & GRAPH_EXTRA_EDGES_NEEDED)) {
+               pptr = insert_parent_or_die(r, g, edge_value, pptr);
                return 1;
        }
 
-       parent_data_ptr = (uint32_t*)(g->chunk_large_edges +
+       parent_data_ptr = (uint32_t*)(g->chunk_extra_edges +
                          4 * (uint64_t)(edge_value & GRAPH_EDGE_LAST_MASK));
        do {
                edge_value = get_be32(parent_data_ptr);
-               pptr = insert_parent_or_die(g,
+               pptr = insert_parent_or_die(r, g,
                                            edge_value & GRAPH_EDGE_LAST_MASK,
                                            pptr);
                parent_data_ptr++;
@@ -370,7 +463,9 @@ static int find_commit_in_graph(struct commit *item, struct commit_graph *g, uin
        }
 }
 
-static int parse_commit_in_graph_one(struct commit_graph *g, struct commit *item)
+static int parse_commit_in_graph_one(struct repository *r,
+                                    struct commit_graph *g,
+                                    struct commit *item)
 {
        uint32_t pos;
 
@@ -378,7 +473,7 @@ static int parse_commit_in_graph_one(struct commit_graph *g, struct commit *item
                return 1;
 
        if (find_commit_in_graph(item, g, &pos))
-               return fill_commit_in_graph(item, g, pos);
+               return fill_commit_in_graph(r, item, g, pos);
 
        return 0;
 }
@@ -387,7 +482,7 @@ int parse_commit_in_graph(struct repository *r, struct commit *item)
 {
        if (!prepare_commit_graph(r))
                return 0;
-       return parse_commit_in_graph_one(r->objects->commit_graph, item);
+       return parse_commit_in_graph_one(r, r->objects->commit_graph, item);
 }
 
 void load_commit_graph_info(struct repository *r, struct commit *item)
@@ -399,19 +494,22 @@ void load_commit_graph_info(struct repository *r, struct commit *item)
                fill_commit_graph_info(item, r->objects->commit_graph, pos);
 }
 
-static struct tree *load_tree_for_commit(struct commit_graph *g, struct commit *c)
+static struct tree *load_tree_for_commit(struct repository *r,
+                                        struct commit_graph *g,
+                                        struct commit *c)
 {
        struct object_id oid;
        const unsigned char *commit_data = g->chunk_commit_data +
                                           GRAPH_DATA_WIDTH * (c->graph_pos);
 
        hashcpy(oid.hash, commit_data);
-       c->maybe_tree = lookup_tree(the_repository, &oid);
+       set_commit_tree(c, lookup_tree(r, &oid));
 
        return c->maybe_tree;
 }
 
-static struct tree *get_commit_tree_in_graph_one(struct commit_graph *g,
+static struct tree *get_commit_tree_in_graph_one(struct repository *r,
+                                                struct commit_graph *g,
                                                 const struct commit *c)
 {
        if (c->maybe_tree)
@@ -419,17 +517,19 @@ static struct tree *get_commit_tree_in_graph_one(struct commit_graph *g,
        if (c->graph_pos == COMMIT_NOT_FROM_GRAPH)
                BUG("get_commit_tree_in_graph_one called from non-commit-graph commit");
 
-       return load_tree_for_commit(g, (struct commit *)c);
+       return load_tree_for_commit(r, g, (struct commit *)c);
 }
 
 struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit *c)
 {
-       return get_commit_tree_in_graph_one(r->objects->commit_graph, c);
+       return get_commit_tree_in_graph_one(r, r->objects->commit_graph, c);
 }
 
 static void write_graph_chunk_fanout(struct hashfile *f,
                                     struct commit **commits,
-                                    int nr_commits)
+                                    int nr_commits,
+                                    struct progress *progress,
+                                    uint64_t *progress_cnt)
 {
        int i, count = 0;
        struct commit **list = commits;
@@ -443,6 +543,7 @@ static void write_graph_chunk_fanout(struct hashfile *f,
                while (count < nr_commits) {
                        if ((*list)->object.oid.hash[0] != i)
                                break;
+                       display_progress(progress, ++*progress_cnt);
                        count++;
                        list++;
                }
@@ -452,12 +553,16 @@ static void write_graph_chunk_fanout(struct hashfile *f,
 }
 
 static void write_graph_chunk_oids(struct hashfile *f, int hash_len,
-                                  struct commit **commits, int nr_commits)
+                                  struct commit **commits, int nr_commits,
+                                  struct progress *progress,
+                                  uint64_t *progress_cnt)
 {
        struct commit **list = commits;
        int count;
-       for (count = 0; count < nr_commits; count++, list++)
+       for (count = 0; count < nr_commits; count++, list++) {
+               display_progress(progress, ++*progress_cnt);
                hashwrite(f, (*list)->object.oid.hash, (int)hash_len);
+       }
 }
 
 static const unsigned char *commit_to_sha1(size_t index, void *table)
@@ -467,7 +572,9 @@ static const unsigned char *commit_to_sha1(size_t index, void *table)
 }
 
 static void write_graph_chunk_data(struct hashfile *f, int hash_len,
-                                  struct commit **commits, int nr_commits)
+                                  struct commit **commits, int nr_commits,
+                                  struct progress *progress,
+                                  uint64_t *progress_cnt)
 {
        struct commit **list = commits;
        struct commit **last = commits + nr_commits;
@@ -477,8 +584,9 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
                struct commit_list *parent;
                int edge_value;
                uint32_t packedDate[2];
+               display_progress(progress, ++*progress_cnt);
 
-               parse_commit(*list);
+               parse_commit_no_graph(*list);
                hashwrite(f, get_commit_tree_oid(*list)->hash, hash_len);
 
                parent = (*list)->parents;
@@ -505,7 +613,7 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
                if (!parent)
                        edge_value = GRAPH_PARENT_NONE;
                else if (parent->next)
-                       edge_value = GRAPH_OCTOPUS_EDGES_NEEDED | num_extra_edges;
+                       edge_value = GRAPH_EXTRA_EDGES_NEEDED | num_extra_edges;
                else {
                        edge_value = sha1_pos(parent->item->object.oid.hash,
                                              commits,
@@ -519,7 +627,7 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
 
                hashwrite_be32(f, edge_value);
 
-               if (edge_value & GRAPH_OCTOPUS_EDGES_NEEDED) {
+               if (edge_value & GRAPH_EXTRA_EDGES_NEEDED) {
                        do {
                                num_extra_edges++;
                                parent = parent->next;
@@ -540,9 +648,11 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
        }
 }
 
-static void write_graph_chunk_large_edges(struct hashfile *f,
+static void write_graph_chunk_extra_edges(struct hashfile *f,
                                          struct commit **commits,
-                                         int nr_commits)
+                                         int nr_commits,
+                                         struct progress *progress,
+                                         uint64_t *progress_cnt)
 {
        struct commit **list = commits;
        struct commit **last = commits + nr_commits;
@@ -550,6 +660,9 @@ static void write_graph_chunk_large_edges(struct hashfile *f,
 
        while (list < last) {
                int num_parents = 0;
+
+               display_progress(progress, ++*progress_cnt);
+
                for (parent = (*list)->parents; num_parents < 3 && parent;
                     parent = parent->next)
                        num_parents++;
@@ -643,15 +756,15 @@ static void add_missing_parents(struct packed_oid_list *oids, struct commit *com
 
 static void close_reachable(struct packed_oid_list *oids, int report_progress)
 {
-       int i, j;
+       int i;
        struct commit *commit;
        struct progress *progress = NULL;
 
        if (report_progress)
                progress = start_delayed_progress(
-                       _("Loading known commits in commit graph"), j = 0);
+                       _("Loading known commits in commit graph"), oids->nr);
        for (i = 0; i < oids->nr; i++) {
-               display_progress(progress, ++j);
+               display_progress(progress, i + 1);
                commit = lookup_commit(the_repository, &oids->list[i]);
                if (commit)
                        commit->object.flags |= UNINTERESTING;
@@ -665,21 +778,21 @@ static void close_reachable(struct packed_oid_list *oids, int report_progress)
         */
        if (report_progress)
                progress = start_delayed_progress(
-                       _("Expanding reachable commits in commit graph"), j = 0);
+                       _("Expanding reachable commits in commit graph"), oids->nr);
        for (i = 0; i < oids->nr; i++) {
-               display_progress(progress, ++j);
+               display_progress(progress, i + 1);
                commit = lookup_commit(the_repository, &oids->list[i]);
 
-               if (commit && !parse_commit(commit))
+               if (commit && !parse_commit_no_graph(commit))
                        add_missing_parents(oids, commit);
        }
        stop_progress(&progress);
 
        if (report_progress)
                progress = start_delayed_progress(
-                       _("Clearing commit marks in commit graph"), j = 0);
+                       _("Clearing commit marks in commit graph"), oids->nr);
        for (i = 0; i < oids->nr; i++) {
-               display_progress(progress, ++j);
+               display_progress(progress, i + 1);
                commit = lookup_commit(the_repository, &oids->list[i]);
 
                if (commit)
@@ -774,12 +887,16 @@ void write_commit_graph(const char *obj_dir,
        struct commit_list *parent;
        struct progress *progress = NULL;
        const unsigned hashsz = the_hash_algo->rawsz;
+       uint64_t progress_cnt = 0;
+       struct strbuf progress_title = STRBUF_INIT;
+       unsigned long approx_nr_objects;
 
        if (!commit_graph_compatible(the_repository))
                return;
 
        oids.nr = 0;
-       oids.alloc = approximate_object_count() / 32;
+       approx_nr_objects = approximate_object_count();
+       oids.alloc = approx_nr_objects / 32;
        oids.progress = NULL;
        oids.progress_done = 0;
 
@@ -809,8 +926,12 @@ void write_commit_graph(const char *obj_dir,
                strbuf_addf(&packname, "%s/pack/", obj_dir);
                dirlen = packname.len;
                if (report_progress) {
-                       oids.progress = start_delayed_progress(
-                               _("Finding commits for commit graph"), 0);
+                       strbuf_addf(&progress_title,
+                                   Q_("Finding commits for commit graph in %d pack",
+                                      "Finding commits for commit graph in %d packs",
+                                      pack_indexes->nr),
+                                   pack_indexes->nr);
+                       oids.progress = start_delayed_progress(progress_title.buf, 0);
                        oids.progress_done = 0;
                }
                for (i = 0; i < pack_indexes->nr; i++) {
@@ -822,19 +943,26 @@ void write_commit_graph(const char *obj_dir,
                                die(_("error adding pack %s"), packname.buf);
                        if (open_pack_index(p))
                                die(_("error opening index for %s"), packname.buf);
-                       for_each_object_in_pack(p, add_packed_commits, &oids, 0);
+                       for_each_object_in_pack(p, add_packed_commits, &oids,
+                                               FOR_EACH_OBJECT_PACK_ORDER);
                        close_pack(p);
                        free(p);
                }
                stop_progress(&oids.progress);
+               strbuf_reset(&progress_title);
                strbuf_release(&packname);
        }
 
        if (commit_hex) {
-               if (report_progress)
-                       progress = start_delayed_progress(
-                               _("Finding commits for commit graph"),
-                               commit_hex->nr);
+               if (report_progress) {
+                       strbuf_addf(&progress_title,
+                                   Q_("Finding commits for commit graph from %d ref",
+                                      "Finding commits for commit graph from %d refs",
+                                      commit_hex->nr),
+                                   commit_hex->nr);
+                       progress = start_delayed_progress(progress_title.buf,
+                                                         commit_hex->nr);
+               }
                for (i = 0; i < commit_hex->nr; i++) {
                        const char *end;
                        struct object_id oid;
@@ -854,25 +982,36 @@ void write_commit_graph(const char *obj_dir,
                        }
                }
                stop_progress(&progress);
+               strbuf_reset(&progress_title);
        }
 
        if (!pack_indexes && !commit_hex) {
                if (report_progress)
                        oids.progress = start_delayed_progress(
-                               _("Finding commits for commit graph"), 0);
-               for_each_packed_object(add_packed_commits, &oids, 0);
+                               _("Finding commits for commit graph among packed objects"),
+                               approx_nr_objects);
+               for_each_packed_object(add_packed_commits, &oids,
+                                      FOR_EACH_OBJECT_PACK_ORDER);
+               if (oids.progress_done < approx_nr_objects)
+                       display_progress(oids.progress, approx_nr_objects);
                stop_progress(&oids.progress);
        }
 
        close_reachable(&oids, report_progress);
 
+       if (report_progress)
+               progress = start_delayed_progress(
+                       _("Counting distinct commits in commit graph"),
+                       oids.nr);
+       display_progress(progress, 0); /* TODO: Measure QSORT() progress */
        QSORT(oids.list, oids.nr, commit_compare);
-
        count_distinct = 1;
        for (i = 1; i < oids.nr; i++) {
+               display_progress(progress, i + 1);
                if (!oideq(&oids.list[i - 1], &oids.list[i]))
                        count_distinct++;
        }
+       stop_progress(&progress);
 
        if (count_distinct >= GRAPH_EDGE_LAST_MASK)
                die(_("the commit graph format cannot write %d commits"), count_distinct);
@@ -882,13 +1021,18 @@ void write_commit_graph(const char *obj_dir,
        ALLOC_ARRAY(commits.list, commits.alloc);
 
        num_extra_edges = 0;
+       if (report_progress)
+               progress = start_delayed_progress(
+                       _("Finding extra edges in commit graph"),
+                       oids.nr);
        for (i = 0; i < oids.nr; i++) {
                int num_parents = 0;
+               display_progress(progress, i + 1);
                if (i > 0 && oideq(&oids.list[i - 1], &oids.list[i]))
                        continue;
 
                commits.list[commits.nr] = lookup_commit(the_repository, &oids.list[i]);
-               parse_commit(commits.list[commits.nr]);
+               parse_commit_no_graph(commits.list[commits.nr]);
 
                for (parent = commits.list[commits.nr]->parents;
                     parent; parent = parent->next)
@@ -900,6 +1044,7 @@ void write_commit_graph(const char *obj_dir,
                commits.nr++;
        }
        num_chunks = num_extra_edges ? 4 : 3;
+       stop_progress(&progress);
 
        if (commits.nr >= GRAPH_EDGE_LAST_MASK)
                die(_("too many commits to write graph"));
@@ -927,7 +1072,7 @@ void write_commit_graph(const char *obj_dir,
        chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP;
        chunk_ids[2] = GRAPH_CHUNKID_DATA;
        if (num_extra_edges)
-               chunk_ids[3] = GRAPH_CHUNKID_LARGEEDGES;
+               chunk_ids[3] = GRAPH_CHUNKID_EXTRAEDGES;
        else
                chunk_ids[3] = 0;
        chunk_ids[4] = 0;
@@ -947,10 +1092,23 @@ void write_commit_graph(const char *obj_dir,
                hashwrite(f, chunk_write, 12);
        }
 
-       write_graph_chunk_fanout(f, commits.list, commits.nr);
-       write_graph_chunk_oids(f, hashsz, commits.list, commits.nr);
-       write_graph_chunk_data(f, hashsz, commits.list, commits.nr);
-       write_graph_chunk_large_edges(f, commits.list, commits.nr);
+       if (report_progress) {
+               strbuf_addf(&progress_title,
+                           Q_("Writing out commit graph in %d pass",
+                              "Writing out commit graph in %d passes",
+                              num_chunks),
+                           num_chunks);
+               progress = start_delayed_progress(
+                       progress_title.buf,
+                       num_chunks * commits.nr);
+       }
+       write_graph_chunk_fanout(f, commits.list, commits.nr, progress, &progress_cnt);
+       write_graph_chunk_oids(f, hashsz, commits.list, commits.nr, progress, &progress_cnt);
+       write_graph_chunk_data(f, hashsz, commits.list, commits.nr, progress, &progress_cnt);
+       if (num_extra_edges)
+               write_graph_chunk_extra_edges(f, commits.list, commits.nr, progress, &progress_cnt);
+       stop_progress(&progress);
+       strbuf_release(&progress_title);
 
        close_commit_graph(the_repository);
        finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC);
@@ -992,15 +1150,7 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g)
                return 1;
        }
 
-       verify_commit_graph_error = 0;
-
-       if (!g->chunk_oid_fanout)
-               graph_report("commit-graph is missing the OID Fanout chunk");
-       if (!g->chunk_oid_lookup)
-               graph_report("commit-graph is missing the OID Lookup chunk");
-       if (!g->chunk_commit_data)
-               graph_report("commit-graph is missing the Commit Data chunk");
-
+       verify_commit_graph_error = verify_commit_graph_lite(g);
        if (verify_commit_graph_error)
                return verify_commit_graph_error;
 
@@ -1019,7 +1169,7 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g)
                hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);
 
                if (i && oidcmp(&prev_oid, &cur_oid) >= 0)
-                       graph_report("commit-graph has incorrect OID order: %s then %s",
+                       graph_report(_("commit-graph has incorrect OID order: %s then %s"),
                                     oid_to_hex(&prev_oid),
                                     oid_to_hex(&cur_oid));
 
@@ -1029,14 +1179,14 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g)
                        uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
 
                        if (i != fanout_value)
-                               graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u",
+                               graph_report(_("commit-graph has incorrect fanout value: fanout[%d] = %u != %u"),
                                             cur_fanout_pos, fanout_value, i);
                        cur_fanout_pos++;
                }
 
                graph_commit = lookup_commit(r, &cur_oid);
-               if (!parse_commit_in_graph_one(g, graph_commit))
-                       graph_report("failed to parse %s from commit-graph",
+               if (!parse_commit_in_graph_one(r, g, graph_commit))
+                       graph_report(_("failed to parse commit %s from commit-graph"),
                                     oid_to_hex(&cur_oid));
        }
 
@@ -1044,7 +1194,7 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g)
                uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
 
                if (g->num_commits != fanout_value)
-                       graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u",
+                       graph_report(_("commit-graph has incorrect fanout value: fanout[%d] = %u != %u"),
                                     cur_fanout_pos, fanout_value, i);
 
                cur_fanout_pos++;
@@ -1066,14 +1216,14 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g)
                graph_commit = lookup_commit(r, &cur_oid);
                odb_commit = (struct commit *)create_object(r, cur_oid.hash, alloc_commit_node(r));
                if (parse_commit_internal(odb_commit, 0, 0)) {
-                       graph_report("failed to parse %s from object database",
+                       graph_report(_("failed to parse commit %s from object database for commit-graph"),
                                     oid_to_hex(&cur_oid));
                        continue;
                }
 
-               if (!oideq(&get_commit_tree_in_graph_one(g, graph_commit)->object.oid,
+               if (!oideq(&get_commit_tree_in_graph_one(r, g, graph_commit)->object.oid,
                           get_commit_tree_oid(odb_commit)))
-                       graph_report("root tree OID for commit %s in commit-graph is %s != %s",
+                       graph_report(_("root tree OID for commit %s in commit-graph is %s != %s"),
                                     oid_to_hex(&cur_oid),
                                     oid_to_hex(get_commit_tree_oid(graph_commit)),
                                     oid_to_hex(get_commit_tree_oid(odb_commit)));
@@ -1083,13 +1233,13 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g)
 
                while (graph_parents) {
                        if (odb_parents == NULL) {
-                               graph_report("commit-graph parent list for commit %s is too long",
+                               graph_report(_("commit-graph parent list for commit %s is too long"),
                                             oid_to_hex(&cur_oid));
                                break;
                        }
 
                        if (!oideq(&graph_parents->item->object.oid, &odb_parents->item->object.oid))
-                               graph_report("commit-graph parent for %s is %s != %s",
+                               graph_report(_("commit-graph parent for %s is %s != %s"),
                                             oid_to_hex(&cur_oid),
                                             oid_to_hex(&graph_parents->item->object.oid),
                                             oid_to_hex(&odb_parents->item->object.oid));
@@ -1102,16 +1252,16 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g)
                }
 
                if (odb_parents != NULL)
-                       graph_report("commit-graph parent list for commit %s terminates early",
+                       graph_report(_("commit-graph parent list for commit %s terminates early"),
                                     oid_to_hex(&cur_oid));
 
                if (!graph_commit->generation) {
                        if (generation_zero == GENERATION_NUMBER_EXISTS)
-                               graph_report("commit-graph has generation number zero for commit %s, but non-zero elsewhere",
+                               graph_report(_("commit-graph has generation number zero for commit %s, but non-zero elsewhere"),
                                             oid_to_hex(&cur_oid));
                        generation_zero = GENERATION_ZERO_EXISTS;
                } else if (generation_zero == GENERATION_ZERO_EXISTS)
-                       graph_report("commit-graph has non-zero generation number for commit %s, but zero elsewhere",
+                       graph_report(_("commit-graph has non-zero generation number for commit %s, but zero elsewhere"),
                                     oid_to_hex(&cur_oid));
 
                if (generation_zero == GENERATION_ZERO_EXISTS)
@@ -1126,13 +1276,13 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g)
                        max_generation--;
 
                if (graph_commit->generation != max_generation + 1)
-                       graph_report("commit-graph generation for commit %s is %u != %u",
+                       graph_report(_("commit-graph generation for commit %s is %u != %u"),
                                     oid_to_hex(&cur_oid),
                                     graph_commit->generation,
                                     max_generation + 1);
 
                if (graph_commit->date != odb_commit->date)
-                       graph_report("commit date for commit %s in commit-graph is %"PRItime" != %"PRItime,
+                       graph_report(_("commit date for commit %s in commit-graph is %"PRItime" != %"PRItime),
                                     oid_to_hex(&cur_oid),
                                     graph_commit->date,
                                     odb_commit->date);