Merge branch 'ds/commit-graph-fsck' into jt/commit-graph-per-object-store
authorJunio C Hamano <gitster@pobox.com>
Tue, 17 Jul 2018 22:46:19 +0000 (15:46 -0700)
committerJunio C Hamano <gitster@pobox.com>
Tue, 17 Jul 2018 22:46:19 +0000 (15:46 -0700)
* ds/commit-graph-fsck: (23 commits)
coccinelle: update commit.cocci
commit-graph: update design document
gc: automatically write commit-graph files
commit-graph: add '--reachable' option
commit-graph: use string-list API for input
fsck: verify commit-graph
commit-graph: verify contents match checksum
commit-graph: test for corrupted octopus edge
commit-graph: verify commit date
commit-graph: verify generation number
commit-graph: verify parent list
commit-graph: verify root tree OIDs
commit-graph: verify objects exist
commit-graph: verify corrupt OID fanout and lookup
commit-graph: verify required chunks are present
commit-graph: verify catches corrupt signature
commit-graph: add 'verify' subcommand
commit-graph: load a root tree from specific graph
commit: force commit to parse from object database
commit-graph: parse commit from chosen graph
...

1  2 
builtin/fsck.c
commit-graph.c
commit.c
commit.h
diff --combined builtin/fsck.c
index 263191942d4c6f785961fd0ed36cbac9b5748206,eca7900ee08841ec3efdaf24555a76e426fbbf28..ea5e2a03e6c882c674dbef9b3f6e99e93b552f64
@@@ -18,6 -18,7 +18,7 @@@
  #include "decorate.h"
  #include "packfile.h"
  #include "object-store.h"
+ #include "run-command.h"
  
  #define REACHABLE 0x0001
  #define SEEN      0x0002
@@@ -47,6 -48,7 +48,7 @@@ static int name_objects
  #define ERROR_REACHABLE 02
  #define ERROR_PACK 04
  #define ERROR_REFS 010
+ #define ERROR_COMMIT_GRAPH 020
  
  static const char *describe_object(struct object *obj)
  {
@@@ -70,7 -72,7 +72,7 @@@ static const char *printable_type(struc
                enum object_type type = oid_object_info(the_repository,
                                                        &obj->oid, NULL);
                if (type > 0)
 -                      object_as_type(obj, type, 0);
 +                      object_as_type(the_repository, obj, type, 0);
        }
  
        ret = type_name(obj->type);
@@@ -392,8 -394,7 +394,8 @@@ static int fsck_obj_buffer(const struc
         * verify_packfile(), data_valid variable for details.
         */
        struct object *obj;
 -      obj = parse_object_buffer(oid, type, size, buffer, eaten);
 +      obj = parse_object_buffer(the_repository, oid, type, size, buffer,
 +                                eaten);
        if (!obj) {
                errors_found |= ERROR_OBJECT;
                return error("%s: object corrupt or missing", oid_to_hex(oid));
@@@ -411,7 -412,7 +413,7 @@@ static void fsck_handle_reflog_oid(cons
        struct object *obj;
  
        if (!is_null_oid(oid)) {
 -              obj = lookup_object(oid->hash);
 +              obj = lookup_object(the_repository, oid->hash);
                if (obj && (obj->flags & HAS_OBJ)) {
                        if (timestamp && name_objects)
                                add_decoration(fsck_walk_options.object_names,
@@@ -453,7 -454,7 +455,7 @@@ static int fsck_handle_ref(const char *
  {
        struct object *obj;
  
 -      obj = parse_object(oid);
 +      obj = parse_object(the_repository, oid);
        if (!obj) {
                if (is_promisor_object(oid)) {
                        /*
@@@ -526,9 -527,7 +528,9 @@@ static int fsck_loose(const struct obje
        if (!contents && type != OBJ_BLOB)
                BUG("read_loose_object streamed a non-blob");
  
 -      obj = parse_object_buffer(oid, type, size, contents, &eaten);
 +      obj = parse_object_buffer(the_repository, oid, type, size,
 +                                contents, &eaten);
 +
        if (!obj) {
                errors_found |= ERROR_OBJECT;
                error("%s: object could not be parsed: %s",
@@@ -617,7 -616,7 +619,7 @@@ static int fsck_cache_tree(struct cache
                fprintf(stderr, "Checking cache tree\n");
  
        if (0 <= it->entry_count) {
 -              struct object *obj = parse_object(&it->oid);
 +              struct object *obj = parse_object(the_repository, &it->oid);
                if (!obj) {
                        error("%s: invalid sha1 pointer in cache-tree",
                              oid_to_hex(&it->oid));
@@@ -766,8 -765,7 +768,8 @@@ int cmd_fsck(int argc, const char **arg
                const char *arg = argv[i];
                struct object_id oid;
                if (!get_oid(arg, &oid)) {
 -                      struct object *obj = lookup_object(oid.hash);
 +                      struct object *obj = lookup_object(the_repository,
 +                                                         oid.hash);
  
                        if (!obj || !(obj->flags & HAS_OBJ)) {
                                if (is_promisor_object(&oid))
                        mode = active_cache[i]->ce_mode;
                        if (S_ISGITLINK(mode))
                                continue;
 -                      blob = lookup_blob(&active_cache[i]->oid);
 +                      blob = lookup_blob(the_repository,
 +                                         &active_cache[i]->oid);
                        if (!blob)
                                continue;
                        obj = &blob->object;
        }
  
        check_connectivity();
+       if (core_commit_graph) {
+               struct child_process commit_graph_verify = CHILD_PROCESS_INIT;
+               const char *verify_argv[] = { "commit-graph", "verify", NULL, NULL, NULL };
+               commit_graph_verify.argv = verify_argv;
+               commit_graph_verify.git_cmd = 1;
+               if (run_command(&commit_graph_verify))
+                       errors_found |= ERROR_COMMIT_GRAPH;
+               prepare_alt_odb(the_repository);
+               for (alt =  the_repository->objects->alt_odb_list; alt; alt = alt->next) {
+                       verify_argv[2] = "--object-dir";
+                       verify_argv[3] = alt->path;
+                       if (run_command(&commit_graph_verify))
+                               errors_found |= ERROR_COMMIT_GRAPH;
+               }
+       }
        return errors_found;
  }
diff --combined commit-graph.c
index 7f907b4bfb2ff722e93e13ec3d03c2c4452bc8c9,212232e752a812c7707d174fcf19db5f0eb02010..41a0133ff7506131d4966bd6bdee9cf6ea7614a2
@@@ -7,10 -7,12 +7,12 @@@
  #include "packfile.h"
  #include "commit.h"
  #include "object.h"
+ #include "refs.h"
  #include "revision.h"
  #include "sha1-lookup.h"
  #include "commit-graph.h"
  #include "object-store.h"
+ #include "alloc.h"
  
  #define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */
  #define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
  
  #define GRAPH_LAST_EDGE 0x80000000
  
+ #define GRAPH_HEADER_SIZE 8
  #define GRAPH_FANOUT_SIZE (4 * 256)
  #define GRAPH_CHUNKLOOKUP_WIDTH 12
- #define GRAPH_MIN_SIZE (5 * GRAPH_CHUNKLOOKUP_WIDTH + GRAPH_FANOUT_SIZE + \
-                       GRAPH_OID_LEN + 8)
+ #define GRAPH_MIN_SIZE (GRAPH_HEADER_SIZE + 4 * GRAPH_CHUNKLOOKUP_WIDTH \
+                       + GRAPH_FANOUT_SIZE + GRAPH_OID_LEN)
  
  char *get_commit_graph_filename(const char *obj_dir)
  {
@@@ -241,8 -244,12 +244,12 @@@ static struct commit_list **insert_pare
  {
        struct commit *c;
        struct object_id oid;
+       if (pos >= g->num_commits)
+               die("invalid parent position %"PRIu64, pos);
        hashcpy(oid.hash, g->chunk_oid_lookup + g->hash_len * pos);
 -      c = lookup_commit(&oid);
 +      c = lookup_commit(the_repository, &oid);
        if (!c)
                die("could not find commit %s", oid_to_hex(&oid));
        c->graph_pos = pos;
@@@ -313,7 -320,7 +320,7 @@@ static int find_commit_in_graph(struct 
        }
  }
  
int parse_commit_in_graph(struct commit *item)
static int parse_commit_in_graph_one(struct commit_graph *g, struct commit *item)
  {
        uint32_t pos;
  
                return 0;
        if (item->object.parsed)
                return 1;
+       if (find_commit_in_graph(item, g, &pos))
+               return fill_commit_in_graph(item, g, pos);
+       return 0;
+ }
+ int parse_commit_in_graph(struct commit *item)
+ {
+       if (!core_commit_graph)
+               return 0;
        prepare_commit_graph();
-       if (commit_graph && find_commit_in_graph(item, commit_graph, &pos))
-               return fill_commit_in_graph(item, commit_graph, pos);
+       if (commit_graph)
+               return parse_commit_in_graph_one(commit_graph, item);
        return 0;
  }
  
@@@ -344,19 -363,25 +363,25 @@@ static struct tree *load_tree_for_commi
                                           GRAPH_DATA_WIDTH * (c->graph_pos);
  
        hashcpy(oid.hash, commit_data);
 -      c->maybe_tree = lookup_tree(&oid);
 +      c->maybe_tree = lookup_tree(the_repository, &oid);
  
        return c->maybe_tree;
  }
  
- struct tree *get_commit_tree_in_graph(const struct commit *c)
+ static struct tree *get_commit_tree_in_graph_one(struct commit_graph *g,
+                                                const struct commit *c)
  {
        if (c->maybe_tree)
                return c->maybe_tree;
        if (c->graph_pos == COMMIT_NOT_FROM_GRAPH)
-               BUG("get_commit_tree_in_graph called from non-commit-graph commit");
+               BUG("get_commit_tree_in_graph_one called from non-commit-graph commit");
+       return load_tree_for_commit(g, (struct commit *)c);
+ }
  
-       return load_tree_for_commit(commit_graph, (struct commit *)c);
+ struct tree *get_commit_tree_in_graph(const struct commit *c)
+ {
+       return get_commit_tree_in_graph_one(commit_graph, c);
  }
  
  static void write_graph_chunk_fanout(struct hashfile *f,
@@@ -568,7 -593,7 +593,7 @@@ static void close_reachable(struct pack
        struct commit *commit;
  
        for (i = 0; i < oids->nr; i++) {
 -              commit = lookup_commit(&oids->list[i]);
 +              commit = lookup_commit(the_repository, &oids->list[i]);
                if (commit)
                        commit->object.flags |= UNINTERESTING;
        }
         * closure.
         */
        for (i = 0; i < oids->nr; i++) {
 -              commit = lookup_commit(&oids->list[i]);
 +              commit = lookup_commit(the_repository, &oids->list[i]);
  
                if (commit && !parse_commit(commit))
                        add_missing_parents(oids, commit);
        }
  
        for (i = 0; i < oids->nr; i++) {
 -              commit = lookup_commit(&oids->list[i]);
 +              commit = lookup_commit(the_repository, &oids->list[i]);
  
                if (commit)
                        commit->object.flags &= ~UNINTERESTING;
@@@ -632,11 -657,28 +657,28 @@@ static void compute_generation_numbers(
        }
  }
  
+ static int add_ref_to_list(const char *refname,
+                          const struct object_id *oid,
+                          int flags, void *cb_data)
+ {
+       struct string_list *list = (struct string_list *)cb_data;
+       string_list_append(list, oid_to_hex(oid));
+       return 0;
+ }
+ void write_commit_graph_reachable(const char *obj_dir, int append)
+ {
+       struct string_list list;
+       string_list_init(&list, 1);
+       for_each_ref(add_ref_to_list, &list);
+       write_commit_graph(obj_dir, NULL, &list, append);
+ }
  void write_commit_graph(const char *obj_dir,
-                       const char **pack_indexes,
-                       int nr_packs,
-                       const char **commit_hex,
-                       int nr_commits,
+                       struct string_list *pack_indexes,
+                       struct string_list *commit_hex,
                        int append)
  {
        struct packed_oid_list oids;
                int dirlen;
                strbuf_addf(&packname, "%s/pack/", obj_dir);
                dirlen = packname.len;
-               for (i = 0; i < nr_packs; i++) {
+               for (i = 0; i < pack_indexes->nr; i++) {
                        struct packed_git *p;
                        strbuf_setlen(&packname, dirlen);
-                       strbuf_addstr(&packname, pack_indexes[i]);
+                       strbuf_addstr(&packname, pack_indexes->items[i].string);
                        p = add_packed_git(packname.buf, packname.len, 1);
                        if (!p)
                                die("error adding pack %s", packname.buf);
        }
  
        if (commit_hex) {
-               for (i = 0; i < nr_commits; i++) {
+               for (i = 0; i < commit_hex->nr; i++) {
                        const char *end;
                        struct object_id oid;
                        struct commit *result;
  
-                       if (commit_hex[i] && parse_oid_hex(commit_hex[i], &oid, &end))
+                       if (commit_hex->items[i].string &&
+                           parse_oid_hex(commit_hex->items[i].string, &oid, &end))
                                continue;
  
 -                      result = lookup_commit_reference_gently(&oid, 1);
 +                      result = lookup_commit_reference_gently(the_repository, &oid, 1);
  
                        if (result) {
                                ALLOC_GROW(oids.list, oids.nr + 1, oids.alloc);
                if (i > 0 && !oidcmp(&oids.list[i-1], &oids.list[i]))
                        continue;
  
 -              commits.list[commits.nr] = lookup_commit(&oids.list[i]);
 +              commits.list[commits.nr] = lookup_commit(the_repository, &oids.list[i]);
                parse_commit(commits.list[commits.nr]);
  
                for (parent = commits.list[commits.nr]->parents;
        oids.alloc = 0;
        oids.nr = 0;
  }
 -              graph_commit = lookup_commit(&cur_oid);
+ #define VERIFY_COMMIT_GRAPH_ERROR_HASH 2
+ static int verify_commit_graph_error;
+ static void graph_report(const char *fmt, ...)
+ {
+       va_list ap;
+       verify_commit_graph_error = 1;
+       va_start(ap, fmt);
+       vfprintf(stderr, fmt, ap);
+       fprintf(stderr, "\n");
+       va_end(ap);
+ }
+ #define GENERATION_ZERO_EXISTS 1
+ #define GENERATION_NUMBER_EXISTS 2
+ int verify_commit_graph(struct repository *r, struct commit_graph *g)
+ {
+       uint32_t i, cur_fanout_pos = 0;
+       struct object_id prev_oid, cur_oid, checksum;
+       int generation_zero = 0;
+       struct hashfile *f;
+       int devnull;
+       if (!g) {
+               graph_report("no commit-graph file loaded");
+               return 1;
+       }
+       verify_commit_graph_error = 0;
+       if (!g->chunk_oid_fanout)
+               graph_report("commit-graph is missing the OID Fanout chunk");
+       if (!g->chunk_oid_lookup)
+               graph_report("commit-graph is missing the OID Lookup chunk");
+       if (!g->chunk_commit_data)
+               graph_report("commit-graph is missing the Commit Data chunk");
+       if (verify_commit_graph_error)
+               return verify_commit_graph_error;
+       devnull = open("/dev/null", O_WRONLY);
+       f = hashfd(devnull, NULL);
+       hashwrite(f, g->data, g->data_len - g->hash_len);
+       finalize_hashfile(f, checksum.hash, CSUM_CLOSE);
+       if (hashcmp(checksum.hash, g->data + g->data_len - g->hash_len)) {
+               graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt"));
+               verify_commit_graph_error = VERIFY_COMMIT_GRAPH_ERROR_HASH;
+       }
+       for (i = 0; i < g->num_commits; i++) {
+               struct commit *graph_commit;
+               hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);
+               if (i && oidcmp(&prev_oid, &cur_oid) >= 0)
+                       graph_report("commit-graph has incorrect OID order: %s then %s",
+                                    oid_to_hex(&prev_oid),
+                                    oid_to_hex(&cur_oid));
+               oidcpy(&prev_oid, &cur_oid);
+               while (cur_oid.hash[0] > cur_fanout_pos) {
+                       uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
+                       if (i != fanout_value)
+                               graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u",
+                                            cur_fanout_pos, fanout_value, i);
+                       cur_fanout_pos++;
+               }
 -              graph_commit = lookup_commit(&cur_oid);
++              graph_commit = lookup_commit(r, &cur_oid);
+               if (!parse_commit_in_graph_one(g, graph_commit))
+                       graph_report("failed to parse %s from commit-graph",
+                                    oid_to_hex(&cur_oid));
+       }
+       while (cur_fanout_pos < 256) {
+               uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
+               if (g->num_commits != fanout_value)
+                       graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u",
+                                    cur_fanout_pos, fanout_value, i);
+               cur_fanout_pos++;
+       }
+       if (verify_commit_graph_error & ~VERIFY_COMMIT_GRAPH_ERROR_HASH)
+               return verify_commit_graph_error;
+       for (i = 0; i < g->num_commits; i++) {
+               struct commit *graph_commit, *odb_commit;
+               struct commit_list *graph_parents, *odb_parents;
+               uint32_t max_generation = 0;
+               hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);
++              graph_commit = lookup_commit(r, &cur_oid);
+               odb_commit = (struct commit *)create_object(r, cur_oid.hash, alloc_commit_node(r));
+               if (parse_commit_internal(odb_commit, 0, 0)) {
+                       graph_report("failed to parse %s from object database",
+                                    oid_to_hex(&cur_oid));
+                       continue;
+               }
+               if (oidcmp(&get_commit_tree_in_graph_one(g, graph_commit)->object.oid,
+                          get_commit_tree_oid(odb_commit)))
+                       graph_report("root tree OID for commit %s in commit-graph is %s != %s",
+                                    oid_to_hex(&cur_oid),
+                                    oid_to_hex(get_commit_tree_oid(graph_commit)),
+                                    oid_to_hex(get_commit_tree_oid(odb_commit)));
+               graph_parents = graph_commit->parents;
+               odb_parents = odb_commit->parents;
+               while (graph_parents) {
+                       if (odb_parents == NULL) {
+                               graph_report("commit-graph parent list for commit %s is too long",
+                                            oid_to_hex(&cur_oid));
+                               break;
+                       }
+                       if (oidcmp(&graph_parents->item->object.oid, &odb_parents->item->object.oid))
+                               graph_report("commit-graph parent for %s is %s != %s",
+                                            oid_to_hex(&cur_oid),
+                                            oid_to_hex(&graph_parents->item->object.oid),
+                                            oid_to_hex(&odb_parents->item->object.oid));
+                       if (graph_parents->item->generation > max_generation)
+                               max_generation = graph_parents->item->generation;
+                       graph_parents = graph_parents->next;
+                       odb_parents = odb_parents->next;
+               }
+               if (odb_parents != NULL)
+                       graph_report("commit-graph parent list for commit %s terminates early",
+                                    oid_to_hex(&cur_oid));
+               if (!graph_commit->generation) {
+                       if (generation_zero == GENERATION_NUMBER_EXISTS)
+                               graph_report("commit-graph has generation number zero for commit %s, but non-zero elsewhere",
+                                            oid_to_hex(&cur_oid));
+                       generation_zero = GENERATION_ZERO_EXISTS;
+               } else if (generation_zero == GENERATION_ZERO_EXISTS)
+                       graph_report("commit-graph has non-zero generation number for commit %s, but zero elsewhere",
+                                    oid_to_hex(&cur_oid));
+               if (generation_zero == GENERATION_ZERO_EXISTS)
+                       continue;
+               /*
+                * If one of our parents has generation GENERATION_NUMBER_MAX, then
+                * our generation is also GENERATION_NUMBER_MAX. Decrement to avoid
+                * extra logic in the following condition.
+                */
+               if (max_generation == GENERATION_NUMBER_MAX)
+                       max_generation--;
+               if (graph_commit->generation != max_generation + 1)
+                       graph_report("commit-graph generation for commit %s is %u != %u",
+                                    oid_to_hex(&cur_oid),
+                                    graph_commit->generation,
+                                    max_generation + 1);
+               if (graph_commit->date != odb_commit->date)
+                       graph_report("commit date for commit %s in commit-graph is %"PRItime" != %"PRItime,
+                                    oid_to_hex(&cur_oid),
+                                    graph_commit->date,
+                                    odb_commit->date);
+       }
+       return verify_commit_graph_error;
+ }
diff --combined commit.c
index b88ced5b0265f705ad1dbd0c5b8dfb8eed6427a7,598cf21cee68349952c4afcf512dfdafd0769010..c0a83d2644c54ad8916900128ba13f200a2b6f97
+++ b/commit.c
@@@ -2,8 -2,6 +2,8 @@@
  #include "tag.h"
  #include "commit.h"
  #include "commit-graph.h"
 +#include "repository.h"
 +#include "object-store.h"
  #include "pkt-line.h"
  #include "utf8.h"
  #include "diff.h"
@@@ -24,26 -22,24 +24,26 @@@ int save_commit_buffer = 1
  
  const char *commit_type = "commit";
  
 -struct commit *lookup_commit_reference_gently(const struct object_id *oid,
 -                                            int quiet)
 +struct commit *lookup_commit_reference_gently(struct repository *r,
 +              const struct object_id *oid, int quiet)
  {
 -      struct object *obj = deref_tag(parse_object(oid), NULL, 0);
 +      struct object *obj = deref_tag(r,
 +                                     parse_object(r, oid),
 +                                     NULL, 0);
  
        if (!obj)
                return NULL;
 -      return object_as_type(obj, OBJ_COMMIT, quiet);
 +      return object_as_type(r, obj, OBJ_COMMIT, quiet);
  }
  
 -struct commit *lookup_commit_reference(const struct object_id *oid)
 +struct commit *lookup_commit_reference(struct repository *r, const struct object_id *oid)
  {
 -      return lookup_commit_reference_gently(oid, 0);
 +      return lookup_commit_reference_gently(r, oid, 0);
  }
  
  struct commit *lookup_commit_or_die(const struct object_id *oid, const char *ref_name)
  {
 -      struct commit *c = lookup_commit_reference(oid);
 +      struct commit *c = lookup_commit_reference(the_repository, oid);
        if (!c)
                die(_("could not parse %s"), ref_name);
        if (oidcmp(oid, &c->object.oid)) {
        return c;
  }
  
 -struct commit *lookup_commit(const struct object_id *oid)
 +struct commit *lookup_commit(struct repository *r, const struct object_id *oid)
  {
 -      struct object *obj = lookup_object(oid->hash);
 +      struct object *obj = lookup_object(r, oid->hash);
        if (!obj)
 -              return create_object(the_repository, oid->hash,
 -                                   alloc_commit_node(the_repository));
 -      return object_as_type(obj, OBJ_COMMIT, 0);
 +              return create_object(r, oid->hash,
 +                                   alloc_commit_node(r));
 +      return object_as_type(r, obj, OBJ_COMMIT, 0);
  }
  
  struct commit *lookup_commit_reference_by_name(const char *name)
@@@ -69,7 -65,7 +69,7 @@@
  
        if (get_oid_committish(name, &oid))
                return NULL;
 -      commit = lookup_commit_reference(&oid);
 +      commit = lookup_commit_reference(the_repository, &oid);
        if (parse_commit(commit))
                return NULL;
        return commit;
@@@ -102,44 -98,41 +102,44 @@@ static timestamp_t parse_commit_date(co
        return parse_timestamp(dateptr, NULL, 10);
  }
  
 -static struct commit_graft **commit_graft;
 -static int commit_graft_alloc, commit_graft_nr;
 -
  static const unsigned char *commit_graft_sha1_access(size_t index, void *table)
  {
        struct commit_graft **commit_graft_table = table;
        return commit_graft_table[index]->oid.hash;
  }
  
 -static int commit_graft_pos(const unsigned char *sha1)
 +static int commit_graft_pos(struct repository *r, const unsigned char *sha1)
  {
 -      return sha1_pos(sha1, commit_graft, commit_graft_nr,
 +      return sha1_pos(sha1, r->parsed_objects->grafts,
 +                      r->parsed_objects->grafts_nr,
                        commit_graft_sha1_access);
  }
  
 -int register_commit_graft(struct commit_graft *graft, int ignore_dups)
 +int register_commit_graft(struct repository *r, struct commit_graft *graft,
 +                        int ignore_dups)
  {
 -      int pos = commit_graft_pos(graft->oid.hash);
 +      int pos = commit_graft_pos(r, graft->oid.hash);
  
        if (0 <= pos) {
                if (ignore_dups)
                        free(graft);
                else {
 -                      free(commit_graft[pos]);
 -                      commit_graft[pos] = graft;
 +                      free(r->parsed_objects->grafts[pos]);
 +                      r->parsed_objects->grafts[pos] = graft;
                }
                return 1;
        }
        pos = -pos - 1;
 -      ALLOC_GROW(commit_graft, commit_graft_nr + 1, commit_graft_alloc);
 -      commit_graft_nr++;
 -      if (pos < commit_graft_nr)
 -              MOVE_ARRAY(commit_graft + pos + 1, commit_graft + pos,
 -                         commit_graft_nr - pos - 1);
 -      commit_graft[pos] = graft;
 +      ALLOC_GROW(r->parsed_objects->grafts,
 +                 r->parsed_objects->grafts_nr + 1,
 +                 r->parsed_objects->grafts_alloc);
 +      r->parsed_objects->grafts_nr++;
 +      if (pos < r->parsed_objects->grafts_nr)
 +              memmove(r->parsed_objects->grafts + pos + 1,
 +                      r->parsed_objects->grafts + pos,
 +                      (r->parsed_objects->grafts_nr - pos - 1) *
 +                      sizeof(*r->parsed_objects->grafts));
 +      r->parsed_objects->grafts[pos] = graft;
        return 0;
  }
  
@@@ -181,7 -174,7 +181,7 @@@ bad_graft_data
        return NULL;
  }
  
 -static int read_graft_file(const char *graft_file)
 +static int read_graft_file(struct repository *r, const char *graft_file)
  {
        FILE *fp = fopen_or_warn(graft_file, "r");
        struct strbuf buf = STRBUF_INIT;
                struct commit_graft *graft = read_graft_line(&buf);
                if (!graft)
                        continue;
 -              if (register_commit_graft(graft, 1))
 +              if (register_commit_graft(r, graft, 1))
                        error("duplicate graft data: %s", buf.buf);
        }
        fclose(fp);
        return 0;
  }
  
 -static void prepare_commit_graft(void)
 +static void prepare_commit_graft(struct repository *r)
  {
 -      static int commit_graft_prepared;
        char *graft_file;
  
 -      if (commit_graft_prepared)
 +      if (r->parsed_objects->commit_graft_prepared)
                return;
        if (!startup_info->have_repository)
                return;
  
 -      graft_file = get_graft_file();
 -      read_graft_file(graft_file);
 +      graft_file = get_graft_file(r);
 +      read_graft_file(r, graft_file);
        /* make sure shallows are read */
 -      is_repository_shallow();
 -      commit_graft_prepared = 1;
 +      is_repository_shallow(r);
 +      r->parsed_objects->commit_graft_prepared = 1;
  }
  
 -struct commit_graft *lookup_commit_graft(const struct object_id *oid)
 +struct commit_graft *lookup_commit_graft(struct repository *r, const struct object_id *oid)
  {
        int pos;
 -      prepare_commit_graft();
 -      pos = commit_graft_pos(oid->hash);
 +      prepare_commit_graft(r);
 +      pos = commit_graft_pos(r, oid->hash);
        if (pos < 0)
                return NULL;
 -      return commit_graft[pos];
 +      return r->parsed_objects->grafts[pos];
  }
  
  int for_each_commit_graft(each_commit_graft_fn fn, void *cb_data)
  {
        int i, ret;
 -      for (i = ret = 0; i < commit_graft_nr && !ret; i++)
 -              ret = fn(commit_graft[i], cb_data);
 +      for (i = ret = 0; i < the_repository->parsed_objects->grafts_nr && !ret; i++)
 +              ret = fn(the_repository->parsed_objects->grafts[i], cb_data);
        return ret;
  }
  
  int unregister_shallow(const struct object_id *oid)
  {
 -      int pos = commit_graft_pos(oid->hash);
 +      int pos = commit_graft_pos(the_repository, oid->hash);
        if (pos < 0)
                return -1;
 -      if (pos + 1 < commit_graft_nr)
 -              MOVE_ARRAY(commit_graft + pos, commit_graft + pos + 1,
 -                         commit_graft_nr - pos - 1);
 -      commit_graft_nr--;
 +      if (pos + 1 < the_repository->parsed_objects->grafts_nr)
 +              MOVE_ARRAY(the_repository->parsed_objects->grafts + pos,
 +                         the_repository->parsed_objects->grafts + pos + 1,
 +                         the_repository->parsed_objects->grafts_nr - pos - 1);
 +      the_repository->parsed_objects->grafts_nr--;
        return 0;
  }
  
@@@ -261,32 -254,18 +261,32 @@@ struct commit_buffer 
        unsigned long size;
  };
  define_commit_slab(buffer_slab, struct commit_buffer);
 -static struct buffer_slab buffer_slab = COMMIT_SLAB_INIT(1, buffer_slab);
  
 -void set_commit_buffer(struct commit *commit, void *buffer, unsigned long size)
 +struct buffer_slab *allocate_commit_buffer_slab(void)
 +{
 +      struct buffer_slab *bs = xmalloc(sizeof(*bs));
 +      init_buffer_slab(bs);
 +      return bs;
 +}
 +
 +void free_commit_buffer_slab(struct buffer_slab *bs)
 +{
 +      clear_buffer_slab(bs);
 +      free(bs);
 +}
 +
 +void set_commit_buffer(struct repository *r, struct commit *commit, void *buffer, unsigned long size)
  {
 -      struct commit_buffer *v = buffer_slab_at(&buffer_slab, commit);
 +      struct commit_buffer *v = buffer_slab_at(
 +              r->parsed_objects->buffer_slab, commit);
        v->buffer = buffer;
        v->size = size;
  }
  
 -const void *get_cached_commit_buffer(const struct commit *commit, unsigned long *sizep)
 +const void *get_cached_commit_buffer(struct repository *r, const struct commit *commit, unsigned long *sizep)
  {
 -      struct commit_buffer *v = buffer_slab_peek(&buffer_slab, commit);
 +      struct commit_buffer *v = buffer_slab_peek(
 +              r->parsed_objects->buffer_slab, commit);
        if (!v) {
                if (sizep)
                        *sizep = 0;
  
  const void *get_commit_buffer(const struct commit *commit, unsigned long *sizep)
  {
 -      const void *ret = get_cached_commit_buffer(commit, sizep);
 +      const void *ret = get_cached_commit_buffer(the_repository, commit, sizep);
        if (!ret) {
                enum object_type type;
                unsigned long size;
  
  void unuse_commit_buffer(const struct commit *commit, const void *buffer)
  {
 -      struct commit_buffer *v = buffer_slab_peek(&buffer_slab, commit);
 +      struct commit_buffer *v = buffer_slab_peek(
 +              the_repository->parsed_objects->buffer_slab, commit);
        if (!(v && v->buffer == buffer))
                free((void *)buffer);
  }
  
  void free_commit_buffer(struct commit *commit)
  {
 -      struct commit_buffer *v = buffer_slab_peek(&buffer_slab, commit);
 +      struct commit_buffer *v = buffer_slab_peek(
 +              the_repository->parsed_objects->buffer_slab, commit);
        if (v) {
                FREE_AND_NULL(v->buffer);
                v->size = 0;
@@@ -363,8 -340,7 +363,8 @@@ void release_commit_memory(struct commi
  
  const void *detach_commit_buffer(struct commit *commit, unsigned long *sizep)
  {
 -      struct commit_buffer *v = buffer_slab_peek(&buffer_slab, commit);
 +      struct commit_buffer *v = buffer_slab_peek(
 +              the_repository->parsed_objects->buffer_slab, commit);
        void *ret;
  
        if (!v) {
        return ret;
  }
  
 -int parse_commit_buffer(struct commit *item, const void *buffer, unsigned long size, int check_graph)
 +int parse_commit_buffer(struct repository *r, struct commit *item, const void *buffer, unsigned long size, int check_graph)
  {
        const char *tail = buffer;
        const char *bufptr = buffer;
        if (get_oid_hex(bufptr + 5, &parent) < 0)
                return error("bad tree pointer in commit %s",
                             oid_to_hex(&item->object.oid));
 -      item->maybe_tree = lookup_tree(&parent);
 +      item->maybe_tree = lookup_tree(r, &parent);
        bufptr += tree_entry_len + 1; /* "tree " + "hex sha1" + "\n" */
        pptr = &item->parents;
  
 -      graft = lookup_commit_graft(&item->object.oid);
 +      graft = lookup_commit_graft(r, &item->object.oid);
        while (bufptr + parent_entry_len < tail && !memcmp(bufptr, "parent ", 7)) {
                struct commit *new_parent;
  
                 */
                if (graft && (graft->nr_parent < 0 || grafts_replace_parents))
                        continue;
 -              new_parent = lookup_commit(&parent);
 +              new_parent = lookup_commit(r, &parent);
                if (new_parent)
                        pptr = &commit_list_insert(new_parent, pptr)->next;
        }
                int i;
                struct commit *new_parent;
                for (i = 0; i < graft->nr_parent; i++) {
 -                      new_parent = lookup_commit(&graft->parent[i]);
 +                      new_parent = lookup_commit(r,
 +                                                 &graft->parent[i]);
                        if (!new_parent)
                                continue;
                        pptr = &commit_list_insert(new_parent, pptr)->next;
        return 0;
  }
  
- int parse_commit_gently(struct commit *item, int quiet_on_missing)
+ int parse_commit_internal(struct commit *item, int quiet_on_missing, int use_commit_graph)
  {
        enum object_type type;
        void *buffer;
                return -1;
        if (item->object.parsed)
                return 0;
-       if (parse_commit_in_graph(item))
+       if (use_commit_graph && parse_commit_in_graph(item))
                return 0;
        buffer = read_object_file(&item->object.oid, &type, &size);
        if (!buffer)
                return error("Object %s not a commit",
                             oid_to_hex(&item->object.oid));
        }
 -      ret = parse_commit_buffer(item, buffer, size, 0);
 +      ret = parse_commit_buffer(the_repository, item, buffer, size, 0);
        if (save_commit_buffer && !ret) {
 -              set_commit_buffer(item, buffer, size);
 +              set_commit_buffer(the_repository, item, buffer, size);
                return 0;
        }
        free(buffer);
        return ret;
  }
  
+ int parse_commit_gently(struct commit *item, int quiet_on_missing)
+ {
+       return parse_commit_internal(item, quiet_on_missing, 1);
+ }
  void parse_commit_or_die(struct commit *item)
  {
        if (parse_commit(item))
@@@ -1712,7 -1693,7 +1718,7 @@@ struct commit *get_merge_parent(const c
        struct object_id oid;
        if (get_oid(name, &oid))
                return NULL;
 -      obj = parse_object(&oid);
 +      obj = parse_object(the_repository, &oid);
        commit = (struct commit *)peel_to_type(name, 0, obj, OBJ_COMMIT);
        if (commit && !merge_remote_util(commit))
                set_merge_remote_desc(commit, name, obj);
diff --combined commit.h
index 8b2cf9692dec335c75074736038b4ebfe0ac822e,7e0f273720f2d54dcdbb40330124200bda63b98f..da0db36eba2bf16277dbb7aadcb3384e29191ecc
+++ b/commit.h
@@@ -63,11 -63,9 +63,11 @@@ enum decoration_type 
  void add_name_decoration(enum decoration_type type, const char *name, struct object *obj);
  const struct name_decoration *get_name_decoration(const struct object *obj);
  
 -struct commit *lookup_commit(const struct object_id *oid);
 -struct commit *lookup_commit_reference(const struct object_id *oid);
 -struct commit *lookup_commit_reference_gently(const struct object_id *oid,
 +struct commit *lookup_commit(struct repository *r, const struct object_id *oid);
 +struct commit *lookup_commit_reference(struct repository *r,
 +                                     const struct object_id *oid);
 +struct commit *lookup_commit_reference_gently(struct repository *r,
 +                                            const struct object_id *oid,
                                              int quiet);
  struct commit *lookup_commit_reference_by_name(const char *name);
  
@@@ -78,7 -76,8 +78,8 @@@
   */
  struct commit *lookup_commit_or_die(const struct object_id *oid, const char *ref_name);
  
 -int parse_commit_buffer(struct commit *item, const void *buffer, unsigned long size, int check_graph);
 +int parse_commit_buffer(struct repository *r, struct commit *item, const void *buffer, unsigned long size, int check_graph);
+ int parse_commit_internal(struct commit *item, int quiet_on_missing, int use_commit_graph);
  int parse_commit_gently(struct commit *item, int quiet_on_missing);
  static inline int parse_commit(struct commit *item)
  {
  }
  void parse_commit_or_die(struct commit *item);
  
 +struct buffer_slab;
 +struct buffer_slab *allocate_commit_buffer_slab(void);
 +void free_commit_buffer_slab(struct buffer_slab *bs);
 +
  /*
   * Associate an object buffer with the commit. The ownership of the
   * memory is handed over to the commit, and must be free()-able.
   */
 -void set_commit_buffer(struct commit *, void *buffer, unsigned long size);
 +void set_commit_buffer(struct repository *r, struct commit *, void *buffer, unsigned long size);
  
  /*
   * Get any cached object buffer associated with the commit. Returns NULL
   * if none. The resulting memory should not be freed.
   */
 -const void *get_cached_commit_buffer(const struct commit *, unsigned long *size);
 +const void *get_cached_commit_buffer(struct repository *, const struct commit *, unsigned long *size);
  
  /*
   * Get the commit's object contents, either from cache or by reading the object
@@@ -200,8 -195,8 +201,8 @@@ struct commit_graft 
  typedef int (*each_commit_graft_fn)(const struct commit_graft *, void *);
  
  struct commit_graft *read_graft_line(struct strbuf *line);
 -int register_commit_graft(struct commit_graft *, int);
 -struct commit_graft *lookup_commit_graft(const struct object_id *oid);
 +int register_commit_graft(struct repository *r, struct commit_graft *, int);
 +struct commit_graft *lookup_commit_graft(struct repository *r, const struct object_id *oid);
  
  extern struct commit_list *get_merge_bases(struct commit *rev1, struct commit *rev2);
  extern struct commit_list *get_merge_bases_many(struct commit *one, int n, struct commit **twos);
@@@ -215,15 -210,15 +216,15 @@@ extern struct commit_list *get_merge_ba
  
  struct oid_array;
  struct ref;
 -extern int register_shallow(const struct object_id *oid);
 +extern int register_shallow(struct repository *r, const struct object_id *oid);
  extern int unregister_shallow(const struct object_id *oid);
  extern int for_each_commit_graft(each_commit_graft_fn, void *);
 -extern int is_repository_shallow(void);
 +extern int is_repository_shallow(struct repository *r);
  extern struct commit_list *get_shallow_commits(struct object_array *heads,
                int depth, int shallow_flag, int not_shallow_flag);
  extern struct commit_list *get_shallow_commits_by_rev_list(
                int ac, const char **av, int shallow_flag, int not_shallow_flag);
 -extern void set_alternate_shallow_file(const char *path, int override);
 +extern void set_alternate_shallow_file(struct repository *r, const char *path, int override);
  extern int write_shallow_commits(struct strbuf *out, int use_pack_protocol,
                                 const struct oid_array *extra);
  extern void setup_alternate_shallow(struct lock_file *shallow_lock,