Merge branch 'pw/rebase-signoff'
[gitweb.git] / packfile.c
index 4a5fe7ab1883843a389ce74bf1c7bd89890d8e51..0bc67d0e00966008f9f8a6fa1c1b6540569a7cc0 100644 (file)
@@ -1,6 +1,7 @@
 #include "cache.h"
-#include "mru.h"
+#include "list.h"
 #include "pack.h"
+#include "repository.h"
 #include "dir.h"
 #include "mergesort.h"
 #include "packfile.h"
@@ -8,6 +9,12 @@
 #include "list.h"
 #include "streaming.h"
 #include "sha1-lookup.h"
+#include "commit.h"
+#include "object.h"
+#include "tag.h"
+#include "tree-walk.h"
+#include "tree.h"
+#include "object-store.h"
 
 char *odb_pack_name(struct strbuf *buf,
                    const unsigned char *sha1,
@@ -39,8 +46,6 @@ static unsigned int pack_open_fds;
 static unsigned int pack_max_fds;
 static size_t peak_pack_mapped;
 static size_t pack_mapped;
-struct packed_git *packed_git;
-struct mru packed_git_mru;
 
 #define SZ_FMT PRIuMAX
 static inline uintmax_t sz_fmt(size_t s) { return s; }
@@ -240,7 +245,7 @@ static int unuse_one_window(struct packed_git *current)
 
        if (current)
                scan_windows(current, &lru_p, &lru_w, &lru_l);
-       for (p = packed_git; p; p = p->next)
+       for (p = the_repository->objects->packed_git; p; p = p->next)
                scan_windows(p, &lru_p, &lru_w, &lru_l);
        if (lru_p) {
                munmap(lru_w->base, lru_w->len);
@@ -306,11 +311,11 @@ static void close_pack(struct packed_git *p)
        close_pack_index(p);
 }
 
-void close_all_packs(void)
+void close_all_packs(struct raw_object_store *o)
 {
        struct packed_git *p;
 
-       for (p = packed_git; p; p = p->next)
+       for (p = o->packed_git; p; p = p->next)
                if (p->do_not_close)
                        die("BUG: want to close pack marked 'do-not-close'");
                else
@@ -378,7 +383,7 @@ static int close_one_pack(void)
        struct pack_window *mru_w = NULL;
        int accept_windows_inuse = 1;
 
-       for (p = packed_git; p; p = p->next) {
+       for (p = the_repository->objects->packed_git; p; p = p->next) {
                if (p->pack_fd == -1)
                        continue;
                find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse);
@@ -643,10 +648,10 @@ struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
                return NULL;
 
        /*
-        * ".pack" is long enough to hold any suffix we're adding (and
+        * ".promisor" is long enough to hold any suffix we're adding (and
         * the use xsnprintf double-checks that)
         */
-       alloc = st_add3(path_len, strlen(".pack"), 1);
+       alloc = st_add3(path_len, strlen(".promisor"), 1);
        p = alloc_packed_git(alloc);
        memcpy(p->pack_name, path, path_len);
 
@@ -654,6 +659,10 @@ struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
        if (!access(p->pack_name, F_OK))
                p->pack_keep = 1;
 
+       xsnprintf(p->pack_name + path_len, alloc - path_len, ".promisor");
+       if (!access(p->pack_name, F_OK))
+               p->pack_promisor = 1;
+
        xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack");
        if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
                free(p);
@@ -671,13 +680,13 @@ struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
        return p;
 }
 
-void install_packed_git(struct packed_git *pack)
+void install_packed_git(struct repository *r, struct packed_git *pack)
 {
        if (pack->pack_fd != -1)
                pack_open_fds++;
 
-       pack->next = packed_git;
-       packed_git = pack;
+       pack->next = r->objects->packed_git;
+       r->objects->packed_git = pack;
 }
 
 void (*report_garbage)(unsigned seen_bits, const char *path);
@@ -726,7 +735,7 @@ static void report_pack_garbage(struct string_list *list)
        report_helper(list, seen_bits, first, list->nr);
 }
 
-static void prepare_packed_git_one(char *objdir, int local)
+static void prepare_packed_git_one(struct repository *r, char *objdir, int local)
 {
        struct strbuf path = STRBUF_INIT;
        size_t dirnamelen;
@@ -759,7 +768,8 @@ static void prepare_packed_git_one(char *objdir, int local)
                base_len = path.len;
                if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
                        /* Don't reopen a pack we already have. */
-                       for (p = packed_git; p; p = p->next) {
+                       for (p = r->objects->packed_git; p;
+                            p = p->next) {
                                size_t len;
                                if (strip_suffix(p->pack_name, ".pack", &len) &&
                                    len == base_len &&
@@ -772,7 +782,7 @@ static void prepare_packed_git_one(char *objdir, int local)
                             * corresponding .pack file that we can map.
                             */
                            (p = add_packed_git(path.buf, path.len, local)) != NULL)
-                               install_packed_git(p);
+                               install_packed_git(r, p);
                }
 
                if (!report_garbage)
@@ -781,7 +791,8 @@ static void prepare_packed_git_one(char *objdir, int local)
                if (ends_with(de->d_name, ".idx") ||
                    ends_with(de->d_name, ".pack") ||
                    ends_with(de->d_name, ".bitmap") ||
-                   ends_with(de->d_name, ".keep"))
+                   ends_with(de->d_name, ".keep") ||
+                   ends_with(de->d_name, ".promisor"))
                        string_list_append(&garbage, path.buf);
                else
                        report_garbage(PACKDIR_FILE_GARBAGE, path.buf);
@@ -792,8 +803,7 @@ static void prepare_packed_git_one(char *objdir, int local)
        strbuf_release(&path);
 }
 
-static int approximate_object_count_valid;
-
+static void prepare_packed_git(struct repository *r);
 /*
  * Give a fast, rough count of the number of objects in the repository. This
  * ignores loose objects completely. If you have a lot of them, then either
@@ -803,19 +813,20 @@ static int approximate_object_count_valid;
  */
 unsigned long approximate_object_count(void)
 {
-       static unsigned long count;
-       if (!approximate_object_count_valid) {
+       if (!the_repository->objects->approximate_object_count_valid) {
+               unsigned long count;
                struct packed_git *p;
 
-               prepare_packed_git();
+               prepare_packed_git(the_repository);
                count = 0;
-               for (p = packed_git; p; p = p->next) {
+               for (p = the_repository->objects->packed_git; p; p = p->next) {
                        if (open_pack_index(p))
                                continue;
                        count += p->num_objects;
                }
+               the_repository->objects->approximate_object_count = count;
        }
-       return count;
+       return the_repository->objects->approximate_object_count;
 }
 
 static void *get_next_packed_git(const void *p)
@@ -856,42 +867,55 @@ static int sort_pack(const void *a_, const void *b_)
        return -1;
 }
 
-static void rearrange_packed_git(void)
+static void rearrange_packed_git(struct repository *r)
 {
-       packed_git = llist_mergesort(packed_git, get_next_packed_git,
-                                    set_next_packed_git, sort_pack);
+       r->objects->packed_git = llist_mergesort(
+               r->objects->packed_git, get_next_packed_git,
+               set_next_packed_git, sort_pack);
 }
 
-static void prepare_packed_git_mru(void)
+static void prepare_packed_git_mru(struct repository *r)
 {
        struct packed_git *p;
 
-       mru_clear(&packed_git_mru);
-       for (p = packed_git; p; p = p->next)
-               mru_append(&packed_git_mru, p);
+       INIT_LIST_HEAD(&r->objects->packed_git_mru);
+
+       for (p = r->objects->packed_git; p; p = p->next)
+               list_add_tail(&p->mru, &r->objects->packed_git_mru);
 }
 
-static int prepare_packed_git_run_once = 0;
-void prepare_packed_git(void)
+static void prepare_packed_git(struct repository *r)
 {
        struct alternate_object_database *alt;
 
-       if (prepare_packed_git_run_once)
+       if (r->objects->packed_git_initialized)
                return;
-       prepare_packed_git_one(get_object_directory(), 1);
-       prepare_alt_odb();
-       for (alt = alt_odb_list; alt; alt = alt->next)
-               prepare_packed_git_one(alt->path, 0);
-       rearrange_packed_git();
-       prepare_packed_git_mru();
-       prepare_packed_git_run_once = 1;
+       prepare_packed_git_one(r, r->objects->objectdir, 1);
+       prepare_alt_odb(r);
+       for (alt = r->objects->alt_odb_list; alt; alt = alt->next)
+               prepare_packed_git_one(r, alt->path, 0);
+       rearrange_packed_git(r);
+       prepare_packed_git_mru(r);
+       r->objects->packed_git_initialized = 1;
 }
 
-void reprepare_packed_git(void)
+void reprepare_packed_git(struct repository *r)
 {
-       approximate_object_count_valid = 0;
-       prepare_packed_git_run_once = 0;
-       prepare_packed_git();
+       r->objects->approximate_object_count_valid = 0;
+       r->objects->packed_git_initialized = 0;
+       prepare_packed_git(r);
+}
+
+struct packed_git *get_packed_git(struct repository *r)
+{
+       prepare_packed_git(r);
+       return r->objects->packed_git;
+}
+
+struct list_head *get_packed_git_mru(struct repository *r)
+{
+       prepare_packed_git(r);
+       return &r->objects->packed_git_mru;
 }
 
 unsigned long unpack_object_header_buffer(const unsigned char *buf,
@@ -1002,7 +1026,7 @@ const struct packed_git *has_packed_and_bad(const unsigned char *sha1)
        struct packed_git *p;
        unsigned i;
 
-       for (p = packed_git; p; p = p->next)
+       for (p = the_repository->objects->packed_git; p; p = p->next)
                for (i = 0; i < p->num_bad_objects; i++)
                        if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
                                return p;
@@ -1084,13 +1108,13 @@ static int retry_bad_packed_offset(struct packed_git *p, off_t obj_offset)
 {
        int type;
        struct revindex_entry *revidx;
-       const unsigned char *sha1;
+       struct object_id oid;
        revidx = find_pack_revindex(p, obj_offset);
        if (!revidx)
                return OBJ_BAD;
-       sha1 = nth_packed_object_sha1(p, revidx->nr);
-       mark_bad_packed_object(p, sha1);
-       type = sha1_object_info(sha1, NULL);
+       nth_packed_object_oid(&oid, p, revidx->nr);
+       mark_bad_packed_object(p, oid.hash);
+       type = oid_object_info(&oid, NULL);
        if (type <= OBJ_NONE)
                return OBJ_BAD;
        return type;
@@ -1350,16 +1374,16 @@ int packed_object_info(struct packed_git *p, off_t obj_offset,
                *oi->disk_sizep = revidx[1].offset - obj_offset;
        }
 
-       if (oi->typep || oi->typename) {
+       if (oi->typep || oi->type_name) {
                enum object_type ptot;
                ptot = packed_to_object_type(p, obj_offset, type, &w_curs,
                                             curpos);
                if (oi->typep)
                        *oi->typep = ptot;
-               if (oi->typename) {
-                       const char *tn = typename(ptot);
+               if (oi->type_name) {
+                       const char *tn = type_name(ptot);
                        if (tn)
-                               strbuf_addstr(oi->typename, tn);
+                               strbuf_addstr(oi->type_name, tn);
                }
                if (ptot < 0) {
                        type = OBJ_BAD;
@@ -1441,7 +1465,7 @@ struct unpack_entry_stack_ent {
        unsigned long size;
 };
 
-static void *read_object(const unsigned char *sha1, enum object_type *type,
+static void *read_object(const struct object_id *oid, enum object_type *type,
                         unsigned long *size)
 {
        struct object_info oi = OBJECT_INFO_INIT;
@@ -1450,7 +1474,7 @@ static void *read_object(const unsigned char *sha1, enum object_type *type,
        oi.sizep = size;
        oi.contentp = &content;
 
-       if (sha1_object_info_extended(sha1, &oi, 0) < 0)
+       if (oid_object_info_extended(oid, &oi, 0) < 0)
                return NULL;
        return content;
 }
@@ -1490,11 +1514,11 @@ void *unpack_entry(struct packed_git *p, off_t obj_offset,
                        struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
                        off_t len = revidx[1].offset - obj_offset;
                        if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
-                               const unsigned char *sha1 =
-                                       nth_packed_object_sha1(p, revidx->nr);
+                               struct object_id oid;
+                               nth_packed_object_oid(&oid, p, revidx->nr);
                                error("bad packed object CRC for %s",
-                                     sha1_to_hex(sha1));
-                               mark_bad_packed_object(p, sha1);
+                                     oid_to_hex(&oid));
+                               mark_bad_packed_object(p, oid.hash);
                                data = NULL;
                                goto out;
                        }
@@ -1577,16 +1601,16 @@ void *unpack_entry(struct packed_git *p, off_t obj_offset,
                         * of a corrupted pack, and is better than failing outright.
                         */
                        struct revindex_entry *revidx;
-                       const unsigned char *base_sha1;
+                       struct object_id base_oid;
                        revidx = find_pack_revindex(p, obj_offset);
                        if (revidx) {
-                               base_sha1 = nth_packed_object_sha1(p, revidx->nr);
+                               nth_packed_object_oid(&base_oid, p, revidx->nr);
                                error("failed to read delta base object %s"
                                      " at offset %"PRIuMAX" from %s",
-                                     sha1_to_hex(base_sha1), (uintmax_t)obj_offset,
+                                     oid_to_hex(&base_oid), (uintmax_t)obj_offset,
                                      p->pack_name);
-                               mark_bad_packed_object(p, base_sha1);
-                               base = read_object(base_sha1, &type, &base_size);
+                               mark_bad_packed_object(p, base_oid.hash);
+                               base = read_object(&base_oid, &type, &base_size);
                                external_base = base;
                        }
                }
@@ -1643,6 +1667,29 @@ void *unpack_entry(struct packed_git *p, off_t obj_offset,
        return data;
 }
 
+int bsearch_pack(const struct object_id *oid, const struct packed_git *p, uint32_t *result)
+{
+       const unsigned char *index_fanout = p->index_data;
+       const unsigned char *index_lookup;
+       int index_lookup_width;
+
+       if (!index_fanout)
+               BUG("bsearch_pack called without a valid pack-index");
+
+       index_lookup = index_fanout + 4 * 256;
+       if (p->index_version == 1) {
+               index_lookup_width = 24;
+               index_lookup += 4;
+       } else {
+               index_lookup_width = 20;
+               index_fanout += 8;
+               index_lookup += 8;
+       }
+
+       return bsearch_hash(oid->hash, (const uint32_t*)index_fanout,
+                           index_lookup, index_lookup_width, result);
+}
+
 const unsigned char *nth_packed_object_sha1(struct packed_git *p,
                                            uint32_t n)
 {
@@ -1702,60 +1749,25 @@ off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
                        return off;
                index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
                check_pack_index_ptr(p, index);
-               return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
-                                  ntohl(*((uint32_t *)(index + 4)));
+               return get_be64(index);
        }
 }
 
 off_t find_pack_entry_one(const unsigned char *sha1,
                                  struct packed_git *p)
 {
-       const uint32_t *level1_ofs = p->index_data;
        const unsigned char *index = p->index_data;
-       unsigned hi, lo, stride;
-       static int debug_lookup = -1;
-
-       if (debug_lookup < 0)
-               debug_lookup = !!getenv("GIT_DEBUG_LOOKUP");
+       struct object_id oid;
+       uint32_t result;
 
        if (!index) {
                if (open_pack_index(p))
                        return 0;
-               level1_ofs = p->index_data;
-               index = p->index_data;
-       }
-       if (p->index_version > 1) {
-               level1_ofs += 2;
-               index += 8;
-       }
-       index += 4 * 256;
-       hi = ntohl(level1_ofs[*sha1]);
-       lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
-       if (p->index_version > 1) {
-               stride = 20;
-       } else {
-               stride = 24;
-               index += 4;
        }
 
-       if (debug_lookup)
-               printf("%02x%02x%02x... lo %u hi %u nr %"PRIu32"\n",
-                      sha1[0], sha1[1], sha1[2], lo, hi, p->num_objects);
-
-       while (lo < hi) {
-               unsigned mi = lo + (hi - lo) / 2;
-               int cmp = hashcmp(index + mi * stride, sha1);
-
-               if (debug_lookup)
-                       printf("lo %u hi %u rg %u mi %u\n",
-                              lo, hi, hi - lo, mi);
-               if (!cmp)
-                       return nth_packed_object_offset(p, mi);
-               if (cmp > 0)
-                       hi = mi;
-               else
-                       lo = mi+1;
-       }
+       hashcpy(oid.hash, sha1);
+       if (bsearch_pack(&oid, p, &result))
+               return nth_packed_object_offset(p, result);
        return 0;
 }
 
@@ -1825,21 +1837,18 @@ static int fill_pack_entry(const unsigned char *sha1,
        return 1;
 }
 
-/*
- * Iff a pack file contains the object named by sha1, return true and
- * store its location to e.
- */
-int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
+int find_pack_entry(struct repository *r, const unsigned char *sha1, struct pack_entry *e)
 {
-       struct mru_entry *p;
+       struct list_head *pos;
 
-       prepare_packed_git();
-       if (!packed_git)
+       prepare_packed_git(r);
+       if (!r->objects->packed_git)
                return 0;
 
-       for (p = packed_git_mru.head; p; p = p->next) {
-               if (fill_pack_entry(sha1, e, p->item)) {
-                       mru_mark(&packed_git_mru, p);
+       list_for_each(pos, &r->objects->packed_git_mru) {
+               struct packed_git *p = list_entry(pos, struct packed_git, mru);
+               if (fill_pack_entry(sha1, e, p)) {
+                       list_move(&p->mru, &r->objects->packed_git_mru);
                        return 1;
                }
        }
@@ -1849,7 +1858,7 @@ int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
 int has_sha1_pack(const unsigned char *sha1)
 {
        struct pack_entry e;
-       return find_pack_entry(sha1, &e);
+       return find_pack_entry(the_repository, sha1, &e);
 }
 
 int has_pack_index(const unsigned char *sha1)
@@ -1885,10 +1894,13 @@ int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags)
        int r = 0;
        int pack_errors = 0;
 
-       prepare_packed_git();
-       for (p = packed_git; p; p = p->next) {
+       prepare_packed_git(the_repository);
+       for (p = the_repository->objects->packed_git; p; p = p->next) {
                if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local)
                        continue;
+               if ((flags & FOR_EACH_OBJECT_PROMISOR_ONLY) &&
+                   !p->pack_promisor)
+                       continue;
                if (open_pack_index(p)) {
                        pack_errors = 1;
                        continue;
@@ -1899,3 +1911,61 @@ int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags)
        }
        return r ? r : pack_errors;
 }
+
+static int add_promisor_object(const struct object_id *oid,
+                              struct packed_git *pack,
+                              uint32_t pos,
+                              void *set_)
+{
+       struct oidset *set = set_;
+       struct object *obj = parse_object(oid);
+       if (!obj)
+               return 1;
+
+       oidset_insert(set, oid);
+
+       /*
+        * If this is a tree, commit, or tag, the objects it refers
+        * to are also promisor objects. (Blobs refer to no objects->)
+        */
+       if (obj->type == OBJ_TREE) {
+               struct tree *tree = (struct tree *)obj;
+               struct tree_desc desc;
+               struct name_entry entry;
+               if (init_tree_desc_gently(&desc, tree->buffer, tree->size))
+                       /*
+                        * Error messages are given when packs are
+                        * verified, so do not print any here.
+                        */
+                       return 0;
+               while (tree_entry_gently(&desc, &entry))
+                       oidset_insert(set, entry.oid);
+       } else if (obj->type == OBJ_COMMIT) {
+               struct commit *commit = (struct commit *) obj;
+               struct commit_list *parents = commit->parents;
+
+               oidset_insert(set, &commit->tree->object.oid);
+               for (; parents; parents = parents->next)
+                       oidset_insert(set, &parents->item->object.oid);
+       } else if (obj->type == OBJ_TAG) {
+               struct tag *tag = (struct tag *) obj;
+               oidset_insert(set, &tag->tagged->oid);
+       }
+       return 0;
+}
+
+int is_promisor_object(const struct object_id *oid)
+{
+       static struct oidset promisor_objects;
+       static int promisor_objects_prepared;
+
+       if (!promisor_objects_prepared) {
+               if (repository_format_partial_clone) {
+                       for_each_packed_object(add_promisor_object,
+                                              &promisor_objects,
+                                              FOR_EACH_OBJECT_PROMISOR_ONLY);
+               }
+               promisor_objects_prepared = 1;
+       }
+       return oidset_contains(&promisor_objects, oid);
+}