Merge branch 'jt/binsearch-with-fanout' into HEAD
authorJunio C Hamano <gitster@pobox.com>
Tue, 13 Mar 2018 20:34:04 +0000 (13:34 -0700)
committerJunio C Hamano <gitster@pobox.com>
Tue, 13 Mar 2018 20:34:04 +0000 (13:34 -0700)
* jt/binsearch-with-fanout:
packfile: refactor hash search with fanout table
packfile: remove GIT_DEBUG_LOOKUP log statements

1  2 
packfile.c
diff --combined packfile.c
index 7dbe8739d17d64fad79eac9e7477d00ea3d80df2,59648a1820706cc155c1e75586d220cedf7ae25c..5d07f330c8a4bf0a748e3fc87ed70f5affafd681
@@@ -1,5 -1,5 +1,5 @@@
  #include "cache.h"
 -#include "mru.h"
 +#include "list.h"
  #include "pack.h"
  #include "dir.h"
  #include "mergesort.h"
@@@ -8,11 -8,6 +8,11 @@@
  #include "list.h"
  #include "streaming.h"
  #include "sha1-lookup.h"
 +#include "commit.h"
 +#include "object.h"
 +#include "tag.h"
 +#include "tree-walk.h"
 +#include "tree.h"
  
  char *odb_pack_name(struct strbuf *buf,
                    const unsigned char *sha1,
@@@ -45,7 -40,7 +45,7 @@@ static unsigned int pack_max_fds
  static size_t peak_pack_mapped;
  static size_t pack_mapped;
  struct packed_git *packed_git;
 -struct mru packed_git_mru;
 +LIST_HEAD(packed_git_mru);
  
  #define SZ_FMT PRIuMAX
  static inline uintmax_t sz_fmt(size_t s) { return s; }
@@@ -648,10 -643,10 +648,10 @@@ struct packed_git *add_packed_git(cons
                return NULL;
  
        /*
 -       * ".pack" is long enough to hold any suffix we're adding (and
 +       * ".promisor" is long enough to hold any suffix we're adding (and
         * the use xsnprintf double-checks that)
         */
 -      alloc = st_add3(path_len, strlen(".pack"), 1);
 +      alloc = st_add3(path_len, strlen(".promisor"), 1);
        p = alloc_packed_git(alloc);
        memcpy(p->pack_name, path, path_len);
  
        if (!access(p->pack_name, F_OK))
                p->pack_keep = 1;
  
 +      xsnprintf(p->pack_name + path_len, alloc - path_len, ".promisor");
 +      if (!access(p->pack_name, F_OK))
 +              p->pack_promisor = 1;
 +
        xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack");
        if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
                free(p);
@@@ -790,8 -781,7 +790,8 @@@ static void prepare_packed_git_one(cha
                if (ends_with(de->d_name, ".idx") ||
                    ends_with(de->d_name, ".pack") ||
                    ends_with(de->d_name, ".bitmap") ||
 -                  ends_with(de->d_name, ".keep"))
 +                  ends_with(de->d_name, ".keep") ||
 +                  ends_with(de->d_name, ".promisor"))
                        string_list_append(&garbage, path.buf);
                else
                        report_garbage(PACKDIR_FILE_GARBAGE, path.buf);
@@@ -876,10 -866,9 +876,10 @@@ static void prepare_packed_git_mru(void
  {
        struct packed_git *p;
  
 -      mru_clear(&packed_git_mru);
 +      INIT_LIST_HEAD(&packed_git_mru);
 +
        for (p = packed_git; p; p = p->next)
 -              mru_append(&packed_git_mru, p);
 +              list_add_tail(&p->mru, &packed_git_mru);
  }
  
  static int prepare_packed_git_run_once = 0;
@@@ -1713,7 -1702,8 +1713,7 @@@ off_t nth_packed_object_offset(const st
                        return off;
                index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
                check_pack_index_ptr(p, index);
 -              return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
 -                                 ntohl(*((uint32_t *)(index + 4)));
 +              return get_be64(index);
        }
  }
  
@@@ -1722,11 -1712,8 +1722,8 @@@ off_t find_pack_entry_one(const unsigne
  {
        const uint32_t *level1_ofs = p->index_data;
        const unsigned char *index = p->index_data;
-       unsigned hi, lo, stride;
-       static int debug_lookup = -1;
-       if (debug_lookup < 0)
-               debug_lookup = !!getenv("GIT_DEBUG_LOOKUP");
+       unsigned stride;
+       uint32_t result;
  
        if (!index) {
                if (open_pack_index(p))
                index += 8;
        }
        index += 4 * 256;
-       hi = ntohl(level1_ofs[*sha1]);
-       lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
        if (p->index_version > 1) {
                stride = 20;
        } else {
                index += 4;
        }
  
-       if (debug_lookup)
-               printf("%02x%02x%02x... lo %u hi %u nr %"PRIu32"\n",
-                      sha1[0], sha1[1], sha1[2], lo, hi, p->num_objects);
-       while (lo < hi) {
-               unsigned mi = lo + (hi - lo) / 2;
-               int cmp = hashcmp(index + mi * stride, sha1);
-               if (debug_lookup)
-                       printf("lo %u hi %u rg %u mi %u\n",
-                              lo, hi, hi - lo, mi);
-               if (!cmp)
-                       return nth_packed_object_offset(p, mi);
-               if (cmp > 0)
-                       hi = mi;
-               else
-                       lo = mi+1;
-       }
+       if (bsearch_hash(sha1, level1_ofs, index, stride, &result))
+               return nth_packed_object_offset(p, result);
        return 0;
  }
  
@@@ -1841,16 -1810,15 +1820,16 @@@ static int fill_pack_entry(const unsign
   */
  int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
  {
 -      struct mru_entry *p;
 +      struct list_head *pos;
  
        prepare_packed_git();
        if (!packed_git)
                return 0;
  
 -      for (p = packed_git_mru.head; p; p = p->next) {
 -              if (fill_pack_entry(sha1, e, p->item)) {
 -                      mru_mark(&packed_git_mru, p);
 +      list_for_each(pos, &packed_git_mru) {
 +              struct packed_git *p = list_entry(pos, struct packed_git, mru);
 +              if (fill_pack_entry(sha1, e, p)) {
 +                      list_move(&p->mru, &packed_git_mru);
                        return 1;
                }
        }
@@@ -1900,9 -1868,6 +1879,9 @@@ int for_each_packed_object(each_packed_
        for (p = packed_git; p; p = p->next) {
                if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local)
                        continue;
 +              if ((flags & FOR_EACH_OBJECT_PROMISOR_ONLY) &&
 +                  !p->pack_promisor)
 +                      continue;
                if (open_pack_index(p)) {
                        pack_errors = 1;
                        continue;
        }
        return r ? r : pack_errors;
  }
 +
 +static int add_promisor_object(const struct object_id *oid,
 +                             struct packed_git *pack,
 +                             uint32_t pos,
 +                             void *set_)
 +{
 +      struct oidset *set = set_;
 +      struct object *obj = parse_object(oid);
 +      if (!obj)
 +              return 1;
 +
 +      oidset_insert(set, oid);
 +
 +      /*
 +       * If this is a tree, commit, or tag, the objects it refers
 +       * to are also promisor objects. (Blobs refer to no objects.)
 +       */
 +      if (obj->type == OBJ_TREE) {
 +              struct tree *tree = (struct tree *)obj;
 +              struct tree_desc desc;
 +              struct name_entry entry;
 +              if (init_tree_desc_gently(&desc, tree->buffer, tree->size))
 +                      /*
 +                       * Error messages are given when packs are
 +                       * verified, so do not print any here.
 +                       */
 +                      return 0;
 +              while (tree_entry_gently(&desc, &entry))
 +                      oidset_insert(set, entry.oid);
 +      } else if (obj->type == OBJ_COMMIT) {
 +              struct commit *commit = (struct commit *) obj;
 +              struct commit_list *parents = commit->parents;
 +
 +              oidset_insert(set, &commit->tree->object.oid);
 +              for (; parents; parents = parents->next)
 +                      oidset_insert(set, &parents->item->object.oid);
 +      } else if (obj->type == OBJ_TAG) {
 +              struct tag *tag = (struct tag *) obj;
 +              oidset_insert(set, &tag->tagged->oid);
 +      }
 +      return 0;
 +}
 +
 +int is_promisor_object(const struct object_id *oid)
 +{
 +      static struct oidset promisor_objects;
 +      static int promisor_objects_prepared;
 +
 +      if (!promisor_objects_prepared) {
 +              if (repository_format_partial_clone) {
 +                      for_each_packed_object(add_promisor_object,
 +                                             &promisor_objects,
 +                                             FOR_EACH_OBJECT_PROMISOR_ONLY);
 +              }
 +              promisor_objects_prepared = 1;
 +      }
 +      return oidset_contains(&promisor_objects, oid);
 +}