Merge branch 'cc/delta-islands'
authorJunio C Hamano <gitster@pobox.com>
Wed, 21 Nov 2018 11:39:02 +0000 (20:39 +0900)
committerJunio C Hamano <gitster@pobox.com>
Wed, 21 Nov 2018 11:39:02 +0000 (20:39 +0900)
A few issues in the implementation of "delta-islands" feature has
been corrected.

* cc/delta-islands:
pack-objects: fix off-by-one in delta-island tree-depth computation
pack-objects: zero-initialize tree_depth/layer arrays
pack-objects: fix tree_depth and layer invariants

1  2 
builtin/pack-objects.c
git-compat-util.h
pack-objects.h
diff --combined builtin/pack-objects.c
index e7ea206c08c6d4e81c3649e3988faf4794ce013c,3d70ab1f427fd846195997be27f0d2d31f349e5c..411aefd6875b2d35ee4a12d1a043ba50027021b3
@@@ -32,7 -32,6 +32,7 @@@
  #include "packfile.h"
  #include "object-store.h"
  #include "dir.h"
 +#include "midx.h"
  
  #define IN_PACK(obj) oe_in_pack(&to_pack, obj)
  #define SIZE(obj) oe_size(&to_pack, obj)
@@@ -42,7 -41,6 +42,7 @@@
  #define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj)
  #define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj)
  #define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val)
 +#define SET_DELTA_EXT(obj, oid) oe_set_delta_ext(&to_pack, obj, oid)
  #define SET_DELTA_SIZE(obj, val) oe_set_delta_size(&to_pack, obj, val)
  #define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val)
  #define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val)
@@@ -62,7 -60,6 +62,7 @@@ static struct packing_data to_pack
  
  static struct pack_idx_entry **written_list;
  static uint32_t nr_result, nr_written, nr_seen;
 +static struct bitmap_index *bitmap_git;
  static uint32_t write_layer;
  
  static int non_empty;
@@@ -84,7 -81,6 +84,7 @@@ static unsigned long pack_size_limit
  static int depth = 50;
  static int delta_search_threads;
  static int pack_to_stdout;
 +static int thin;
  static int num_preferred_base;
  static struct progress *progress_state;
  
@@@ -148,7 -144,7 +148,7 @@@ static void *get_delta(struct object_en
  
        buf = read_object_file(&entry->idx.oid, &type, &size);
        if (!buf)
 -              die("unable to read %s", oid_to_hex(&entry->idx.oid));
 +              die(_("unable to read %s"), oid_to_hex(&entry->idx.oid));
        base_buf = read_object_file(&DELTA(entry)->idx.oid, &type,
                                    &base_size);
        if (!base_buf)
                    oid_to_hex(&DELTA(entry)->idx.oid));
        delta_buf = diff_delta(base_buf, base_size,
                               buf, size, &delta_size, 0);
 +      /*
 +       * We succesfully computed this delta once but dropped it for
 +       * memory reasons. Something is very wrong if this time we
 +       * recompute and create a different delta.
 +       */
        if (!delta_buf || delta_size != DELTA_SIZE(entry))
 -              die("delta size changed");
 +              BUG("delta size changed");
        free(buf);
        free(base_buf);
        return delta_buf;
@@@ -419,7 -410,7 +419,7 @@@ static off_t write_reuse_object(struct 
        datalen = revidx[1].offset - offset;
        if (!pack_to_stdout && p->index_version > 1 &&
            check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) {
 -              error("bad packed object CRC for %s",
 +              error(_("bad packed object CRC for %s"),
                      oid_to_hex(&entry->idx.oid));
                unuse_pack(&w_curs);
                return write_no_reuse_object(f, entry, limit, usable_delta);
  
        if (!pack_to_stdout && p->index_version == 1 &&
            check_pack_inflate(p, &w_curs, offset, datalen, entry_size)) {
 -              error("corrupt packed object for %s",
 +              error(_("corrupt packed object for %s"),
                      oid_to_hex(&entry->idx.oid));
                unuse_pack(&w_curs);
                return write_no_reuse_object(f, entry, limit, usable_delta);
@@@ -561,7 -552,7 +561,7 @@@ static enum write_one_status write_one(
         */
        recursing = (e->idx.offset == 1);
        if (recursing) {
 -              warning("recursive delta detected for object %s",
 +              warning(_("recursive delta detected for object %s"),
                        oid_to_hex(&e->idx.oid));
                return WRITE_ONE_RECURSIVE;
        } else if (e->idx.offset || e->preferred_base) {
  
        /* make sure off_t is sufficiently large not to wrap */
        if (signed_add_overflows(*offset, size))
 -              die("pack too large for current definition of off_t");
 +              die(_("pack too large for current definition of off_t"));
        *offset += size;
        return WRITE_ONE_WRITTEN;
  }
@@@ -772,8 -763,7 +772,8 @@@ static struct object_entry **compute_wr
                compute_layer_order(wo, &wo_end);
  
        if (wo_end != to_pack.nr_objects)
 -              die("ordered %u objects, expected %"PRIu32, wo_end, to_pack.nr_objects);
 +              die(_("ordered %u objects, expected %"PRIu32),
 +                  wo_end, to_pack.nr_objects);
  
        return wo;
  }
@@@ -785,15 -775,15 +785,15 @@@ static off_t write_reused_pack(struct h
        int fd;
  
        if (!is_pack_valid(reuse_packfile))
 -              die("packfile is invalid: %s", reuse_packfile->pack_name);
 +              die(_("packfile is invalid: %s"), reuse_packfile->pack_name);
  
        fd = git_open(reuse_packfile->pack_name);
        if (fd < 0)
 -              die_errno("unable to open packfile for reuse: %s",
 +              die_errno(_("unable to open packfile for reuse: %s"),
                          reuse_packfile->pack_name);
  
        if (lseek(fd, sizeof(struct pack_header), SEEK_SET) == -1)
 -              die_errno("unable to seek in reused packfile");
 +              die_errno(_("unable to seek in reused packfile"));
  
        if (reuse_packfile_offset < 0)
                reuse_packfile_offset = reuse_packfile->pack_size - the_hash_algo->rawsz;
                int read_pack = xread(fd, buffer, sizeof(buffer));
  
                if (read_pack <= 0)
 -                      die_errno("unable to read from reused packfile");
 +                      die_errno(_("unable to read from reused packfile"));
  
                if (read_pack > to_write)
                        read_pack = to_write;
@@@ -907,7 -897,7 +907,7 @@@ static void write_pack_file(void
                         * to preserve this property.
                         */
                        if (stat(pack_tmp_name, &st) < 0) {
 -                              warning_errno("failed to stat %s", pack_tmp_name);
 +                              warning_errno(_("failed to stat %s"), pack_tmp_name);
                        } else if (!last_mtime) {
                                last_mtime = st.st_mtime;
                        } else {
                                utb.actime = st.st_atime;
                                utb.modtime = --last_mtime;
                                if (utime(pack_tmp_name, &utb) < 0)
 -                                      warning_errno("failed utime() on %s", pack_tmp_name);
 +                                      warning_errno(_("failed utime() on %s"), pack_tmp_name);
                        }
  
                        strbuf_addf(&tmpname, "%s-", base_name);
        free(write_order);
        stop_progress(&progress_state);
        if (written != nr_result)
 -              die("wrote %"PRIu32" objects while expecting %"PRIu32,
 -                      written, nr_result);
 +              die(_("wrote %"PRIu32" objects while expecting %"PRIu32),
 +                  written, nr_result);
  }
  
  static int no_try_delta(const char *path)
  
        if (!check)
                check = attr_check_initl("delta", NULL);
 -      if (git_check_attr(path, check))
 -              return 0;
 +      git_check_attr(&the_index, path, check);
        if (ATTR_FALSE(check->items[0].value))
                return 1;
        return 0;
@@@ -1058,7 -1049,6 +1058,7 @@@ static int want_object_in_pack(const st
  {
        int want;
        struct list_head *pos;
 +      struct multi_pack_index *m;
  
        if (!exclude && local && has_loose_object_nonlocal(oid))
                return 0;
                if (want != -1)
                        return want;
        }
 +
 +      for (m = get_multi_pack_index(the_repository); m; m = m->next) {
 +              struct pack_entry e;
 +              if (fill_midx_entry(oid, &e, m)) {
 +                      struct packed_git *p = e.p;
 +                      off_t offset;
 +
 +                      if (p == *found_pack)
 +                              offset = *found_offset;
 +                      else
 +                              offset = find_pack_entry_one(oid->hash, p);
 +
 +                      if (offset) {
 +                              if (!*found_pack) {
 +                                      if (!is_pack_valid(p))
 +                                              continue;
 +                                      *found_offset = offset;
 +                                      *found_pack = p;
 +                              }
 +                              want = want_found_object(exclude, p);
 +                              if (want != -1)
 +                                      return want;
 +                      }
 +              }
 +      }
 +
        list_for_each(pos, get_packed_git_mru(the_repository)) {
                struct packed_git *p = list_entry(pos, struct packed_git, mru);
                off_t offset;
@@@ -1247,7 -1211,7 +1247,7 @@@ static struct pbase_tree_cache *pbase_t
         */
        for (neigh = 0; neigh < 8; neigh++) {
                ent = pbase_tree_cache[my_ix];
 -              if (ent && !oidcmp(&ent->oid, oid)) {
 +              if (ent && oideq(&ent->oid, oid)) {
                        ent->ref++;
                        return ent;
                }
@@@ -1429,7 -1393,7 +1429,7 @@@ static void add_preferred_base(struct o
                return;
  
        for (it = pbase_tree; it; it = it->next) {
 -              if (!oidcmp(&it->pcache.oid, &tree_oid)) {
 +              if (oideq(&it->pcache.oid, &tree_oid)) {
                        free(data);
                        return;
                }
@@@ -1469,57 -1433,6 +1469,57 @@@ static void cleanup_preferred_base(void
        done_pbase_paths_num = done_pbase_paths_alloc = 0;
  }
  
 +/*
 + * Return 1 iff the object specified by "delta" can be sent
 + * literally as a delta against the base in "base_sha1". If
 + * so, then *base_out will point to the entry in our packing
 + * list, or NULL if we must use the external-base list.
 + *
 + * Depth value does not matter - find_deltas() will
 + * never consider reused delta as the base object to
 + * deltify other objects against, in order to avoid
 + * circular deltas.
 + */
 +static int can_reuse_delta(const unsigned char *base_sha1,
 +                         struct object_entry *delta,
 +                         struct object_entry **base_out)
 +{
 +      struct object_entry *base;
 +
 +      if (!base_sha1)
 +              return 0;
 +
 +      /*
 +       * First see if we're already sending the base (or it's explicitly in
 +       * our "excluded" list).
 +       */
 +      base = packlist_find(&to_pack, base_sha1, NULL);
 +      if (base) {
 +              if (!in_same_island(&delta->idx.oid, &base->idx.oid))
 +                      return 0;
 +              *base_out = base;
 +              return 1;
 +      }
 +
 +      /*
 +       * Otherwise, reachability bitmaps may tell us if the receiver has it,
 +       * even if it was buried too deep in history to make it into the
 +       * packing list.
 +       */
 +      if (thin && bitmap_has_sha1_in_uninteresting(bitmap_git, base_sha1)) {
 +              if (use_delta_islands) {
 +                      struct object_id base_oid;
 +                      hashcpy(base_oid.hash, base_sha1);
 +                      if (!in_same_island(&delta->idx.oid, &base_oid))
 +                              return 0;
 +              }
 +              *base_out = NULL;
 +              return 1;
 +      }
 +
 +      return 0;
 +}
 +
  static void check_object(struct object_entry *entry)
  {
        unsigned long canonical_size;
                        while (c & 128) {
                                ofs += 1;
                                if (!ofs || MSB(ofs, 7)) {
 -                                      error("delta base offset overflow in pack for %s",
 +                                      error(_("delta base offset overflow in pack for %s"),
                                              oid_to_hex(&entry->idx.oid));
                                        goto give_up;
                                }
                        }
                        ofs = entry->in_pack_offset - ofs;
                        if (ofs <= 0 || ofs >= entry->in_pack_offset) {
 -                              error("delta base offset out of bound for %s",
 +                              error(_("delta base offset out of bound for %s"),
                                      oid_to_hex(&entry->idx.oid));
                                goto give_up;
                        }
                        break;
                }
  
 -              if (base_ref && (base_entry = packlist_find(&to_pack, base_ref, NULL)) &&
 -                  in_same_island(&entry->idx.oid, &base_entry->idx.oid)) {
 -                      /*
 -                       * If base_ref was set above that means we wish to
 -                       * reuse delta data, and we even found that base
 -                       * in the list of objects we want to pack. Goodie!
 -                       *
 -                       * Depth value does not matter - find_deltas() will
 -                       * never consider reused delta as the base object to
 -                       * deltify other objects against, in order to avoid
 -                       * circular deltas.
 -                       */
 +              if (can_reuse_delta(base_ref, entry, &base_entry)) {
                        oe_set_type(entry, entry->in_pack_type);
                        SET_SIZE(entry, in_pack_size); /* delta size */
 -                      SET_DELTA(entry, base_entry);
                        SET_DELTA_SIZE(entry, in_pack_size);
 -                      entry->delta_sibling_idx = base_entry->delta_child_idx;
 -                      SET_DELTA_CHILD(base_entry, entry);
 +
 +                      if (base_entry) {
 +                              SET_DELTA(entry, base_entry);
 +                              entry->delta_sibling_idx = base_entry->delta_child_idx;
 +                              SET_DELTA_CHILD(base_entry, entry);
 +                      } else {
 +                              SET_DELTA_EXT(entry, base_ref);
 +                      }
 +
                        unuse_pack(&w_curs);
                        return;
                }
@@@ -1953,30 -1871,30 +1953,30 @@@ static int delta_cacheable(unsigned lon
        return 0;
  }
  
 -#ifndef NO_PTHREADS
 -
 +/* Protect access to object database */
  static pthread_mutex_t read_mutex;
  #define read_lock()           pthread_mutex_lock(&read_mutex)
  #define read_unlock()         pthread_mutex_unlock(&read_mutex)
  
 +/* Protect delta_cache_size */
  static pthread_mutex_t cache_mutex;
  #define cache_lock()          pthread_mutex_lock(&cache_mutex)
  #define cache_unlock()                pthread_mutex_unlock(&cache_mutex)
  
 +/*
 + * Protect object list partitioning (e.g. struct thread_param) and
 + * progress_state
 + */
  static pthread_mutex_t progress_mutex;
  #define progress_lock()               pthread_mutex_lock(&progress_mutex)
  #define progress_unlock()     pthread_mutex_unlock(&progress_mutex)
  
 -#else
 -
 -#define read_lock()           (void)0
 -#define read_unlock()         (void)0
 -#define cache_lock()          (void)0
 -#define cache_unlock()                (void)0
 -#define progress_lock()               (void)0
 -#define progress_unlock()     (void)0
 -
 -#endif
 +/*
 + * Access to struct object_entry is unprotected since each thread owns
 + * a portion of the main object list. Just don't access object entries
 + * ahead in the list because they can be stolen and would need
 + * progress_mutex for protection.
 + */
  
  /*
   * Return the size of the object without doing any delta
@@@ -2080,12 -1998,12 +2080,12 @@@ static int try_delta(struct unpacked *t
                trg->data = read_object_file(&trg_entry->idx.oid, &type, &sz);
                read_unlock();
                if (!trg->data)
 -                      die("object %s cannot be read",
 +                      die(_("object %s cannot be read"),
                            oid_to_hex(&trg_entry->idx.oid));
                if (sz != trg_size)
 -                      die("object %s inconsistent object length (%lu vs %lu)",
 -                          oid_to_hex(&trg_entry->idx.oid), sz,
 -                          trg_size);
 +                      die(_("object %s inconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"),
 +                          oid_to_hex(&trg_entry->idx.oid), (uintmax_t)sz,
 +                          (uintmax_t)trg_size);
                *mem_usage += sz;
        }
        if (!src->data) {
                        if (src_entry->preferred_base) {
                                static int warned = 0;
                                if (!warned++)
 -                                      warning("object %s cannot be read",
 +                                      warning(_("object %s cannot be read"),
                                                oid_to_hex(&src_entry->idx.oid));
                                /*
                                 * Those objects are not included in the
                                 */
                                return 0;
                        }
 -                      die("object %s cannot be read",
 +                      die(_("object %s cannot be read"),
                            oid_to_hex(&src_entry->idx.oid));
                }
                if (sz != src_size)
 -                      die("object %s inconsistent object length (%lu vs %lu)",
 -                          oid_to_hex(&src_entry->idx.oid), sz,
 -                          src_size);
 +                      die(_("object %s inconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"),
 +                          oid_to_hex(&src_entry->idx.oid), (uintmax_t)sz,
 +                          (uintmax_t)src_size);
                *mem_usage += sz;
        }
        if (!src->index) {
                if (!src->index) {
                        static int warned = 0;
                        if (!warned++)
 -                              warning("suboptimal pack - out of memory");
 +                              warning(_("suboptimal pack - out of memory"));
                        return 0;
                }
                *mem_usage += sizeof_delta_index(src->index);
        delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size);
        if (!delta_buf)
                return 0;
 -      if (delta_size >= (1U << OE_DELTA_SIZE_BITS)) {
 -              free(delta_buf);
 -              return 0;
 -      }
  
        if (DELTA(trg_entry)) {
                /* Prefer only shallower same-sized deltas. */
@@@ -2335,6 -2257,8 +2335,6 @@@ static void find_deltas(struct object_e
        free(array);
  }
  
 -#ifndef NO_PTHREADS
 -
  static void try_to_free_from_threads(size_t size)
  {
        read_lock();
  static try_to_free_t old_try_to_free_routine;
  
  /*
 + * The main object list is split into smaller lists, each is handed to
 + * one worker.
 + *
   * The main thread waits on the condition that (at least) one of the workers
   * has stopped working (which is indicated in the .working member of
   * struct thread_params).
 + *
   * When a work thread has completed its work, it sets .working to 0 and
   * signals the main thread and waits on the condition that .data_ready
   * becomes 1.
 + *
 + * The main thread steals half of the work from the worker that has
 + * most work left to hand it to the idle worker.
   */
  
  struct thread_params {
@@@ -2448,8 -2365,8 +2448,8 @@@ static void ll_find_deltas(struct objec
                return;
        }
        if (progress > pack_to_stdout)
 -              fprintf(stderr, "Delta compression using up to %d threads.\n",
 -                              delta_search_threads);
 +              fprintf_ln(stderr, _("Delta compression using up to %d threads"),
 +                         delta_search_threads);
        p = xcalloc(delta_search_threads, sizeof(*p));
  
        /* Partition the work amongst work threads. */
                ret = pthread_create(&p[i].thread, NULL,
                                     threaded_find_deltas, &p[i]);
                if (ret)
 -                      die("unable to create thread: %s", strerror(ret));
 +                      die(_("unable to create thread: %s"), strerror(ret));
                active_threads++;
        }
  
        free(p);
  }
  
 -#else
 -#define ll_find_deltas(l, s, w, d, p) find_deltas(l, &s, w, d, p)
 -#endif
 -
  static void add_tag_chain(const struct object_id *oid)
  {
        struct tag *tag;
        tag = lookup_tag(the_repository, oid);
        while (1) {
                if (!tag || parse_tag(tag) || !tag->tagged)
 -                      die("unable to pack objects reachable from tag %s",
 +                      die(_("unable to pack objects reachable from tag %s"),
                            oid_to_hex(oid));
  
                add_object_entry(&tag->object.oid, OBJ_TAG, NULL, 0);
@@@ -2649,7 -2570,7 +2649,7 @@@ static void prepare_pack(int window, in
                if (!entry->preferred_base) {
                        nr_deltas++;
                        if (oe_type(entry) < 0)
 -                              die("unable to get type of object %s",
 +                              die(_("unable to get type of object %s"),
                                    oid_to_hex(&entry->idx.oid));
                } else {
                        if (oe_type(entry) < 0) {
                ll_find_deltas(delta_list, n, window+1, depth, &nr_done);
                stop_progress(&progress_state);
                if (nr_done != nr_deltas)
 -                      die("inconsistency with delta count");
 +                      die(_("inconsistency with delta count"));
        }
        free(delta_list);
  }
@@@ -2713,18 -2634,20 +2713,18 @@@ static int git_pack_config(const char *
        if (!strcmp(k, "pack.threads")) {
                delta_search_threads = git_config_int(k, v);
                if (delta_search_threads < 0)
 -                      die("invalid number of threads specified (%d)",
 +                      die(_("invalid number of threads specified (%d)"),
                            delta_search_threads);
 -#ifdef NO_PTHREADS
 -              if (delta_search_threads != 1) {
 -                      warning("no threads support, ignoring %s", k);
 +              if (!HAVE_THREADS && delta_search_threads != 1) {
 +                      warning(_("no threads support, ignoring %s"), k);
                        delta_search_threads = 0;
                }
 -#endif
                return 0;
        }
        if (!strcmp(k, "pack.indexversion")) {
                pack_idx_opts.version = git_config_int(k, v);
                if (pack_idx_opts.version > 2)
 -                      die("bad pack.indexversion=%"PRIu32,
 +                      die(_("bad pack.indexversion=%"PRIu32),
                            pack_idx_opts.version);
                return 0;
        }
@@@ -2742,7 -2665,7 +2742,7 @@@ static void read_object_list_from_stdin
                        if (feof(stdin))
                                break;
                        if (!ferror(stdin))
 -                              die("fgets returned NULL, not EOF, not error!");
 +                              die("BUG: fgets returned NULL, not EOF, not error!");
                        if (errno != EINTR)
                                die_errno("fgets");
                        clearerr(stdin);
                }
                if (line[0] == '-') {
                        if (get_oid_hex(line+1, &oid))
 -                              die("expected edge object ID, got garbage:\n %s",
 +                              die(_("expected edge object ID, got garbage:\n %s"),
                                    line);
                        add_preferred_base(&oid);
                        continue;
                }
                if (parse_oid_hex(line, &oid, &p))
 -                      die("expected object ID, got garbage:\n %s", line);
 +                      die(_("expected object ID, got garbage:\n %s"), line);
  
                add_preferred_base_object(p + 1);
                add_object_entry(&oid, OBJ_NONE, p + 1, 0);
@@@ -2786,9 -2709,11 +2786,11 @@@ static void show_object(struct object *
  
        if (use_delta_islands) {
                const char *p;
-               unsigned depth = 0;
+               unsigned depth;
                struct object_entry *ent;
  
+               /* the empty string is a root tree, which is depth 0 */
+               depth = *name ? 1 : 0;
                for (p = strchr(name, '/'); p; p = strchr(p + 1, '/'))
                        depth++;
  
@@@ -2904,14 -2829,14 +2906,14 @@@ static void add_objects_in_unpacked_pac
  
        memset(&in_pack, 0, sizeof(in_pack));
  
 -      for (p = get_packed_git(the_repository); p; p = p->next) {
 +      for (p = get_all_packs(the_repository); p; p = p->next) {
                struct object_id oid;
                struct object *o;
  
                if (!p->pack_local || p->pack_keep || p->pack_keep_in_core)
                        continue;
                if (open_pack_index(p))
 -                      die("cannot open pack index");
 +                      die(_("cannot open pack index"));
  
                ALLOC_GROW(in_pack.array,
                           in_pack.nr + p->num_objects,
@@@ -2942,7 -2867,7 +2944,7 @@@ static int add_loose_object(const struc
        enum object_type type = oid_object_info(the_repository, oid, NULL);
  
        if (type < 0) {
 -              warning("loose object at %s could not be examined", path);
 +              warning(_("loose object at %s could not be examined"), path);
                return 0;
        }
  
@@@ -2968,7 -2893,7 +2970,7 @@@ static int has_sha1_pack_kept_or_nonloc
        struct packed_git *p;
  
        p = (last_found != (void *)1) ? last_found :
 -                                      get_packed_git(the_repository);
 +                                      get_all_packs(the_repository);
  
        while (p) {
                if ((!p->pack_local || p->pack_keep ||
                        return 1;
                }
                if (p == last_found)
 -                      p = get_packed_git(the_repository);
 +                      p = get_all_packs(the_repository);
                else
                        p = p->next;
                if (p == last_found)
@@@ -3014,12 -2939,12 +3016,12 @@@ static void loosen_unused_packed_object
        uint32_t i;
        struct object_id oid;
  
 -      for (p = get_packed_git(the_repository); p; p = p->next) {
 +      for (p = get_all_packs(the_repository); p; p = p->next) {
                if (!p->pack_local || p->pack_keep || p->pack_keep_in_core)
                        continue;
  
                if (open_pack_index(p))
 -                      die("cannot open pack index");
 +                      die(_("cannot open pack index"));
  
                for (i = 0; i < p->num_objects; i++) {
                        nth_packed_object_oid(&oid, p, i);
                            !has_sha1_pack_kept_or_nonlocal(&oid) &&
                            !loosened_object_can_be_discarded(&oid, p->mtime))
                                if (force_object_loose(&oid, p->mtime))
 -                                      die("unable to force loose object");
 +                                      die(_("unable to force loose object"));
                }
        }
  }
@@@ -3049,6 -2974,7 +3051,6 @@@ static int pack_options_allow_reuse(voi
  
  static int get_object_list_from_bitmap(struct rev_info *revs)
  {
 -      struct bitmap_index *bitmap_git;
        if (!(bitmap_git = prepare_bitmap_walk(revs)))
                return -1;
  
        }
  
        traverse_bitmap_commit_list(bitmap_git, &add_object_entry_from_bitmap);
 -      free_bitmap_index(bitmap_git);
        return 0;
  }
  
@@@ -3084,19 -3011,14 +3086,19 @@@ static void get_object_list(int ac, con
        struct rev_info revs;
        char line[1000];
        int flags = 0;
 +      int save_warning;
  
 -      init_revisions(&revs, NULL);
 +      repo_init_revisions(the_repository, &revs, NULL);
        save_commit_buffer = 0;
 +      revs.allow_exclude_promisor_objects_opt = 1;
        setup_revisions(ac, av, &revs, NULL);
  
        /* make sure shallows are read */
        is_repository_shallow(the_repository);
  
 +      save_warning = warn_on_object_refname_ambiguity;
 +      warn_on_object_refname_ambiguity = 0;
 +
        while (fgets(line, sizeof(line), stdin) != NULL) {
                int len = strlen(line);
                if (len && line[len - 1] == '\n')
                                use_bitmap_index = 0;
                                continue;
                        }
 -                      die("not a rev '%s'", line);
 +                      die(_("not a rev '%s'"), line);
                }
                if (handle_revision_arg(line, &revs, flags, REVARG_CANNOT_BE_FILENAME))
 -                      die("bad revision '%s'", line);
 +                      die(_("bad revision '%s'"), line);
        }
  
 +      warn_on_object_refname_ambiguity = save_warning;
 +
        if (use_bitmap_index && !get_object_list_from_bitmap(&revs))
                return;
  
                load_delta_islands();
  
        if (prepare_revision_walk(&revs))
 -              die("revision walk setup failed");
 +              die(_("revision walk setup failed"));
        mark_edges_uninteresting(&revs, show_edge);
  
        if (!fn_show_object)
                revs.ignore_missing_links = 1;
                if (add_unseen_recent_objects_to_traversal(&revs,
                                unpack_unreachable_expiration))
 -                      die("unable to add recent objects");
 +                      die(_("unable to add recent objects"));
                if (prepare_revision_walk(&revs))
 -                      die("revision walk setup failed");
 +                      die(_("revision walk setup failed"));
                traverse_commit_list(&revs, record_recent_commit,
                                     record_recent_object, NULL);
        }
@@@ -3169,7 -3089,7 +3171,7 @@@ static void add_extra_kept_packs(const 
        if (!names->nr)
                return;
  
 -      for (p = get_packed_git(the_repository); p; p = p->next) {
 +      for (p = get_all_packs(the_repository); p; p = p->next) {
                const char *name = basename(p->pack_name);
                int i;
  
@@@ -3193,9 -3113,6 +3195,9 @@@ static int option_parse_index_version(c
  {
        char *c;
        const char *val = arg;
 +
 +      BUG_ON_OPT_NEG(unset);
 +
        pack_idx_opts.version = strtoul(val, &c, 10);
        if (pack_idx_opts.version > 2)
                die(_("unsupported index version %s"), val);
@@@ -3224,6 -3141,7 +3226,6 @@@ static int option_parse_unpack_unreacha
  int cmd_pack_objects(int argc, const char **argv, const char *prefix)
  {
        int use_internal_rev_list = 0;
 -      int thin = 0;
        int shallow = 0;
        int all_progress_implied = 0;
        struct argv_array rp = ARGV_ARRAY_INIT;
                OPT_BOOL(0, "all-progress-implied",
                         &all_progress_implied,
                         N_("similar to --all-progress when progress meter is shown")),
 -              { OPTION_CALLBACK, 0, "index-version", NULL, N_("version[,offset]"),
 +              { OPTION_CALLBACK, 0, "index-version", NULL, N_("<version>[,<offset>]"),
                  N_("write the pack index file in the specified idx format version"),
 -                0, option_parse_index_version },
 +                PARSE_OPT_NONEG, option_parse_index_version },
                OPT_MAGNITUDE(0, "max-pack-size", &pack_size_limit,
                              N_("maximum size of each output pack file")),
                OPT_BOOL(0, "local", &local,
        if (DFS_NUM_STATES > (1 << OE_DFS_STATE_BITS))
                BUG("too many dfs states, increase OE_DFS_STATE_BITS");
  
 -      check_replace_refs = 0;
 +      read_replace_refs = 0;
  
        reset_pack_idx_option(&pack_idx_opts);
        git_config(git_pack_config, NULL);
        if (pack_compression_level == -1)
                pack_compression_level = Z_DEFAULT_COMPRESSION;
        else if (pack_compression_level < 0 || pack_compression_level > Z_BEST_COMPRESSION)
 -              die("bad pack compression level %d", pack_compression_level);
 +              die(_("bad pack compression level %d"), pack_compression_level);
  
        if (!delta_search_threads)      /* --threads=0 means autodetect */
                delta_search_threads = online_cpus();
  
 -#ifdef NO_PTHREADS
 -      if (delta_search_threads != 1)
 -              warning("no threads support, ignoring --threads");
 -#endif
 +      if (!HAVE_THREADS && delta_search_threads != 1)
 +              warning(_("no threads support, ignoring --threads"));
        if (!pack_to_stdout && !pack_size_limit)
                pack_size_limit = pack_size_limit_cfg;
        if (pack_to_stdout && pack_size_limit)
 -              die("--max-pack-size cannot be used to build a pack for transfer.");
 +              die(_("--max-pack-size cannot be used to build a pack for transfer"));
        if (pack_size_limit && pack_size_limit < 1024*1024) {
 -              warning("minimum pack size limit is 1 MiB");
 +              warning(_("minimum pack size limit is 1 MiB"));
                pack_size_limit = 1024*1024;
        }
  
        if (!pack_to_stdout && thin)
 -              die("--thin cannot be used to build an indexable pack.");
 +              die(_("--thin cannot be used to build an indexable pack"));
  
        if (keep_unreachable && unpack_unreachable)
 -              die("--keep-unreachable and --unpack-unreachable are incompatible.");
 +              die(_("--keep-unreachable and --unpack-unreachable are incompatible"));
        if (!rev_list_all || !rev_list_reflog || !rev_list_index)
                unpack_unreachable_expiration = 0;
  
        if (filter_options.choice) {
                if (!pack_to_stdout)
 -                      die("cannot use --filter without --stdout.");
 +                      die(_("cannot use --filter without --stdout"));
                use_bitmap_index = 0;
        }
  
        add_extra_kept_packs(&keep_pack_list);
        if (ignore_packed_keep_on_disk) {
                struct packed_git *p;
 -              for (p = get_packed_git(the_repository); p; p = p->next)
 +              for (p = get_all_packs(the_repository); p; p = p->next)
                        if (p->pack_local && p->pack_keep)
                                break;
                if (!p) /* no keep-able packs found */
                 * it also covers non-local objects
                 */
                struct packed_git *p;
 -              for (p = get_packed_git(the_repository); p; p = p->next) {
 +              for (p = get_all_packs(the_repository); p; p = p->next) {
                        if (!p->pack_local) {
                                have_non_local_packs = 1;
                                break;
                prepare_pack(window, depth);
        write_pack_file();
        if (progress)
 -              fprintf(stderr, "Total %"PRIu32" (delta %"PRIu32"),"
 -                      " reused %"PRIu32" (delta %"PRIu32")\n",
 -                      written, written_delta, reused, reused_delta);
 +              fprintf_ln(stderr,
 +                         _("Total %"PRIu32" (delta %"PRIu32"),"
 +                           " reused %"PRIu32" (delta %"PRIu32")"),
 +                         written, written_delta, reused, reused_delta);
        return 0;
  }
diff --combined git-compat-util.h
index f16058182f76bcb9fe50e00a239993de0fb28b12,b904af3f227a17ab2859d9d59e0b28a649c3109a..09b0102cae8c8c0e39dc239003ca599a896730cf
  #define _SGI_SOURCE 1
  
  #if defined(WIN32) && !defined(__CYGWIN__) /* Both MinGW and MSVC */
 -# if defined (_MSC_VER) && !defined(_WIN32_WINNT)
 -#  define _WIN32_WINNT 0x0502
 +# if !defined(_WIN32_WINNT)
 +#  define _WIN32_WINNT 0x0600
  # endif
  #define WIN32_LEAN_AND_MEAN  /* stops windows.h including winsock.h */
  #include <winsock2.h>
  #include <regex.h>
  #include <utime.h>
  #include <syslog.h>
 -#ifndef NO_SYS_POLL_H
 +#if !defined(NO_POLL_H)
 +#include <poll.h>
 +#elif !defined(NO_SYS_POLL_H)
  #include <sys/poll.h>
  #else
 +/* Pull the compat stuff */
  #include <poll.h>
  #endif
  #ifdef HAVE_BSD_SYSCTL
  #endif
  #ifdef NO_INTPTR_T
  /*
 - * On I16LP32, ILP32 and LP64 "long" is the save bet, however
 + * On I16LP32, ILP32 and LP64 "long" is the safe bet, however
   * on LLP86, IL33LLP64 and P64 it needs to be "long long",
   * while on IP16 and IP16L32 it is "int" (resp. "short")
   * Size needs to match (or exceed) 'sizeof(void *)'.
@@@ -345,14 -342,6 +345,14 @@@ typedef uintmax_t timestamp_t
  #define _PATH_DEFPATH "/usr/local/bin:/usr/bin:/bin"
  #endif
  
 +#ifndef platform_core_config
 +static inline int noop_core_config(const char *var, const char *value, void *cb)
 +{
 +      return 0;
 +}
 +#define platform_core_config noop_core_config
 +#endif
 +
  #ifndef has_dos_drive_prefix
  static inline int git_has_dos_drive_prefix(const char *path)
  {
@@@ -393,10 -382,6 +393,10 @@@ static inline char *git_find_last_dir_s
  #define find_last_dir_sep git_find_last_dir_sep
  #endif
  
 +#ifndef query_user_email
 +#define query_user_email() NULL
 +#endif
 +
  #if defined(__HP_cc) && (__HP_cc >= 61000)
  #define NORETURN __attribute__((noreturn))
  #define NORETURN_PTR
  #define LAST_ARG_MUST_BE_NULL
  #endif
  
 +#define MAYBE_UNUSED __attribute__((__unused__))
 +
  #include "compat/bswap.h"
  
  #include "wildmatch.h"
@@@ -861,6 -844,7 +861,7 @@@ extern FILE *fopen_or_warn(const char *
  #define FREE_AND_NULL(p) do { free(p); (p) = NULL; } while (0)
  
  #define ALLOC_ARRAY(x, alloc) (x) = xmalloc(st_mult(sizeof(*(x)), (alloc)))
+ #define CALLOC_ARRAY(x, alloc) (x) = xcalloc((alloc), sizeof(*(x)));
  #define REALLOC_ARRAY(x, alloc) (x) = xrealloc((x), st_mult(sizeof(*(x)), (alloc)))
  
  #define COPY_ARRAY(dst, src, n) copy_array((dst), (src), (n), sizeof(*(dst)) + \
@@@ -1256,10 -1240,4 +1257,10 @@@ extern void unleak_memory(const void *p
  #define UNLEAK(var) do {} while (0)
  #endif
  
 +/*
 + * This include must come after system headers, since it introduces macros that
 + * replace system names.
 + */
 +#include "banned.h"
 +
  #endif
diff --combined pack-objects.h
index feb6a6a05edba0772f4cfd8b4529bce9d3858751,a89f76e5b1f9e4b1a9af4b6e8cf8fd01ee11e808..dc869f26c2b753593a875f275d896933b7723b31
@@@ -2,8 -2,6 +2,8 @@@
  #define PACK_OBJECTS_H
  
  #include "object-store.h"
 +#include "thread-utils.h"
 +#include "pack.h"
  
  #define DEFAULT_DELTA_CACHE_SIZE (256 * 1024 * 1024)
  
@@@ -16,7 -14,7 +16,7 @@@
   * above this limit. Don't lower it too much.
   */
  #define OE_SIZE_BITS          31
 -#define OE_DELTA_SIZE_BITS    20
 +#define OE_DELTA_SIZE_BITS    23
  
  /*
   * State flags for depth-first search used for analyzing delta cycles.
@@@ -96,12 -94,11 +96,12 @@@ struct object_entry 
                                     */
        unsigned delta_size_:OE_DELTA_SIZE_BITS; /* delta data size (uncompressed) */
        unsigned delta_size_valid:1;
 +      unsigned char in_pack_header_size;
        unsigned in_pack_idx:OE_IN_PACK_BITS;   /* already in pack */
        unsigned z_delta_size:OE_Z_DELTA_BITS;
        unsigned type_valid:1;
 -      unsigned type_:TYPE_BITS;
        unsigned no_try_delta:1;
 +      unsigned type_:TYPE_BITS;
        unsigned in_pack_type:TYPE_BITS; /* could be delta */
  
        unsigned preferred_base:1; /*
        unsigned tagged:1; /* near the very tip of refs */
        unsigned filled:1; /* assigned write-order */
        unsigned dfs_state:OE_DFS_STATE_BITS;
 -      unsigned char in_pack_header_size;
        unsigned depth:OE_DEPTH_BITS;
 +      unsigned ext_base:1; /* delta_idx points outside packlist */
  
        /*
         * pahole results on 64-bit linux (gcc and clang)
         *
 -       *   size: 80, bit_padding: 20 bits, holes: 8 bits
 +       *   size: 80, bit_padding: 9 bits
         *
         * and on 32-bit (gcc)
         *
 -       *   size: 76, bit_padding: 20 bits, holes: 8 bits
 +       *   size: 76, bit_padding: 9 bits
         */
  };
  
@@@ -134,7 -131,6 +134,7 @@@ struct packing_data 
        uint32_t index_size;
  
        unsigned int *in_pack_pos;
 +      unsigned long *delta_size;
  
        /*
         * Only one of these can be non-NULL and they have different
        struct packed_git **in_pack_by_idx;
        struct packed_git **in_pack;
  
 +      pthread_mutex_t lock;
 +
 +      /*
 +       * This list contains entries for bases which we know the other side
 +       * has (e.g., via reachability bitmaps), but which aren't in our
 +       * "objects" list.
 +       */
 +      struct object_entry *ext_bases;
 +      uint32_t nr_ext, alloc_ext;
 +
        uintmax_t oe_size_limit;
 +      uintmax_t oe_delta_size_limit;
  
        /* delta islands */
        unsigned int *tree_depth;
  };
  
  void prepare_packing_data(struct packing_data *pdata);
 +
 +static inline void packing_data_lock(struct packing_data *pdata)
 +{
 +      pthread_mutex_lock(&pdata->lock);
 +}
 +static inline void packing_data_unlock(struct packing_data *pdata)
 +{
 +      pthread_mutex_unlock(&pdata->lock);
 +}
 +
  struct object_entry *packlist_alloc(struct packing_data *pdata,
                                    const unsigned char *sha1,
                                    uint32_t index_pos);
@@@ -257,12 -232,9 +257,12 @@@ static inline struct object_entry *oe_d
                const struct packing_data *pack,
                const struct object_entry *e)
  {
 -      if (e->delta_idx)
 +      if (!e->delta_idx)
 +              return NULL;
 +      if (e->ext_base)
 +              return &pack->ext_bases[e->delta_idx - 1];
 +      else
                return &pack->objects[e->delta_idx - 1];
 -      return NULL;
  }
  
  static inline void oe_set_delta(struct packing_data *pack,
                e->delta_idx = 0;
  }
  
 +void oe_set_delta_ext(struct packing_data *pack,
 +                    struct object_entry *e,
 +                    const unsigned char *sha1);
 +
  static inline struct object_entry *oe_delta_child(
                const struct packing_data *pack,
                const struct object_entry *e)
@@@ -369,34 -337,18 +369,34 @@@ static inline unsigned long oe_delta_si
  {
        if (e->delta_size_valid)
                return e->delta_size_;
 -      return oe_size(pack, e);
 +
 +      /*
 +       * pack->delta_size[] can't be NULL because oe_set_delta_size()
 +       * must have been called when a new delta is saved with
 +       * oe_set_delta().
 +       * If oe_delta() returns NULL (i.e. default state, which means
 +       * delta_size_valid is also false), then the caller must never
 +       * call oe_delta_size().
 +       */
 +      return pack->delta_size[e - pack->objects];
  }
  
  static inline void oe_set_delta_size(struct packing_data *pack,
                                     struct object_entry *e,
                                     unsigned long size)
  {
 -      e->delta_size_ = size;
 -      e->delta_size_valid = e->delta_size_ == size;
 -      if (!e->delta_size_valid && size != oe_size(pack, e))
 -              BUG("this can only happen in check_object() "
 -                  "where delta size is the same as entry size");
 +      if (size < pack->oe_delta_size_limit) {
 +              e->delta_size_ = size;
 +              e->delta_size_valid = 1;
 +      } else {
 +              packing_data_lock(pack);
 +              if (!pack->delta_size)
 +                      ALLOC_ARRAY(pack->delta_size, pack->nr_alloc);
 +              packing_data_unlock(pack);
 +
 +              pack->delta_size[e - pack->objects] = size;
 +              e->delta_size_valid = 0;
 +      }
  }
  
  static inline unsigned int oe_tree_depth(struct packing_data *pack,
@@@ -412,7 -364,7 +412,7 @@@ static inline void oe_set_tree_depth(st
                                     unsigned int tree_depth)
  {
        if (!pack->tree_depth)
-               ALLOC_ARRAY(pack->tree_depth, pack->nr_objects);
+               CALLOC_ARRAY(pack->tree_depth, pack->nr_alloc);
        pack->tree_depth[e - pack->objects] = tree_depth;
  }
  
@@@ -429,7 -381,7 +429,7 @@@ static inline void oe_set_layer(struct 
                                unsigned char layer)
  {
        if (!pack->layer)
-               ALLOC_ARRAY(pack->layer, pack->nr_objects);
+               CALLOC_ARRAY(pack->layer, pack->nr_alloc);
        pack->layer[e - pack->objects] = layer;
  }