#include "packfile.h"
#include "object-store.h"
#include "dir.h"
+#include "midx.h"
#define IN_PACK(obj) oe_in_pack(&to_pack, obj)
#define SIZE(obj) oe_size(&to_pack, obj)
#define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj)
#define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj)
#define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val)
+ #define SET_DELTA_EXT(obj, oid) oe_set_delta_ext(&to_pack, obj, oid)
#define SET_DELTA_SIZE(obj, val) oe_set_delta_size(&to_pack, obj, val)
#define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val)
#define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val)
static struct pack_idx_entry **written_list;
static uint32_t nr_result, nr_written, nr_seen;
+ static struct bitmap_index *bitmap_git;
static int non_empty;
static int reuse_delta = 1, reuse_object = 1;
static int depth = 50;
static int delta_search_threads;
static int pack_to_stdout;
+ static int thin;
static int num_preferred_base;
static struct progress *progress_state;
{
int want;
struct list_head *pos;
+ struct multi_pack_index *m;
if (!exclude && local && has_loose_object_nonlocal(oid))
return 0;
if (want != -1)
return want;
}
+
+ for (m = get_multi_pack_index(the_repository); m; m = m->next) {
+ struct pack_entry e;
+ if (fill_midx_entry(oid, &e, m)) {
+ struct packed_git *p = e.p;
+ off_t offset;
+
+ if (p == *found_pack)
+ offset = *found_offset;
+ else
+ offset = find_pack_entry_one(oid->hash, p);
+
+ if (offset) {
+ if (!*found_pack) {
+ if (!is_pack_valid(p))
+ continue;
+ *found_offset = offset;
+ *found_pack = p;
+ }
+ want = want_found_object(exclude, p);
+ if (want != -1)
+ return want;
+ }
+ }
+ }
+
list_for_each(pos, get_packed_git_mru(the_repository)) {
struct packed_git *p = list_entry(pos, struct packed_git, mru);
off_t offset;
break;
}
- if (base_ref && (base_entry = packlist_find(&to_pack, base_ref, NULL))) {
+ if (base_ref && (
+ (base_entry = packlist_find(&to_pack, base_ref, NULL)) ||
+ (thin &&
+ bitmap_has_sha1_in_uninteresting(bitmap_git, base_ref)))) {
/*
* If base_ref was set above that means we wish to
- * reuse delta data, and we even found that base
- * in the list of objects we want to pack. Goodie!
+ * reuse delta data, and either we found that object in
+ * the list of objects we want to pack, or it's one we
+ * know the receiver has.
*
* Depth value does not matter - find_deltas() will
* never consider reused delta as the base object to
*/
oe_set_type(entry, entry->in_pack_type);
SET_SIZE(entry, in_pack_size); /* delta size */
- SET_DELTA(entry, base_entry);
SET_DELTA_SIZE(entry, in_pack_size);
- entry->delta_sibling_idx = base_entry->delta_child_idx;
- SET_DELTA_CHILD(base_entry, entry);
+
+ if (base_entry) {
+ SET_DELTA(entry, base_entry);
+ entry->delta_sibling_idx = base_entry->delta_child_idx;
+ SET_DELTA_CHILD(base_entry, entry);
+ } else {
+ SET_DELTA_EXT(entry, base_ref);
+ }
+
unuse_pack(&w_curs);
return;
}
delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size);
if (!delta_buf)
return 0;
- if (delta_size >= (1U << OE_DELTA_SIZE_BITS)) {
- free(delta_buf);
- return 0;
- }
if (DELTA(trg_entry)) {
/* Prefer only shallower same-sized deltas. */
pthread_mutex_init(&cache_mutex, NULL);
pthread_mutex_init(&progress_mutex, NULL);
pthread_cond_init(&progress_cond, NULL);
+ pthread_mutex_init(&to_pack.lock, NULL);
old_try_to_free_routine = set_try_to_free_routine(try_to_free_from_threads);
}
memset(&in_pack, 0, sizeof(in_pack));
- for (p = get_packed_git(the_repository); p; p = p->next) {
+ for (p = get_all_packs(the_repository); p; p = p->next) {
struct object_id oid;
struct object *o;
struct packed_git *p;
p = (last_found != (void *)1) ? last_found :
- get_packed_git(the_repository);
+ get_all_packs(the_repository);
while (p) {
if ((!p->pack_local || p->pack_keep ||
return 1;
}
if (p == last_found)
- p = get_packed_git(the_repository);
+ p = get_all_packs(the_repository);
else
p = p->next;
if (p == last_found)
uint32_t i;
struct object_id oid;
- for (p = get_packed_git(the_repository); p; p = p->next) {
+ for (p = get_all_packs(the_repository); p; p = p->next) {
if (!p->pack_local || p->pack_keep || p->pack_keep_in_core)
continue;
static int get_object_list_from_bitmap(struct rev_info *revs)
{
- struct bitmap_index *bitmap_git;
if (!(bitmap_git = prepare_bitmap_walk(revs)))
return -1;
}
traverse_bitmap_commit_list(bitmap_git, &add_object_entry_from_bitmap);
- free_bitmap_index(bitmap_git);
return 0;
}
if (!names->nr)
return;
- for (p = get_packed_git(the_repository); p; p = p->next) {
+ for (p = get_all_packs(the_repository); p; p = p->next) {
const char *name = basename(p->pack_name);
int i;
int cmd_pack_objects(int argc, const char **argv, const char *prefix)
{
int use_internal_rev_list = 0;
- int thin = 0;
int shallow = 0;
int all_progress_implied = 0;
struct argv_array rp = ARGV_ARRAY_INIT;
add_extra_kept_packs(&keep_pack_list);
if (ignore_packed_keep_on_disk) {
struct packed_git *p;
- for (p = get_packed_git(the_repository); p; p = p->next)
+ for (p = get_all_packs(the_repository); p; p = p->next)
if (p->pack_local && p->pack_keep)
break;
if (!p) /* no keep-able packs found */
* it also covers non-local objects
*/
struct packed_git *p;
- for (p = get_packed_git(the_repository); p; p = p->next) {
+ for (p = get_all_packs(the_repository); p; p = p->next) {
if (!p->pack_local) {
have_non_local_packs = 1;
break;
/* Bitmap result of the last performed walk */
struct bitmap *result;
+ /* "have" bitmap from the last performed walk */
+ struct bitmap *haves;
+
/* Version of the bitmap index */
unsigned int version;
assert(!bitmap_git->map && !bitmap_git->loaded);
- for (p = get_packed_git(the_repository); p; p = p->next) {
+ for (p = get_all_packs(the_repository); p; p = p->next) {
if (open_pack_bitmap_1(bitmap_git, p) == 0)
ret = 0;
}
bitmap_and_not(wants_bitmap, haves_bitmap);
bitmap_git->result = wants_bitmap;
+ bitmap_git->haves = haves_bitmap;
- bitmap_free(haves_bitmap);
return bitmap_git;
cleanup:
free(b->ext_index.objects);
free(b->ext_index.hashes);
bitmap_free(b->result);
+ bitmap_free(b->haves);
free(b);
}
+
+ int bitmap_has_sha1_in_uninteresting(struct bitmap_index *bitmap_git,
+ const unsigned char *sha1)
+ {
+ int pos;
+
+ if (!bitmap_git)
+ return 0; /* no bitmap loaded */
+ if (!bitmap_git->result)
+ BUG("failed to perform bitmap walk before querying");
+ if (!bitmap_git->haves)
+ return 0; /* walk had no "haves" */
+
+ pos = bitmap_position_packfile(bitmap_git, sha1);
+ if (pos < 0)
+ return 0;
+
+ return bitmap_get(bitmap_git->haves, pos);
+ }
* (i.e. in_pack_idx also zero) should return NULL.
*/
mapping[cnt++] = NULL;
- for (p = get_packed_git(the_repository); p; p = p->next, cnt++) {
+ for (p = get_all_packs(the_repository); p; p = p->next, cnt++) {
if (cnt == nr) {
free(mapping);
return;
pdata->oe_size_limit = git_env_ulong("GIT_TEST_OE_SIZE",
1U << OE_SIZE_BITS);
+ pdata->oe_delta_size_limit = git_env_ulong("GIT_TEST_OE_DELTA_SIZE",
+ 1UL << OE_DELTA_SIZE_BITS);
}
struct object_entry *packlist_alloc(struct packing_data *pdata,
if (!pdata->in_pack_by_idx)
REALLOC_ARRAY(pdata->in_pack, pdata->nr_alloc);
+ if (pdata->delta_size)
+ REALLOC_ARRAY(pdata->delta_size, pdata->nr_alloc);
}
new_entry = pdata->objects + pdata->nr_objects++;
return new_entry;
}
+
+ void oe_set_delta_ext(struct packing_data *pdata,
+ struct object_entry *delta,
+ const unsigned char *sha1)
+ {
+ struct object_entry *base;
+
+ ALLOC_GROW(pdata->ext_bases, pdata->nr_ext + 1, pdata->alloc_ext);
+ base = &pdata->ext_bases[pdata->nr_ext++];
+ memset(base, 0, sizeof(*base));
+ hashcpy(base->idx.oid.hash, sha1);
+
+ /* These flags mark that we are not part of the actual pack output. */
+ base->preferred_base = 1;
+ base->filled = 1;
+
+ delta->ext_base = 1;
+ delta->delta_idx = base - pdata->ext_bases + 1;
+ }
#define PACK_OBJECTS_H
#include "object-store.h"
+#include "thread-utils.h"
#include "pack.h"
#define DEFAULT_DELTA_CACHE_SIZE (256 * 1024 * 1024)
* above this limit. Don't lower it too much.
*/
#define OE_SIZE_BITS 31
-#define OE_DELTA_SIZE_BITS 20
+#define OE_DELTA_SIZE_BITS 23
/*
* State flags for depth-first search used for analyzing delta cycles.
*/
unsigned delta_size_:OE_DELTA_SIZE_BITS; /* delta data size (uncompressed) */
unsigned delta_size_valid:1;
+ unsigned char in_pack_header_size;
unsigned in_pack_idx:OE_IN_PACK_BITS; /* already in pack */
unsigned z_delta_size:OE_Z_DELTA_BITS;
unsigned type_valid:1;
- unsigned type_:TYPE_BITS;
unsigned no_try_delta:1;
+ unsigned type_:TYPE_BITS;
unsigned in_pack_type:TYPE_BITS; /* could be delta */
unsigned preferred_base:1; /*
* we do not pack this, but is available
unsigned tagged:1; /* near the very tip of refs */
unsigned filled:1; /* assigned write-order */
unsigned dfs_state:OE_DFS_STATE_BITS;
- unsigned char in_pack_header_size;
unsigned depth:OE_DEPTH_BITS;
+ unsigned ext_base:1; /* delta_idx points outside packlist */
/*
* pahole results on 64-bit linux (gcc and clang)
*
- * size: 80, bit_padding: 20 bits, holes: 8 bits
+ * size: 80, bit_padding: 9 bits
*
* and on 32-bit (gcc)
*
- * size: 76, bit_padding: 20 bits, holes: 8 bits
+ * size: 76, bit_padding: 9 bits
*/
};
uint32_t index_size;
unsigned int *in_pack_pos;
+ unsigned long *delta_size;
/*
* Only one of these can be non-NULL and they have different
struct packed_git **in_pack_by_idx;
struct packed_git **in_pack;
+#ifndef NO_PTHREADS
+ pthread_mutex_t lock;
+#endif
+
+ /*
+ * This list contains entries for bases which we know the other side
+ * has (e.g., via reachability bitmaps), but which aren't in our
+ * "objects" list.
+ */
+ struct object_entry *ext_bases;
+ uint32_t nr_ext, alloc_ext;
+
uintmax_t oe_size_limit;
+ uintmax_t oe_delta_size_limit;
};
void prepare_packing_data(struct packing_data *pdata);
+
+static inline void packing_data_lock(struct packing_data *pdata)
+{
+#ifndef NO_PTHREADS
+ pthread_mutex_lock(&pdata->lock);
+#endif
+}
+static inline void packing_data_unlock(struct packing_data *pdata)
+{
+#ifndef NO_PTHREADS
+ pthread_mutex_unlock(&pdata->lock);
+#endif
+}
+
struct object_entry *packlist_alloc(struct packing_data *pdata,
const unsigned char *sha1,
uint32_t index_pos);
const struct packing_data *pack,
const struct object_entry *e)
{
- if (e->delta_idx)
+ if (!e->delta_idx)
+ return NULL;
+ if (e->ext_base)
+ return &pack->ext_bases[e->delta_idx - 1];
+ else
return &pack->objects[e->delta_idx - 1];
- return NULL;
}
static inline void oe_set_delta(struct packing_data *pack,
e->delta_idx = 0;
}
+ void oe_set_delta_ext(struct packing_data *pack,
+ struct object_entry *e,
+ const unsigned char *sha1);
+
static inline struct object_entry *oe_delta_child(
const struct packing_data *pack,
const struct object_entry *e)
{
if (e->delta_size_valid)
return e->delta_size_;
- return oe_size(pack, e);
+
+ /*
+ * pack->detla_size[] can't be NULL because oe_set_delta_size()
+ * must have been called when a new delta is saved with
+ * oe_set_delta().
+ * If oe_delta() returns NULL (i.e. default state, which means
+ * delta_size_valid is also false), then the caller must never
+ * call oe_delta_size().
+ */
+ return pack->delta_size[e - pack->objects];
}
static inline void oe_set_delta_size(struct packing_data *pack,
struct object_entry *e,
unsigned long size)
{
- e->delta_size_ = size;
- e->delta_size_valid = e->delta_size_ == size;
- if (!e->delta_size_valid && size != oe_size(pack, e))
- BUG("this can only happen in check_object() "
- "where delta size is the same as entry size");
+ if (size < pack->oe_delta_size_limit) {
+ e->delta_size_ = size;
+ e->delta_size_valid = 1;
+ } else {
+ packing_data_lock(pack);
+ if (!pack->delta_size)
+ ALLOC_ARRAY(pack->delta_size, pack->nr_alloc);
+ packing_data_unlock(pack);
+
+ pack->delta_size[e - pack->objects] = size;
+ e->delta_size_valid = 0;
+ }
}
#endif