#include "diff.h"
#include "revision.h"
#include "list-objects.h"
+#include "list-objects-filter.h"
+#include "list-objects-filter-options.h"
#include "pack-objects.h"
#include "progress.h"
#include "refs.h"
static int write_bitmap_index;
static uint16_t write_bitmap_options;
+static int exclude_promisor_objects;
+
static unsigned long delta_cache_size = 0;
static unsigned long max_delta_cache_size = 256 * 1024 * 1024;
static unsigned long cache_max_small_delta_size = 1000;
static unsigned long window_memory_limit = 0;
+static struct list_objects_filter_options filter_options;
+
+enum missing_action {
+ MA_ERROR = 0, /* fail if any missing objects are encountered */
+ MA_ALLOW_ANY, /* silently allow ALL missing objects */
+ MA_ALLOW_PROMISOR, /* silently allow all missing PROMISOR objects */
+};
+static enum missing_action arg_missing_action;
+static show_object_fn fn_show_object;
+
/*
* stats
*/
return stream.total_out;
}
-static unsigned long write_large_blob_data(struct git_istream *st, struct sha1file *f,
- const unsigned char *sha1)
+static unsigned long write_large_blob_data(struct git_istream *st, struct hashfile *f,
+ const struct object_id *oid)
{
git_zstream stream;
unsigned char ibuf[1024 * 16];
int zret = Z_OK;
readlen = read_istream(st, ibuf, sizeof(ibuf));
if (readlen == -1)
- die(_("unable to read %s"), sha1_to_hex(sha1));
+ die(_("unable to read %s"), oid_to_hex(oid));
stream.next_in = ibuf;
stream.avail_in = readlen;
stream.next_out = obuf;
stream.avail_out = sizeof(obuf);
zret = git_deflate(&stream, readlen ? 0 : Z_FINISH);
- sha1write(f, obuf, stream.next_out - obuf);
+ hashwrite(f, obuf, stream.next_out - obuf);
olen += stream.next_out - obuf;
}
if (stream.avail_in)
stream.total_in == len) ? 0 : -1;
}
-static void copy_pack_data(struct sha1file *f,
+static void copy_pack_data(struct hashfile *f,
struct packed_git *p,
struct pack_window **w_curs,
off_t offset,
in = use_pack(p, w_curs, offset, &avail);
if (avail > len)
avail = (unsigned long)len;
- sha1write(f, in, avail);
+ hashwrite(f, in, avail);
offset += avail;
len -= avail;
}
}
/* Return 0 if we will bust the pack-size limit */
-static unsigned long write_no_reuse_object(struct sha1file *f, struct object_entry *entry,
+static unsigned long write_no_reuse_object(struct hashfile *f, struct object_entry *entry,
unsigned long limit, int usable_delta)
{
unsigned long size, datalen;
free(buf);
return 0;
}
- sha1write(f, header, hdrlen);
- sha1write(f, dheader + pos, sizeof(dheader) - pos);
+ hashwrite(f, header, hdrlen);
+ hashwrite(f, dheader + pos, sizeof(dheader) - pos);
hdrlen += sizeof(dheader) - pos;
} else if (type == OBJ_REF_DELTA) {
/*
free(buf);
return 0;
}
- sha1write(f, header, hdrlen);
- sha1write(f, entry->delta->idx.oid.hash, 20);
+ hashwrite(f, header, hdrlen);
+ hashwrite(f, entry->delta->idx.oid.hash, 20);
hdrlen += 20;
} else {
if (limit && hdrlen + datalen + 20 >= limit) {
free(buf);
return 0;
}
- sha1write(f, header, hdrlen);
+ hashwrite(f, header, hdrlen);
}
if (st) {
- datalen = write_large_blob_data(st, f, entry->idx.oid.hash);
+ datalen = write_large_blob_data(st, f, &entry->idx.oid);
close_istream(st);
} else {
- sha1write(f, buf, datalen);
+ hashwrite(f, buf, datalen);
free(buf);
}
}
/* Return 0 if we will bust the pack-size limit */
-static off_t write_reuse_object(struct sha1file *f, struct object_entry *entry,
+static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,
unsigned long limit, int usable_delta)
{
struct packed_git *p = entry->in_pack;
unuse_pack(&w_curs);
return 0;
}
- sha1write(f, header, hdrlen);
- sha1write(f, dheader + pos, sizeof(dheader) - pos);
+ hashwrite(f, header, hdrlen);
+ hashwrite(f, dheader + pos, sizeof(dheader) - pos);
hdrlen += sizeof(dheader) - pos;
reused_delta++;
} else if (type == OBJ_REF_DELTA) {
unuse_pack(&w_curs);
return 0;
}
- sha1write(f, header, hdrlen);
- sha1write(f, entry->delta->idx.oid.hash, 20);
+ hashwrite(f, header, hdrlen);
+ hashwrite(f, entry->delta->idx.oid.hash, 20);
hdrlen += 20;
reused_delta++;
} else {
unuse_pack(&w_curs);
return 0;
}
- sha1write(f, header, hdrlen);
+ hashwrite(f, header, hdrlen);
}
copy_pack_data(f, p, &w_curs, offset, datalen);
unuse_pack(&w_curs);
}
/* Return 0 if we will bust the pack-size limit */
-static off_t write_object(struct sha1file *f,
+static off_t write_object(struct hashfile *f,
struct object_entry *entry,
off_t write_offset)
{
WRITE_ONE_RECURSIVE = 2 /* already scheduled to be written */
};
-static enum write_one_status write_one(struct sha1file *f,
+static enum write_one_status write_one(struct hashfile *f,
struct object_entry *e,
off_t *offset)
{
static int mark_tagged(const char *path, const struct object_id *oid, int flag,
void *cb_data)
{
- unsigned char peeled[20];
+ struct object_id peeled;
struct object_entry *entry = packlist_find(&to_pack, oid->hash, NULL);
if (entry)
entry->tagged = 1;
- if (!peel_ref(path, peeled)) {
- entry = packlist_find(&to_pack, peeled, NULL);
+ if (!peel_ref(path, &peeled)) {
+ entry = packlist_find(&to_pack, peeled.hash, NULL);
if (entry)
entry->tagged = 1;
}
return wo;
}
-static off_t write_reused_pack(struct sha1file *f)
+static off_t write_reused_pack(struct hashfile *f)
{
unsigned char buffer[8192];
off_t to_write, total;
if (read_pack > to_write)
read_pack = to_write;
- sha1write(f, buffer, read_pack);
+ hashwrite(f, buffer, read_pack);
to_write -= read_pack;
/*
static void write_pack_file(void)
{
uint32_t i = 0, j;
- struct sha1file *f;
+ struct hashfile *f;
off_t offset;
uint32_t nr_remaining = nr_result;
time_t last_mtime = 0;
write_order = compute_write_order();
do {
- unsigned char sha1[20];
+ struct object_id oid;
char *pack_tmp_name = NULL;
if (pack_to_stdout)
- f = sha1fd_throughput(1, "<stdout>", progress_state);
+ f = hashfd_throughput(1, "<stdout>", progress_state);
else
f = create_tmp_packfile(&pack_tmp_name);
* If so, rewrite it like in fast-import
*/
if (pack_to_stdout) {
- sha1close(f, sha1, CSUM_CLOSE);
+ finalize_hashfile(f, oid.hash, CSUM_CLOSE);
} else if (nr_written == nr_remaining) {
- sha1close(f, sha1, CSUM_FSYNC);
+ finalize_hashfile(f, oid.hash, CSUM_FSYNC);
} else {
- int fd = sha1close(f, sha1, 0);
- fixup_pack_header_footer(fd, sha1, pack_tmp_name,
- nr_written, sha1, offset);
+ int fd = finalize_hashfile(f, oid.hash, 0);
+ fixup_pack_header_footer(fd, oid.hash, pack_tmp_name,
+ nr_written, oid.hash, offset);
close(fd);
if (write_bitmap_index) {
warning(_(no_split_warning));
strbuf_addf(&tmpname, "%s-", base_name);
if (write_bitmap_index) {
- bitmap_writer_set_checksum(sha1);
+ bitmap_writer_set_checksum(oid.hash);
bitmap_writer_build_type_index(written_list, nr_written);
}
finish_tmp_packfile(&tmpname, pack_tmp_name,
written_list, nr_written,
- &pack_idx_opts, sha1);
+ &pack_idx_opts, oid.hash);
if (write_bitmap_index) {
- strbuf_addf(&tmpname, "%s.bitmap", sha1_to_hex(sha1));
+ strbuf_addf(&tmpname, "%s.bitmap", oid_to_hex(&oid));
stop_progress(&progress_state);
strbuf_release(&tmpname);
free(pack_tmp_name);
- puts(sha1_to_hex(sha1));
+ puts(oid_to_hex(&oid));
}
/* mark written objects as written to previous pack */
* found the item, since that saves us from having to look it up again a
* few lines later when we want to add the new entry.
*/
-static int have_duplicate_entry(const unsigned char *sha1,
+static int have_duplicate_entry(const struct object_id *oid,
int exclude,
uint32_t *index_pos)
{
struct object_entry *entry;
- entry = packlist_find(&to_pack, sha1, index_pos);
+ entry = packlist_find(&to_pack, oid->hash, index_pos);
if (!entry)
return 0;
* function finds if there is any pack that has the object and returns the pack
* and its offset in these variables.
*/
-static int want_object_in_pack(const unsigned char *sha1,
+static int want_object_in_pack(const struct object_id *oid,
int exclude,
struct packed_git **found_pack,
off_t *found_offset)
int want;
struct list_head *pos;
- if (!exclude && local && has_loose_object_nonlocal(sha1))
+ if (!exclude && local && has_loose_object_nonlocal(oid->hash))
return 0;
/*
if (p == *found_pack)
offset = *found_offset;
else
- offset = find_pack_entry_one(sha1, p);
+ offset = find_pack_entry_one(oid->hash, p);
if (offset) {
if (!*found_pack) {
return 1;
}
-static void create_object_entry(const unsigned char *sha1,
+static void create_object_entry(const struct object_id *oid,
enum object_type type,
uint32_t hash,
int exclude,
{
struct object_entry *entry;
- entry = packlist_alloc(&to_pack, sha1, index_pos);
+ entry = packlist_alloc(&to_pack, oid->hash, index_pos);
entry->hash = hash;
if (type)
entry->type = type;
"disabling bitmap writing, as some objects are not being packed"
);
-static int add_object_entry(const unsigned char *sha1, enum object_type type,
+static int add_object_entry(const struct object_id *oid, enum object_type type,
const char *name, int exclude)
{
struct packed_git *found_pack = NULL;
off_t found_offset = 0;
uint32_t index_pos;
- if (have_duplicate_entry(sha1, exclude, &index_pos))
+ if (have_duplicate_entry(oid, exclude, &index_pos))
return 0;
- if (!want_object_in_pack(sha1, exclude, &found_pack, &found_offset)) {
+ if (!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) {
/* The pack is missing an object, so it will not have closure */
if (write_bitmap_index) {
warning(_(no_closure_warning));
return 0;
}
- create_object_entry(sha1, type, pack_name_hash(name),
+ create_object_entry(oid, type, pack_name_hash(name),
exclude, name && no_try_delta(name),
index_pos, found_pack, found_offset);
return 1;
}
-static int add_object_entry_from_bitmap(const unsigned char *sha1,
+static int add_object_entry_from_bitmap(const struct object_id *oid,
enum object_type type,
int flags, uint32_t name_hash,
struct packed_git *pack, off_t offset)
{
uint32_t index_pos;
- if (have_duplicate_entry(sha1, 0, &index_pos))
+ if (have_duplicate_entry(oid, 0, &index_pos))
return 0;
- if (!want_object_in_pack(sha1, 0, &pack, &offset))
+ if (!want_object_in_pack(oid, 0, &pack, &offset))
return 0;
- create_object_entry(sha1, type, name_hash, 0, 0, index_pos, pack, offset);
+ create_object_entry(oid, type, name_hash, 0, 0, index_pos, pack, offset);
display_progress(progress_state, nr_result);
return 1;
}
struct pbase_tree_cache {
- unsigned char sha1[20];
+ struct object_id oid;
int ref;
int temporary;
void *tree_data;
};
static struct pbase_tree_cache *(pbase_tree_cache[256]);
-static int pbase_tree_cache_ix(const unsigned char *sha1)
+static int pbase_tree_cache_ix(const struct object_id *oid)
{
- return sha1[0] % ARRAY_SIZE(pbase_tree_cache);
+ return oid->hash[0] % ARRAY_SIZE(pbase_tree_cache);
}
static int pbase_tree_cache_ix_incr(int ix)
{
struct pbase_tree_cache pcache;
} *pbase_tree;
-static struct pbase_tree_cache *pbase_tree_get(const unsigned char *sha1)
+static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid)
{
struct pbase_tree_cache *ent, *nent;
void *data;
unsigned long size;
enum object_type type;
int neigh;
- int my_ix = pbase_tree_cache_ix(sha1);
+ int my_ix = pbase_tree_cache_ix(oid);
int available_ix = -1;
/* pbase-tree-cache acts as a limited hashtable.
*/
for (neigh = 0; neigh < 8; neigh++) {
ent = pbase_tree_cache[my_ix];
- if (ent && !hashcmp(ent->sha1, sha1)) {
+ if (ent && !oidcmp(&ent->oid, oid)) {
ent->ref++;
return ent;
}
/* Did not find one. Either we got a bogus request or
* we need to read and perhaps cache.
*/
- data = read_sha1_file(sha1, &type, &size);
+ data = read_sha1_file(oid->hash, &type, &size);
if (!data)
return NULL;
if (type != OBJ_TREE) {
free(ent->tree_data);
nent = ent;
}
- hashcpy(nent->sha1, sha1);
+ oidcpy(&nent->oid, oid);
nent->tree_data = data;
nent->tree_size = size;
nent->ref = 1;
if (cmp < 0)
return;
if (name[cmplen] != '/') {
- add_object_entry(entry.oid->hash,
+ add_object_entry(entry.oid,
object_type(entry.mode),
fullname, 1);
return;
const char *down = name+cmplen+1;
int downlen = name_cmp_len(down);
- tree = pbase_tree_get(entry.oid->hash);
+ tree = pbase_tree_get(entry.oid);
if (!tree)
return;
init_tree_desc(&sub, tree->tree_data, tree->tree_size);
int lo = 0;
int hi = done_pbase_paths_num;
while (lo < hi) {
- int mi = (hi + lo) / 2;
+ int mi = lo + (hi - lo) / 2;
if (done_pbase_paths[mi] == hash)
return mi;
if (done_pbase_paths[mi] < hash)
cmplen = name_cmp_len(name);
for (it = pbase_tree; it; it = it->next) {
if (cmplen == 0) {
- add_object_entry(it->pcache.sha1, OBJ_TREE, NULL, 1);
+ add_object_entry(&it->pcache.oid, OBJ_TREE, NULL, 1);
}
else {
struct tree_desc tree;
}
}
-static void add_preferred_base(unsigned char *sha1)
+static void add_preferred_base(struct object_id *oid)
{
struct pbase_tree *it;
void *data;
unsigned long size;
- unsigned char tree_sha1[20];
+ struct object_id tree_oid;
if (window <= num_preferred_base++)
return;
- data = read_object_with_reference(sha1, tree_type, &size, tree_sha1);
+ data = read_object_with_reference(oid->hash, tree_type, &size, tree_oid.hash);
if (!data)
return;
for (it = pbase_tree; it; it = it->next) {
- if (!hashcmp(it->pcache.sha1, tree_sha1)) {
+ if (!oidcmp(&it->pcache.oid, &tree_oid)) {
free(data);
return;
}
it->next = pbase_tree;
pbase_tree = it;
- hashcpy(it->pcache.sha1, tree_sha1);
+ oidcpy(&it->pcache.oid, &tree_oid);
it->pcache.tree_data = data;
it->pcache.tree_size = size;
}
die("unable to pack objects reachable from tag %s",
oid_to_hex(oid));
- add_object_entry(tag->object.oid.hash, OBJ_TAG, NULL, 0);
+ add_object_entry(&tag->object.oid, OBJ_TAG, NULL, 0);
if (tag->tagged->type != OBJ_TAG)
return;
struct object_id peeled;
if (starts_with(path, "refs/tags/") && /* is a tag? */
- !peel_ref(path, peeled.hash) && /* peelable? */
+ !peel_ref(path, &peeled) && /* peelable? */
packlist_find(&to_pack, peeled.hash, NULL)) /* object packed? */
add_tag_chain(oid);
return 0;
static void read_object_list_from_stdin(void)
{
- char line[40 + 1 + PATH_MAX + 2];
- unsigned char sha1[20];
+ char line[GIT_MAX_HEXSZ + 1 + PATH_MAX + 2];
+ struct object_id oid;
+ const char *p;
for (;;) {
if (!fgets(line, sizeof(line), stdin)) {
continue;
}
if (line[0] == '-') {
- if (get_sha1_hex(line+1, sha1))
- die("expected edge sha1, got garbage:\n %s",
+ if (get_oid_hex(line+1, &oid))
+ die("expected edge object ID, got garbage:\n %s",
line);
- add_preferred_base(sha1);
+ add_preferred_base(&oid);
continue;
}
- if (get_sha1_hex(line, sha1))
- die("expected sha1, got garbage:\n %s", line);
+ if (parse_oid_hex(line, &oid, &p))
+ die("expected object ID, got garbage:\n %s", line);
- add_preferred_base_object(line+41);
- add_object_entry(sha1, 0, line+41, 0);
+ add_preferred_base_object(p + 1);
+ add_object_entry(&oid, 0, p + 1, 0);
}
}
static void show_commit(struct commit *commit, void *data)
{
- add_object_entry(commit->object.oid.hash, OBJ_COMMIT, NULL, 0);
+ add_object_entry(&commit->object.oid, OBJ_COMMIT, NULL, 0);
commit->object.flags |= OBJECT_ADDED;
if (write_bitmap_index)
static void show_object(struct object *obj, const char *name, void *data)
{
add_preferred_base_object(name);
- add_object_entry(obj->oid.hash, obj->type, name, 0);
+ add_object_entry(&obj->oid, obj->type, name, 0);
obj->flags |= OBJECT_ADDED;
}
+static void show_object__ma_allow_any(struct object *obj, const char *name, void *data)
+{
+ assert(arg_missing_action == MA_ALLOW_ANY);
+
+ /*
+ * Quietly ignore ALL missing objects. This avoids problems with
+ * staging them now and getting an odd error later.
+ */
+ if (!has_object_file(&obj->oid))
+ return;
+
+ show_object(obj, name, data);
+}
+
+static void show_object__ma_allow_promisor(struct object *obj, const char *name, void *data)
+{
+ assert(arg_missing_action == MA_ALLOW_PROMISOR);
+
+ /*
+ * Quietly ignore EXPECTED missing objects. This avoids problems with
+ * staging them now and getting an odd error later.
+ */
+ if (!has_object_file(&obj->oid) && is_promisor_object(&obj->oid))
+ return;
+
+ show_object(obj, name, data);
+}
+
+static int option_parse_missing_action(const struct option *opt,
+ const char *arg, int unset)
+{
+ assert(arg);
+ assert(!unset);
+
+ if (!strcmp(arg, "error")) {
+ arg_missing_action = MA_ERROR;
+ fn_show_object = show_object;
+ return 0;
+ }
+
+ if (!strcmp(arg, "allow-any")) {
+ arg_missing_action = MA_ALLOW_ANY;
+ fetch_if_missing = 0;
+ fn_show_object = show_object__ma_allow_any;
+ return 0;
+ }
+
+ if (!strcmp(arg, "allow-promisor")) {
+ arg_missing_action = MA_ALLOW_PROMISOR;
+ fetch_if_missing = 0;
+ fn_show_object = show_object__ma_allow_promisor;
+ return 0;
+ }
+
+ die(_("invalid value for --missing"));
+ return 0;
+}
+
static void show_edge(struct commit *commit)
{
- add_preferred_base(commit->object.oid.hash);
+ add_preferred_base(&commit->object.oid);
}
struct in_pack_object {
};
struct in_pack {
- int alloc;
- int nr;
+ unsigned int alloc;
+ unsigned int nr;
struct in_pack_object *array;
};
memset(&in_pack, 0, sizeof(in_pack));
for (p = packed_git; p; p = p->next) {
- const unsigned char *sha1;
+ struct object_id oid;
struct object *o;
if (!p->pack_local || p->pack_keep)
in_pack.alloc);
for (i = 0; i < p->num_objects; i++) {
- sha1 = nth_packed_object_sha1(p, i);
- o = lookup_unknown_object(sha1);
+ nth_packed_object_oid(&oid, p, i);
+ o = lookup_unknown_object(oid.hash);
if (!(o->flags & OBJECT_ADDED))
mark_in_pack_object(o, p, &in_pack);
o->flags |= OBJECT_ADDED;
QSORT(in_pack.array, in_pack.nr, ofscmp);
for (i = 0; i < in_pack.nr; i++) {
struct object *o = in_pack.array[i].object;
- add_object_entry(o->oid.hash, o->type, "", 0);
+ add_object_entry(&o->oid, o->type, "", 0);
}
}
free(in_pack.array);
return 0;
}
- add_object_entry(oid->hash, type, "", 0);
+ add_object_entry(oid, type, "", 0);
return 0;
}
NULL, NULL, NULL);
}
-static int has_sha1_pack_kept_or_nonlocal(const unsigned char *sha1)
+static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid)
{
static struct packed_git *last_found = (void *)1;
struct packed_git *p;
while (p) {
if ((!p->pack_local || p->pack_keep) &&
- find_pack_entry_one(sha1, p)) {
+ find_pack_entry_one(oid->hash, p)) {
last_found = p;
return 1;
}
for (i = 0; i < p->num_objects; i++) {
nth_packed_object_oid(&oid, p, i);
if (!packlist_find(&to_pack, oid.hash, NULL) &&
- !has_sha1_pack_kept_or_nonlocal(oid.hash) &&
+ !has_sha1_pack_kept_or_nonlocal(&oid) &&
!loosened_object_can_be_discarded(&oid, p->mtime))
- if (force_object_loose(oid.hash, p->mtime))
+ if (force_object_loose(&oid, p->mtime))
die("unable to force loose object");
}
}
if (prepare_revision_walk(&revs))
die("revision walk setup failed");
mark_edges_uninteresting(&revs, show_edge);
- traverse_commit_list(&revs, show_commit, show_object, NULL);
+
+ if (!fn_show_object)
+ fn_show_object = show_object;
+ traverse_commit_list_filtered(&filter_options, &revs,
+ show_commit, fn_show_object, NULL,
+ NULL);
if (unpack_unreachable_expiration) {
revs.ignore_missing_links = 1;
N_("use a bitmap index if available to speed up counting objects")),
OPT_BOOL(0, "write-bitmap-index", &write_bitmap_index,
N_("write a bitmap index together with the pack index")),
+ OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),
+ { OPTION_CALLBACK, 0, "missing", NULL, N_("action"),
+ N_("handling for missing objects"), PARSE_OPT_NONEG,
+ option_parse_missing_action },
+ OPT_BOOL(0, "exclude-promisor-objects", &exclude_promisor_objects,
+ N_("do not pack objects in promisor packfiles")),
OPT_END(),
};
argv_array_push(&rp, "--unpacked");
}
+ if (exclude_promisor_objects) {
+ use_internal_rev_list = 1;
+ fetch_if_missing = 0;
+ argv_array_push(&rp, "--exclude-promisor-objects");
+ }
+
if (!reuse_object)
reuse_delta = 0;
if (pack_compression_level == -1)
if (!rev_list_all || !rev_list_reflog || !rev_list_index)
unpack_unreachable_expiration = 0;
+ if (filter_options.choice) {
+ if (!pack_to_stdout)
+ die("cannot use --filter without --stdout.");
+ use_bitmap_index = 0;
+ }
+
/*
* "soft" reasons not to use bitmaps - for on-disk repack by default we want
*