#include "builtin.h"
#include "cache.h"
+#include "attr.h"
#include "object.h"
#include "blob.h"
#include "commit.h"
#include "progress.h"
static const char pack_usage[] = "\
-git-pack-objects [{ -q | --progress | --all-progress }] \n\
+git-pack-objects [{ -q | --progress | --all-progress }] [--max-pack-size=N] \n\
[--local] [--incremental] [--window=N] [--depth=N] \n\
[--no-reuse-delta] [--no-reuse-object] [--delta-base-offset] \n\
[--non-empty] [--revs [--unpacked | --all]*] [--reflog] \n\
struct object_entry *delta_sibling; /* other deltified objects who
* uses the same base as me
*/
+ void *delta_data; /* cached delta (uncompressed) */
unsigned long delta_size; /* delta data size (uncompressed) */
enum object_type type;
enum object_type in_pack_type; /* could be delta */
unsigned char in_pack_header_size;
unsigned char preferred_base; /* we do not pack this, but is available
- * to be used as the base objectto delta
+ * to be used as the base object to delta
* objects against.
*/
+ unsigned char no_try_delta;
};
/*
static int pack_compression_level = Z_DEFAULT_COMPRESSION;
static int pack_compression_seen;
+static unsigned long delta_cache_size = 0;
+static unsigned long max_delta_cache_size = 0;
+
/*
* The object names in objects array are hashed with this hashtable,
* to help looking up the entry by object name.
z_stream stream;
unsigned long maxsize;
void *out;
- buf = read_sha1_file(entry->sha1, &type, &size);
- if (!buf)
- die("unable to read %s", sha1_to_hex(entry->sha1));
- if (size != entry->size)
- die("object %s size inconsistency (%lu vs %lu)",
- sha1_to_hex(entry->sha1), size, entry->size);
- if (usable_delta) {
- buf = delta_against(buf, size, entry);
+ if (entry->delta_data && usable_delta) {
+ buf = entry->delta_data;
size = entry->delta_size;
obj_type = (allow_ofs_delta && entry->delta->offset) ?
OBJ_OFS_DELTA : OBJ_REF_DELTA;
} else {
- /*
- * recover real object type in case
- * check_object() wanted to re-use a delta,
- * but we couldn't since base was in previous split pack
- */
- obj_type = type;
+ buf = read_sha1_file(entry->sha1, &type, &size);
+ if (!buf)
+ die("unable to read %s", sha1_to_hex(entry->sha1));
+ if (size != entry->size)
+ die("object %s size inconsistency (%lu vs %lu)",
+ sha1_to_hex(entry->sha1), size, entry->size);
+ if (usable_delta) {
+ buf = delta_against(buf, size, entry);
+ size = entry->delta_size;
+ obj_type = (allow_ofs_delta && entry->delta->offset) ?
+ OBJ_OFS_DELTA : OBJ_REF_DELTA;
+ } else {
+ /*
+ * recover real object type in case
+ * check_object() wanted to re-use a delta,
+ * but we couldn't since base was in previous split pack
+ */
+ obj_type = type;
+ }
}
/* compress the data to store and put compressed length in datalen */
memset(&stream, 0, sizeof(stream));
e->offset = 0;
return 0;
}
+ written_list[nr_written++] = e;
/* make sure off_t is sufficiently large not to wrap */
if (offset > offset + size)
static void write_pack_file(void)
{
- uint32_t i;
+ uint32_t i = 0, j;
struct sha1file *f;
- off_t offset, last_obj_offset = 0;
+ off_t offset, offset_one, last_obj_offset = 0;
struct pack_header hdr;
- int do_progress = progress;
-
- if (pack_to_stdout) {
- f = sha1fd(1, "<stdout>");
- do_progress >>= 1;
- } else {
- int fd = open_object_dir_tmp("tmp_pack_XXXXXX");
- if (fd < 0)
- die("unable to create %s: %s\n", tmpname, strerror(errno));
- pack_tmp_name = xstrdup(tmpname);
- f = sha1fd(fd, pack_tmp_name);
- }
+ int do_progress = progress >> pack_to_stdout;
+ uint32_t nr_remaining = nr_result;
if (do_progress)
start_progress(&progress_state, "Writing %u objects...", "", nr_result);
+ written_list = xmalloc(nr_objects * sizeof(struct object_entry *));
- hdr.hdr_signature = htonl(PACK_SIGNATURE);
- hdr.hdr_version = htonl(PACK_VERSION);
- hdr.hdr_entries = htonl(nr_result);
- sha1write(f, &hdr, sizeof(hdr));
- offset = sizeof(hdr);
- if (!nr_result)
- goto done;
- for (i = 0; i < nr_objects; i++) {
- last_obj_offset = offset;
- offset = write_one(f, objects + i, offset);
- if (do_progress)
- display_progress(&progress_state, written);
- }
- if (do_progress)
- stop_progress(&progress_state);
- done:
- if (written != nr_result)
- die("wrote %u objects while expecting %u", written, nr_result);
- sha1close(f, pack_file_sha1, 1);
+ do {
+ if (pack_to_stdout) {
+ f = sha1fd(1, "<stdout>");
+ } else {
+ int fd = open_object_dir_tmp("tmp_pack_XXXXXX");
+ if (fd < 0)
+ die("unable to create %s: %s\n", tmpname, strerror(errno));
+ pack_tmp_name = xstrdup(tmpname);
+ f = sha1fd(fd, pack_tmp_name);
+ }
+
+ hdr.hdr_signature = htonl(PACK_SIGNATURE);
+ hdr.hdr_version = htonl(PACK_VERSION);
+ hdr.hdr_entries = htonl(nr_remaining);
+ sha1write(f, &hdr, sizeof(hdr));
+ offset = sizeof(hdr);
+ nr_written = 0;
+ for (; i < nr_objects; i++) {
+ last_obj_offset = offset;
+ offset_one = write_one(f, objects + i, offset);
+ if (!offset_one)
+ break;
+ offset = offset_one;
+ if (do_progress)
+ display_progress(&progress_state, written);
+ }
+
+ /*
+ * Did we write the wrong # entries in the header?
+ * If so, rewrite it like in fast-import
+ */
+ if (pack_to_stdout || nr_written == nr_remaining) {
+ sha1close(f, pack_file_sha1, 1);
+ } else {
+ sha1close(f, pack_file_sha1, 0);
+ fixup_pack_header_footer(f->fd, pack_file_sha1, pack_tmp_name, nr_written);
+ close(f->fd);
+ }
- if (!pack_to_stdout) {
+ if (!pack_to_stdout) {
unsigned char object_list_sha1[20];
mode_t mode = umask(0);
die("unable to rename temporary index file: %s",
strerror(errno));
puts(sha1_to_hex(object_list_sha1));
+ }
+
+ /* mark written objects as written to previous pack */
+ for (j = 0; j < nr_written; j++) {
+ written_list[j]->offset = (off_t)-1;
+ }
+ nr_remaining -= nr_written;
+ } while (nr_remaining && i < nr_objects);
+
+ free(written_list);
+ if (do_progress)
+ stop_progress(&progress_state);
+ if (written != nr_result)
+ die("wrote %u objects while expecting %u", written, nr_result);
+ /*
+ * We have scanned through [0 ... i). Since we have written
+ * the correct number of objects, the remaining [i ... nr_objects)
+ * items must be either already written (due to out-of-order delta base)
+ * or a preferred base. Count those which are neither and complain if any.
+ */
+ for (j = 0; i < nr_objects; i++) {
+ struct object_entry *e = objects + i;
+ j += !e->offset && !e->preferred_base;
}
+ if (j)
+ die("wrote %u objects as expected but %u unwritten", written, j);
}
static int sha1_sort(const void *_a, const void *_b)
idx_tmp_name = xstrdup(tmpname);
f = sha1fd(fd, idx_tmp_name);
- if (nr_result) {
- uint32_t j = 0;
- sorted_by_sha =
- xcalloc(nr_result, sizeof(struct object_entry *));
- for (i = 0; i < nr_objects; i++)
- if (!objects[i].preferred_base)
- sorted_by_sha[j++] = objects + i;
- if (j != nr_result)
- die("listed %u objects while expecting %u", j, nr_result);
- qsort(sorted_by_sha, nr_result, sizeof(*sorted_by_sha), sha1_sort);
+ if (nr_written) {
+ sorted_by_sha = written_list;
+ qsort(sorted_by_sha, nr_written, sizeof(*sorted_by_sha), sha1_sort);
list = sorted_by_sha;
- last = sorted_by_sha + nr_result;
+ last = sorted_by_sha + nr_written;
} else
sorted_by_sha = list = last = NULL;
/* Write the actual SHA1 entries. */
list = sorted_by_sha;
- for (i = 0; i < nr_result; i++) {
+ for (i = 0; i < nr_written; i++) {
struct object_entry *entry = *list++;
if (index_version < 2) {
uint32_t offset = htonl(entry->offset);
/* write the crc32 table */
list = sorted_by_sha;
- for (i = 0; i < nr_objects; i++) {
+ for (i = 0; i < nr_written; i++) {
struct object_entry *entry = *list++;
uint32_t crc32_val = htonl(entry->crc32);
sha1write(f, &crc32_val, 4);
/* write the 32-bit offset table */
list = sorted_by_sha;
- for (i = 0; i < nr_objects; i++) {
+ for (i = 0; i < nr_written; i++) {
struct object_entry *entry = *list++;
uint32_t offset = (entry->offset <= index_off32_limit) ?
entry->offset : (0x80000000 | nr_large_offset++);
sha1write(f, pack_file_sha1, 20);
sha1close(f, NULL, 1);
- free(sorted_by_sha);
SHA1_Final(sha1, &ctx);
}
unsigned char c;
unsigned hash = 0;
+ if (!name)
+ return 0;
+
/*
* This effectively just creates a sortable number from the
* last sixteen non-whitespace characters. Last characters
return hash;
}
+static void setup_delta_attr_check(struct git_attr_check *check)
+{
+ static struct git_attr *attr_delta;
+
+ if (!attr_delta)
+ attr_delta = git_attr("delta", 5);
+
+ check[0].attr = attr_delta;
+}
+
+static int no_try_delta(const char *path)
+{
+ struct git_attr_check check[1];
+
+ setup_delta_attr_check(check);
+ if (git_checkattr(path, ARRAY_SIZE(check), check))
+ return 0;
+ if (ATTR_FALSE(check->value))
+ return 1;
+ return 0;
+}
+
static int add_object_entry(const unsigned char *sha1, enum object_type type,
- unsigned hash, int exclude)
+ const char *name, int exclude)
{
struct object_entry *entry;
struct packed_git *p, *found_pack = NULL;
off_t found_offset = 0;
int ix;
+ unsigned hash = name_hash(name);
ix = nr_objects ? locate_object_entry_hash(sha1) : -1;
if (ix >= 0) {
if (progress)
display_progress(&progress_state, nr_objects);
+ if (name && no_try_delta(name))
+ entry->no_try_delta = 1;
+
return 1;
}
if (cmp < 0)
return;
if (name[cmplen] != '/') {
- unsigned hash = name_hash(fullname);
add_object_entry(entry.sha1,
S_ISDIR(entry.mode) ? OBJ_TREE : OBJ_BLOB,
- hash, 1);
+ fullname, 1);
return;
}
if (S_ISDIR(entry.mode)) {
return 0;
}
-static void add_preferred_base_object(const char *name, unsigned hash)
+static void add_preferred_base_object(const char *name)
{
struct pbase_tree *it;
int cmplen;
+ unsigned hash = name_hash(name);
if (!num_preferred_base || check_pbase_path(hash))
return;
cmplen = name_cmp_len(name);
for (it = pbase_tree; it; it = it->next) {
if (cmplen == 0) {
- add_object_entry(it->pcache.sha1, OBJ_TREE, 0, 1);
+ add_object_entry(it->pcache.sha1, OBJ_TREE, NULL, 1);
}
else {
struct tree_desc tree;
struct delta_index *index;
};
+static int delta_cacheable(struct unpacked *trg, struct unpacked *src,
+ unsigned long src_size, unsigned long trg_size,
+ unsigned long delta_size)
+{
+ if (max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size)
+ return 0;
+
+ /* cache delta, if objects are large enough compared to delta size */
+ if ((src_size >> 20) + (trg_size >> 21) > (delta_size >> 10))
+ return 1;
+
+ return 0;
+}
+
/*
* We search for deltas _backwards_ in a list sorted by type and
* by size, so that we see progressively smaller and smaller files.
}
if (!src->index) {
src->index = create_delta_index(src->data, src_size);
- if (!src->index)
- die("out of memory");
+ if (!src->index) {
+ static int warned = 0;
+ if (!warned++)
+ warning("suboptimal pack - out of memory");
+ return 0;
+ }
}
delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size);
if (!delta_buf)
return 0;
+ if (trg_entry->delta_data) {
+ delta_cache_size -= trg_entry->delta_size;
+ free(trg_entry->delta_data);
+ }
+ trg_entry->delta_data = 0;
trg_entry->delta = src_entry;
trg_entry->delta_size = delta_size;
trg_entry->depth = src_entry->depth + 1;
- free(delta_buf);
+
+ if (delta_cacheable(src, trg, src_size, trg_size, delta_size)) {
+ trg_entry->delta_data = xrealloc(delta_buf, delta_size);
+ delta_cache_size += trg_entry->delta_size;
+ } else
+ free(delta_buf);
return 1;
}
if (entry->size < 50)
continue;
+
+ if (entry->no_try_delta)
+ continue;
+
free_delta_index(n->index);
n->index = NULL;
free(n->data);
pack_compression_seen = 1;
return 0;
}
+ if (!strcmp(k, "pack.deltacachesize")) {
+ max_delta_cache_size = git_config_int(k, v);
+ return 0;
+ }
return git_default_config(k, v);
}
{
char line[40 + 1 + PATH_MAX + 2];
unsigned char sha1[20];
- unsigned hash;
for (;;) {
if (!fgets(line, sizeof(line), stdin)) {
if (get_sha1_hex(line, sha1))
die("expected sha1, got garbage:\n %s", line);
- hash = name_hash(line+41);
- add_preferred_base_object(line+41, hash);
- add_object_entry(sha1, 0, hash, 0);
+ add_preferred_base_object(line+41);
+ add_object_entry(sha1, 0, line+41, 0);
}
}
static void show_commit(struct commit *commit)
{
- add_object_entry(commit->object.sha1, OBJ_COMMIT, 0, 0);
+ add_object_entry(commit->object.sha1, OBJ_COMMIT, NULL, 0);
}
static void show_object(struct object_array_entry *p)
{
- unsigned hash = name_hash(p->name);
- add_preferred_base_object(p->name, hash);
- add_object_entry(p->item->sha1, p->item->type, hash, 0);
+ add_preferred_base_object(p->name);
+ add_object_entry(p->item->sha1, p->item->type, p->name, 0);
}
static void show_edge(struct commit *commit)
pack_compression_level = level;
continue;
}
+ if (!prefixcmp(arg, "--max-pack-size=")) {
+ char *end;
+ pack_size_limit = strtoul(arg+16, &end, 0) * 1024 * 1024;
+ if (!arg[16] || *end)
+ usage(pack_usage);
+ continue;
+ }
if (!prefixcmp(arg, "--window=")) {
char *end;
window = strtoul(arg+9, &end, 0);
if (pack_to_stdout != !base_name)
usage(pack_usage);
+ if (pack_to_stdout && pack_size_limit)
+ die("--max-pack-size cannot be used to build a pack for transfer.");
+
if (!pack_to_stdout && thin)
die("--thin cannot be used to build an indexable pack.");