pack-objects: equal objects in size should delta against newer objects
[gitweb.git] / builtin-pack-objects.c
index b5ed9ce2c89daab7a69d399a6599e203dc04971a..869ca1ab269859c6e23676f329e8c59e4f6aeb20 100644 (file)
@@ -23,7 +23,7 @@ git-pack-objects [{ -q | --progress | --all-progress }] \n\
 struct object_entry {
        unsigned char sha1[20];
        unsigned long size;     /* uncompressed size */
-       unsigned long offset;   /* offset into the final pack file;
+       off_t offset;   /* offset into the final pack file;
                                 * nonzero if already written.
                                 */
        unsigned int depth;     /* delta depth */
@@ -35,7 +35,7 @@ struct object_entry {
 #define in_pack_header_size delta_size /* only when reusing pack data */
        struct object_entry *delta;     /* delta base object */
        struct packed_git *in_pack;     /* already in pack */
-       unsigned int in_pack_offset;
+       off_t in_pack_offset;
        struct object_entry *delta_child; /* deltified objects who bases me */
        struct object_entry *delta_sibling; /* other deltified objects who
                                             * uses the same base as me
@@ -44,6 +44,7 @@ struct object_entry {
                                 * be used as the base objectto delta huge
                                 * objects against.
                                 */
+       uint32_t crc32;         /* crc of raw pack data for this object */
 };
 
 /*
@@ -68,7 +69,7 @@ static int allow_ofs_delta;
 
 static struct object_entry **sorted_by_sha, **sorted_by_type;
 static struct object_entry *objects;
-static int nr_objects, nr_alloc, nr_result;
+static uint32_t nr_objects, nr_alloc, nr_result;
 static const char *base_name;
 static unsigned char pack_file_sha1[20];
 static int progress = 1;
@@ -101,7 +102,7 @@ static int object_ix_hashsz;
  * get the object sha1 from the main index.
  */
 struct revindex_entry {
-       unsigned int offset;
+       off_t offset;
        unsigned int nr;
 };
 struct pack_revindex {
@@ -114,10 +115,8 @@ static int pack_revindex_hashsz;
 /*
  * stats
  */
-static int written;
-static int written_delta;
-static int reused;
-static int reused_delta;
+static uint32_t written, written_delta;
+static uint32_t reused, reused_delta;
 
 static int pack_revindex_ix(struct packed_git *p)
 {
@@ -166,16 +165,37 @@ static int cmp_offset(const void *a_, const void *b_)
 static void prepare_pack_revindex(struct pack_revindex *rix)
 {
        struct packed_git *p = rix->p;
-       int num_ent = num_packed_objects(p);
+       int num_ent = p->num_objects;
        int i;
-       void *index = p->index_base + 256;
+       const char *index = p->index_data;
 
        rix->revindex = xmalloc(sizeof(*rix->revindex) * (num_ent + 1));
-       for (i = 0; i < num_ent; i++) {
-               unsigned int hl = *((unsigned int *)((char *) index + 24*i));
-               rix->revindex[i].offset = ntohl(hl);
-               rix->revindex[i].nr = i;
+       index += 4 * 256;
+
+       if (p->index_version > 1) {
+               const uint32_t *off_32 =
+                       (uint32_t *)(index + 8 + p->num_objects * (20 + 4));
+               const uint32_t *off_64 = off_32 + p->num_objects;
+               for (i = 0; i < num_ent; i++) {
+                       uint32_t off = ntohl(*off_32++);
+                       if (!(off & 0x80000000)) {
+                               rix->revindex[i].offset = off;
+                       } else {
+                               rix->revindex[i].offset =
+                                       ((uint64_t)ntohl(*off_64++)) << 32;
+                               rix->revindex[i].offset |=
+                                       ntohl(*off_64++);
+                       }
+                       rix->revindex[i].nr = i;
+               }
+       } else {
+               for (i = 0; i < num_ent; i++) {
+                       uint32_t hl = *((uint32_t *)(index + 24 * i));
+                       rix->revindex[i].offset = ntohl(hl);
+                       rix->revindex[i].nr = i;
+               }
        }
+
        /* This knows the pack format -- the 20-byte trailer
         * follows immediately after the last object data.
         */
@@ -185,7 +205,7 @@ static void prepare_pack_revindex(struct pack_revindex *rix)
 }
 
 static struct revindex_entry * find_packed_object(struct packed_git *p,
-                                                 unsigned int ofs)
+                                                 off_t ofs)
 {
        int num;
        int lo, hi;
@@ -199,7 +219,7 @@ static struct revindex_entry * find_packed_object(struct packed_git *p,
                prepare_pack_revindex(rix);
        revindex = rix->revindex;
        lo = 0;
-       hi = num_packed_objects(p) + 1;
+       hi = p->num_objects + 1;
        do {
                int mi = (lo + hi) / 2;
                if (revindex[mi].offset == ofs) {
@@ -213,25 +233,18 @@ static struct revindex_entry * find_packed_object(struct packed_git *p,
        die("internal error: pack revindex corrupt");
 }
 
-static unsigned long find_packed_object_size(struct packed_git *p,
-                                            unsigned long ofs)
+static const unsigned char *find_packed_object_name(struct packed_git *p,
+                                                   off_t ofs)
 {
        struct revindex_entry *entry = find_packed_object(p, ofs);
-       return entry[1].offset - ofs;
-}
-
-static unsigned char *find_packed_object_name(struct packed_git *p,
-                                             unsigned long ofs)
-{
-       struct revindex_entry *entry = find_packed_object(p, ofs);
-       return (unsigned char *)(p->index_base + 256) + 24 * entry->nr + 4;
+       return nth_packed_object_sha1(p, entry->nr);
 }
 
 static void *delta_against(void *buf, unsigned long size, struct object_entry *entry)
 {
        unsigned long othersize, delta_size;
-       char type[10];
-       void *otherbuf = read_sha1_file(entry->delta->sha1, type, &othersize);
+       enum object_type type;
+       void *otherbuf = read_sha1_file(entry->delta->sha1, &type, &othersize);
        void *delta_buf;
 
        if (!otherbuf)
@@ -278,8 +291,8 @@ static int encode_header(enum object_type type, unsigned long size, unsigned cha
  */
 static int check_pack_inflate(struct packed_git *p,
                struct pack_window **w_curs,
-               unsigned long offset,
-               unsigned long len,
+               off_t offset,
+               off_t len,
                unsigned long expect)
 {
        z_stream stream;
@@ -302,11 +315,33 @@ static int check_pack_inflate(struct packed_git *p,
                stream.total_in == len) ? 0 : -1;
 }
 
+static int check_pack_crc(struct packed_git *p, struct pack_window **w_curs,
+                         off_t offset, off_t len, unsigned int nr)
+{
+       const uint32_t *index_crc;
+       uint32_t data_crc = crc32(0, Z_NULL, 0);
+
+       do {
+               unsigned int avail;
+               void *data = use_pack(p, w_curs, offset, &avail);
+               if (avail > len)
+                       avail = len;
+               data_crc = crc32(data_crc, data, avail);
+               offset += avail;
+               len -= avail;
+       } while (len);
+
+       index_crc = p->index_data;
+       index_crc += 2 + 256 + p->num_objects * (20/4) + nr;
+
+       return data_crc != ntohl(*index_crc);
+}
+
 static void copy_pack_data(struct sha1file *f,
                struct packed_git *p,
                struct pack_window **w_curs,
-               unsigned long offset,
-               unsigned long len)
+               off_t offset,
+               off_t len)
 {
        unsigned char *in;
        unsigned int avail;
@@ -314,7 +349,7 @@ static void copy_pack_data(struct sha1file *f,
        while (len) {
                in = use_pack(p, w_curs, offset, &avail);
                if (avail > len)
-                       avail = len;
+                       avail = (unsigned int)len;
                sha1write(f, in, avail);
                offset += avail;
                len -= avail;
@@ -375,13 +410,17 @@ static unsigned long write_object(struct sha1file *f,
                                  struct object_entry *entry)
 {
        unsigned long size;
-       char type[10];
+       enum object_type type;
        void *buf;
        unsigned char header[10];
-       unsigned hdrlen, datalen;
+       unsigned hdrlen;
+       off_t datalen;
        enum object_type obj_type;
        int to_reuse = 0;
 
+       if (!pack_to_stdout)
+               crc32_begin(f);
+
        obj_type = entry->type;
        if (! entry->in_pack)
                to_reuse = 0;   /* can't reuse what we don't have */
@@ -416,7 +455,7 @@ static unsigned long write_object(struct sha1file *f,
        }
 
        if (!to_reuse) {
-               buf = read_sha1_file(entry->sha1, type, &size);
+               buf = read_sha1_file(entry->sha1, &type, &size);
                if (!buf)
                        die("unable to read %s", sha1_to_hex(entry->sha1));
                if (size != entry->size)
@@ -441,7 +480,7 @@ static unsigned long write_object(struct sha1file *f,
                         * encoding of the relative offset for the delta
                         * base from this object's position in the pack.
                         */
-                       unsigned long ofs = entry->offset - entry->delta->offset;
+                       off_t ofs = entry->offset - entry->delta->offset;
                        unsigned pos = sizeof(header) - 1;
                        header[pos] = ofs & 127;
                        while (ofs >>= 7)
@@ -462,7 +501,8 @@ static unsigned long write_object(struct sha1file *f,
        else {
                struct packed_git *p = entry->in_pack;
                struct pack_window *w_curs = NULL;
-               unsigned long offset;
+               struct revindex_entry *revidx;
+               off_t offset;
 
                if (entry->delta) {
                        obj_type = (allow_ofs_delta && entry->delta->offset) ?
@@ -472,7 +512,7 @@ static unsigned long write_object(struct sha1file *f,
                hdrlen = encode_header(obj_type, entry->size, header);
                sha1write(f, header, hdrlen);
                if (obj_type == OBJ_OFS_DELTA) {
-                       unsigned long ofs = entry->offset - entry->delta->offset;
+                       off_t ofs = entry->offset - entry->delta->offset;
                        unsigned pos = sizeof(header) - 1;
                        header[pos] = ofs & 127;
                        while (ofs >>= 7)
@@ -484,12 +524,17 @@ static unsigned long write_object(struct sha1file *f,
                        hdrlen += 20;
                }
 
-               offset = entry->in_pack_offset + entry->in_pack_header_size;
-               datalen = find_packed_object_size(p, entry->in_pack_offset)
-                               - entry->in_pack_header_size;
-               if (!pack_to_stdout && check_pack_inflate(p, &w_curs,
-                               offset, datalen, entry->size))
-                       die("corrupt delta in pack %s", sha1_to_hex(entry->sha1));
+               offset = entry->in_pack_offset;
+               revidx = find_packed_object(p, offset);
+               datalen = revidx[1].offset - offset;
+               if (!pack_to_stdout && p->index_version > 1 &&
+                   check_pack_crc(p, &w_curs, offset, datalen, revidx->nr))
+                       die("bad packed object CRC for %s", sha1_to_hex(entry->sha1));
+               offset += entry->in_pack_header_size;
+               datalen -= entry->in_pack_header_size;
+               if (!pack_to_stdout && p->index_version == 1 &&
+                   check_pack_inflate(p, &w_curs, offset, datalen, entry->size))
+                       die("corrupt packed object for %s", sha1_to_hex(entry->sha1));
                copy_pack_data(f, p, &w_curs, offset, datalen);
                unuse_pack(&w_curs);
                reused++;
@@ -497,30 +542,39 @@ static unsigned long write_object(struct sha1file *f,
        if (entry->delta)
                written_delta++;
        written++;
+       if (!pack_to_stdout)
+               entry->crc32 = crc32_end(f);
        return hdrlen + datalen;
 }
 
-static unsigned long write_one(struct sha1file *f,
+static off_t write_one(struct sha1file *f,
                               struct object_entry *e,
-                              unsigned long offset)
+                              off_t offset)
 {
+       unsigned long size;
+
+       /* offset is non zero if object is written already. */
        if (e->offset || e->preferred_base)
-               /* offset starts from header size and cannot be zero
-                * if it is written already.
-                */
                return offset;
-       /* if we are deltified, write out its base object first. */
+
+       /* if we are deltified, write out base object first. */
        if (e->delta)
                offset = write_one(f, e->delta, offset);
+
        e->offset = offset;
-       return offset + write_object(f, e);
+       size = write_object(f, e);
+
+       /* make sure off_t is sufficiently large not to wrap */
+       if (offset > offset + size)
+               die("pack too large for current definition of off_t");
+       return offset + size;
 }
 
-static void write_pack_file(void)
+static off_t write_pack_file(void)
 {
-       int i;
+       uint32_t i;
        struct sha1file *f;
-       unsigned long offset;
+       off_t offset, last_obj_offset = 0;
        struct pack_header hdr;
        unsigned last_percent = 999;
        int do_progress = progress;
@@ -533,7 +587,7 @@ static void write_pack_file(void)
                f = sha1create("%s-%s.%s", base_name,
                               sha1_to_hex(object_list_sha1), "pack");
        if (do_progress)
-               fprintf(stderr, "Writing %d objects.\n", nr_result);
+               fprintf(stderr, "Writing %u objects.\n", nr_result);
 
        hdr.hdr_signature = htonl(PACK_SIGNATURE);
        hdr.hdr_version = htonl(PACK_VERSION);
@@ -543,6 +597,7 @@ static void write_pack_file(void)
        if (!nr_result)
                goto done;
        for (i = 0; i < nr_objects; i++) {
+               last_obj_offset = offset;
                offset = write_one(f, objects + i, offset);
                if (do_progress) {
                        unsigned percent = written * 100 / nr_result;
@@ -558,18 +613,35 @@ static void write_pack_file(void)
                fputc('\n', stderr);
  done:
        if (written != nr_result)
-               die("wrote %d objects while expecting %d", written, nr_result);
+               die("wrote %u objects while expecting %u", written, nr_result);
        sha1close(f, pack_file_sha1, 1);
+
+       return last_obj_offset;
 }
 
-static void write_index_file(void)
+static uint32_t index_default_version = 1;
+static uint32_t index_off32_limit = 0x7fffffff;
+
+static void write_index_file(off_t last_obj_offset)
 {
-       int i;
+       uint32_t i;
        struct sha1file *f = sha1create("%s-%s.%s", base_name,
                                        sha1_to_hex(object_list_sha1), "idx");
        struct object_entry **list = sorted_by_sha;
        struct object_entry **last = list + nr_result;
        uint32_t array[256];
+       uint32_t index_version;
+
+       /* if last object's offset is >= 2^31 we should use index V2 */
+       index_version = (last_obj_offset >> 31) ? 2 : index_default_version;
+
+       /* index versions 2 and above need a header */
+       if (index_version >= 2) {
+               struct pack_idx_header hdr;
+               hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
+               hdr.idx_version = htonl(index_version);
+               sha1write(f, &hdr, sizeof(hdr));
+       }
 
        /*
         * Write the first-level table (the list is sorted,
@@ -595,10 +667,49 @@ static void write_index_file(void)
        list = sorted_by_sha;
        for (i = 0; i < nr_result; i++) {
                struct object_entry *entry = *list++;
-               uint32_t offset = htonl(entry->offset);
-               sha1write(f, &offset, 4);
+               if (index_version < 2) {
+                       uint32_t offset = htonl(entry->offset);
+                       sha1write(f, &offset, 4);
+               }
                sha1write(f, entry->sha1, 20);
        }
+
+       if (index_version >= 2) {
+               unsigned int nr_large_offset = 0;
+
+               /* write the crc32 table */
+               list = sorted_by_sha;
+               for (i = 0; i < nr_objects; i++) {
+                       struct object_entry *entry = *list++;
+                       uint32_t crc32_val = htonl(entry->crc32);
+                       sha1write(f, &crc32_val, 4);
+               }
+
+               /* write the 32-bit offset table */
+               list = sorted_by_sha;
+               for (i = 0; i < nr_objects; i++) {
+                       struct object_entry *entry = *list++;
+                       uint32_t offset = (entry->offset <= index_off32_limit) ?
+                               entry->offset : (0x80000000 | nr_large_offset++);
+                       offset = htonl(offset);
+                       sha1write(f, &offset, 4);
+               }
+
+               /* write the large offset table */
+               list = sorted_by_sha;
+               while (nr_large_offset) {
+                       struct object_entry *entry = *list++;
+                       uint64_t offset = entry->offset;
+                       if (offset > index_off32_limit) {
+                               uint32_t split[2];
+                               split[0]        = htonl(offset >> 32);
+                               split[1] = htonl(offset & 0xffffffff);
+                               sha1write(f, split, 8);
+                               nr_large_offset--;
+                       }
+               }
+       }
+
        sha1write(f, pack_file_sha1, 20);
        sha1close(f, NULL, 1);
 }
@@ -633,7 +744,7 @@ static struct object_entry *locate_object_entry(const unsigned char *sha1)
 
 static void rehash_objects(void)
 {
-       int i;
+       uint32_t i;
        struct object_entry *oe;
 
        object_ix_hashsz = nr_objects * 3;
@@ -670,16 +781,23 @@ static unsigned name_hash(const char *name)
 
 static int add_object_entry(const unsigned char *sha1, unsigned hash, int exclude)
 {
-       unsigned int idx = nr_objects;
        struct object_entry *entry;
-       struct packed_git *p;
-       unsigned int found_offset = 0;
-       struct packed_git *found_pack = NULL;
-       int ix, status = 0;
+       struct packed_git *p, *found_pack = NULL;
+       off_t found_offset = 0;
+       int ix;
+
+       ix = nr_objects ? locate_object_entry_hash(sha1) : -1;
+       if (ix >= 0) {
+               if (exclude) {
+                       entry = objects + object_ix[ix] - 1;
+                       entry->preferred_base = 1;
+               }
+               return 0;
+       }
 
        if (!exclude) {
                for (p = packed_git; p; p = p->next) {
-                       unsigned long offset = find_pack_entry_one(sha1, p);
+                       off_t offset = find_pack_entry_one(sha1, p);
                        if (offset) {
                                if (incremental)
                                        return 0;
@@ -692,44 +810,34 @@ static int add_object_entry(const unsigned char *sha1, unsigned hash, int exclud
                        }
                }
        }
-       if ((entry = locate_object_entry(sha1)) != NULL)
-               goto already_added;
 
-       if (idx >= nr_alloc) {
-               unsigned int needed = (idx + 1024) * 3 / 2;
-               objects = xrealloc(objects, needed * sizeof(*entry));
-               nr_alloc = needed;
+       if (nr_objects >= nr_alloc) {
+               nr_alloc = (nr_alloc  + 1024) * 3 / 2;
+               objects = xrealloc(objects, nr_alloc * sizeof(*entry));
        }
-       entry = objects + idx;
-       nr_objects = idx + 1;
+
+       entry = objects + nr_objects++;
        memset(entry, 0, sizeof(*entry));
        hashcpy(entry->sha1, sha1);
        entry->hash = hash;
+       if (exclude)
+               entry->preferred_base = 1;
+       if (found_pack) {
+               entry->in_pack = found_pack;
+               entry->in_pack_offset = found_offset;
+       }
 
        if (object_ix_hashsz * 3 <= nr_objects * 4)
                rehash_objects();
-       else {
-               ix = locate_object_entry_hash(entry->sha1);
-               if (0 <= ix)
-                       die("internal error in object hashing.");
-               object_ix[-1 - ix] = idx + 1;
-       }
-       status = 1;
+       else
+               object_ix[-1 - ix] = nr_objects;
 
- already_added:
        if (progress_update) {
-               fprintf(stderr, "Counting objects...%d\r", nr_objects);
+               fprintf(stderr, "Counting objects...%u\r", nr_objects);
                progress_update = 0;
        }
-       if (exclude)
-               entry->preferred_base = 1;
-       else {
-               if (found_pack) {
-                       entry->in_pack = found_pack;
-                       entry->in_pack_offset = found_offset;
-               }
-       }
-       return status;
+
+       return 1;
 }
 
 struct pbase_tree_cache {
@@ -765,7 +873,7 @@ static struct pbase_tree_cache *pbase_tree_get(const unsigned char *sha1)
        struct pbase_tree_cache *ent, *nent;
        void *data;
        unsigned long size;
-       char type[20];
+       enum object_type type;
        int neigh;
        int my_ix = pbase_tree_cache_ix(sha1);
        int available_ix = -1;
@@ -792,10 +900,10 @@ static struct pbase_tree_cache *pbase_tree_get(const unsigned char *sha1)
        /* Did not find one.  Either we got a bogus request or
         * we need to read and perhaps cache.
         */
-       data = read_sha1_file(sha1, type, &size);
+       data = read_sha1_file(sha1, &type, &size);
        if (!data)
                return NULL;
-       if (strcmp(type, tree_type)) {
+       if (type != OBJ_TREE) {
                free(data);
                return NULL;
        }
@@ -851,22 +959,21 @@ static void add_pbase_object(struct tree_desc *tree,
                             const char *fullname)
 {
        struct name_entry entry;
+       int cmp;
 
        while (tree_entry(tree,&entry)) {
-               unsigned long size;
-               char type[20];
-
-               if (entry.pathlen != cmplen ||
-                   memcmp(entry.path, name, cmplen) ||
-                   !has_sha1_file(entry.sha1) ||
-                   sha1_object_info(entry.sha1, type, &size))
+               cmp = tree_entry_len(entry.path, entry.sha1) != cmplen ? 1 :
+                     memcmp(name, entry.path, cmplen);
+               if (cmp > 0)
                        continue;
+               if (cmp < 0)
+                       return;
                if (name[cmplen] != '/') {
                        unsigned hash = name_hash(fullname);
                        add_object_entry(entry.sha1, hash, 1);
                        return;
                }
-               if (!strcmp(type, tree_type)) {
+               if (S_ISDIR(entry.mode)) {
                        struct tree_desc sub;
                        struct pbase_tree_cache *tree;
                        const char *down = name+cmplen+1;
@@ -875,8 +982,7 @@ static void add_pbase_object(struct tree_desc *tree,
                        tree = pbase_tree_get(entry.sha1);
                        if (!tree)
                                return;
-                       sub.buf = tree->tree_data;
-                       sub.size = tree->tree_size;
+                       init_tree_desc(&sub, tree->tree_data, tree->tree_size);
 
                        add_pbase_object(&sub, down, downlen, fullname);
                        pbase_tree_put(tree);
@@ -927,20 +1033,19 @@ static int check_pbase_path(unsigned hash)
 static void add_preferred_base_object(const char *name, unsigned hash)
 {
        struct pbase_tree *it;
-       int cmplen = name_cmp_len(name);
+       int cmplen;
 
-       if (check_pbase_path(hash))
+       if (!num_preferred_base || check_pbase_path(hash))
                return;
 
+       cmplen = name_cmp_len(name);
        for (it = pbase_tree; it; it = it->next) {
                if (cmplen == 0) {
-                       hash = name_hash("");
-                       add_object_entry(it->pcache.sha1, hash, 1);
+                       add_object_entry(it->pcache.sha1, 0, 1);
                }
                else {
                        struct tree_desc tree;
-                       tree.buf = it->pcache.tree_data;
-                       tree.size = it->pcache.tree_size;
+                       init_tree_desc(&tree, it->pcache.tree_data, it->pcache.tree_size);
                        add_pbase_object(&tree, name, cmplen, name);
                }
        }
@@ -978,22 +1083,20 @@ static void add_preferred_base(unsigned char *sha1)
 
 static void check_object(struct object_entry *entry)
 {
-       char type[20];
-
        if (entry->in_pack && !entry->preferred_base) {
                struct packed_git *p = entry->in_pack;
                struct pack_window *w_curs = NULL;
-               unsigned long left = p->pack_size - entry->in_pack_offset;
                unsigned long size, used;
+               unsigned int avail;
                unsigned char *buf;
                struct object_entry *base_entry = NULL;
 
-               buf = use_pack(p, &w_curs, entry->in_pack_offset, NULL);
+               buf = use_pack(p, &w_curs, entry->in_pack_offset, &avail);
 
                /* We want in_pack_type even if we do not reuse delta.
                 * There is no point not reusing non-delta representations.
                 */
-               used = unpack_object_header_gently(buf, left,
+               used = unpack_object_header_gently(buf, avail,
                                                   &entry->in_pack_type, &size);
 
                /* Check if it is delta, and the base is also an object
@@ -1001,8 +1104,9 @@ static void check_object(struct object_entry *entry)
                 * delta.
                 */
                if (!no_reuse_delta) {
-                       unsigned char c, *base_name;
-                       unsigned long ofs;
+                       unsigned char c;
+                       const unsigned char *base_name;
+                       off_t ofs;
                        unsigned long used_0;
                        /* there is at least 20 bytes left in the pack */
                        switch (entry->in_pack_type) {
@@ -1019,7 +1123,7 @@ static void check_object(struct object_entry *entry)
                                ofs = c & 127;
                                while (c & 128) {
                                        ofs += 1;
-                                       if (!ofs || ofs & ~(~0UL >> 7))
+                                       if (!ofs || MSB(ofs, 7))
                                                die("delta base offset overflow in pack for %s",
                                                    sha1_to_hex(entry->sha1));
                                        c = buf[used_0++];
@@ -1062,21 +1166,10 @@ static void check_object(struct object_entry *entry)
                /* Otherwise we would do the usual */
        }
 
-       if (sha1_object_info(entry->sha1, type, &entry->size))
+       entry->type = sha1_object_info(entry->sha1, &entry->size);
+       if (entry->type < 0)
                die("unable to get type of object %s",
                    sha1_to_hex(entry->sha1));
-
-       if (!strcmp(type, commit_type)) {
-               entry->type = OBJ_COMMIT;
-       } else if (!strcmp(type, tree_type)) {
-               entry->type = OBJ_TREE;
-       } else if (!strcmp(type, blob_type)) {
-               entry->type = OBJ_BLOB;
-       } else if (!strcmp(type, tag_type)) {
-               entry->type = OBJ_TAG;
-       } else
-               die("unable to pack object %s of type %s",
-                   sha1_to_hex(entry->sha1), type);
 }
 
 static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
@@ -1094,7 +1187,7 @@ static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
 
 static void get_object_details(void)
 {
-       int i;
+       uint32_t i;
        struct object_entry *entry;
 
        prepare_pack_ix();
@@ -1133,7 +1226,7 @@ static int sort_comparator(const void *_a, const void *_b)
 static struct object_entry **create_sorted_list(entry_sort_t sort)
 {
        struct object_entry **list = xmalloc(nr_objects * sizeof(struct object_entry *));
-       int i;
+       uint32_t i;
 
        for (i = 0; i < nr_objects; i++)
                list[i] = objects + i;
@@ -1150,7 +1243,7 @@ static int sha1_sort(const struct object_entry *a, const struct object_entry *b)
 static struct object_entry **create_final_object_list(void)
 {
        struct object_entry **list;
-       int i, j;
+       uint32_t i, j;
 
        for (i = nr_result = 0; i < nr_objects; i++)
                if (!objects[i].preferred_base)
@@ -1183,7 +1276,7 @@ static int type_size_sort(const struct object_entry *a, const struct object_entr
                return -1;
        if (a->size > b->size)
                return 1;
-       return a < b ? -1 : (a > b);
+       return a > b ? -1 : (a < b);  /* newest last */
 }
 
 struct unpacked {
@@ -1206,7 +1299,7 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
        struct object_entry *trg_entry = trg->entry;
        struct object_entry *src_entry = src->entry;
        unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;
-       char type[10];
+       enum object_type type;
        void *delta_buf;
 
        /* Don't bother doing diffs between different types */
@@ -1257,13 +1350,13 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
 
        /* Load data if not already done */
        if (!trg->data) {
-               trg->data = read_sha1_file(trg_entry->sha1, type, &sz);
+               trg->data = read_sha1_file(trg_entry->sha1, &type, &sz);
                if (sz != trg_size)
                        die("object %s inconsistent object length (%lu vs %lu)",
                            sha1_to_hex(trg_entry->sha1), sz, trg_size);
        }
        if (!src->data) {
-               src->data = read_sha1_file(src_entry->sha1, type, &sz);
+               src->data = read_sha1_file(src_entry->sha1, &type, &sz);
                if (sz != src_size)
                        die("object %s inconsistent object length (%lu vs %lu)",
                            sha1_to_hex(src_entry->sha1), sz, src_size);
@@ -1292,20 +1385,20 @@ static void progress_interval(int signum)
 
 static void find_deltas(struct object_entry **list, int window, int depth)
 {
-       int i, idx;
+       uint32_t i = nr_objects, idx = 0, processed = 0;
        unsigned int array_size = window * sizeof(struct unpacked);
-       struct unpacked *array = xmalloc(array_size);
-       unsigned processed = 0;
+       struct unpacked *array;
        unsigned last_percent = 999;
 
+       if (!nr_objects)
+               return;
+       array = xmalloc(array_size);
        memset(array, 0, array_size);
-       i = nr_objects;
-       idx = 0;
        if (progress)
-               fprintf(stderr, "Deltifying %d objects.\n", nr_result);
+               fprintf(stderr, "Deltifying %u objects.\n", nr_result);
 
-       while (--i >= 0) {
-               struct object_entry *entry = list[i];
+       do {
+               struct object_entry *entry = list[--i];
                struct unpacked *n = array + idx;
                int j;
 
@@ -1338,7 +1431,7 @@ static void find_deltas(struct object_entry **list, int window, int depth)
 
                j = window;
                while (--j > 0) {
-                       unsigned int other_idx = idx + j;
+                       uint32_t other_idx = idx + j;
                        struct unpacked *m;
                        if (other_idx >= window)
                                other_idx -= window;
@@ -1358,7 +1451,7 @@ static void find_deltas(struct object_entry **list, int window, int depth)
                idx++;
                if (idx >= window)
                        idx = 0;
-       }
+       } while (i > 0);
 
        if (progress)
                fputc('\n', stderr);
@@ -1399,7 +1492,7 @@ static int reuse_cached_pack(unsigned char *sha1)
        }
 
        if (progress)
-               fprintf(stderr, "Reusing %d objects pack %s\n", nr_objects,
+               fprintf(stderr, "Reusing %u objects pack %s\n", nr_objects,
                        sha1_to_hex(sha1));
 
        if (pack_to_stdout) {
@@ -1493,9 +1586,7 @@ static void read_object_list_from_stdin(void)
 
 static void show_commit(struct commit *commit)
 {
-       unsigned hash = name_hash("");
-       add_preferred_base_object("", hash);
-       add_object_entry(commit->object.sha1, hash, 0);
+       add_object_entry(commit->object.sha1, 0, 0);
 }
 
 static void show_object(struct object_array_entry *p)
@@ -1550,10 +1641,13 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
        struct object_entry **list;
        int use_internal_rev_list = 0;
        int thin = 0;
-       int i;
-       const char *rp_av[64];
+       uint32_t i;
+       const char **rp_av;
+       int rp_ac_alloc = 64;
        int rp_ac;
 
+       rp_av = xcalloc(rp_ac_alloc, sizeof(*rp_av));
+
        rp_av[0] = "pack-objects";
        rp_av[1] = "--objects"; /* --thin will make it --objects-edge */
        rp_ac = 2;
@@ -1626,8 +1720,11 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
                    !strcmp("--reflog", arg) ||
                    !strcmp("--all", arg)) {
                        use_internal_rev_list = 1;
-                       if (ARRAY_SIZE(rp_av) - 1 <= rp_ac)
-                               die("too many internal rev-list options");
+                       if (rp_ac >= rp_ac_alloc - 1) {
+                               rp_ac_alloc = alloc_nr(rp_ac_alloc);
+                               rp_av = xrealloc(rp_av,
+                                                rp_ac_alloc * sizeof(*rp_av));
+                       }
                        rp_av[rp_ac++] = arg;
                        continue;
                }
@@ -1637,6 +1734,17 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
                        rp_av[1] = "--objects-edge";
                        continue;
                }
+               if (!prefixcmp(arg, "--index-version=")) {
+                       char *c;
+                       index_default_version = strtoul(arg + 16, &c, 10);
+                       if (index_default_version > 2)
+                               die("bad %s", arg);
+                       if (*c == ',')
+                               index_off32_limit = strtoul(c+1, &c, 0);
+                       if (*c || index_off32_limit & 0x80000000)
+                               die("bad %s", arg);
+                       continue;
+               }
                usage(pack_usage);
        }
 
@@ -1677,7 +1785,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
        }
 
        if (progress)
-               fprintf(stderr, "Done counting %d objects.\n", nr_objects);
+               fprintf(stderr, "Done counting %u objects.\n", nr_objects);
        sorted_by_sha = create_final_object_list();
        if (non_empty && !nr_result)
                return 0;
@@ -1690,11 +1798,12 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
        }
        SHA1_Final(object_list_sha1, &ctx);
        if (progress && (nr_objects != nr_result))
-               fprintf(stderr, "Result has %d objects.\n", nr_result);
+               fprintf(stderr, "Result has %u objects.\n", nr_result);
 
        if (reuse_cached_pack(object_list_sha1))
                ;
        else {
+               off_t last_obj_offset;
                if (nr_result)
                        prepare_pack(window, depth);
                if (progress == 1 && pack_to_stdout) {
@@ -1704,14 +1813,14 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
                        signal(SIGALRM, SIG_IGN );
                        progress_update = 0;
                }
-               write_pack_file();
+               last_obj_offset = write_pack_file();
                if (!pack_to_stdout) {
-                       write_index_file();
+                       write_index_file(last_obj_offset);
                        puts(sha1_to_hex(object_list_sha1));
                }
        }
        if (progress)
-               fprintf(stderr, "Total %d (delta %d), reused %d (delta %d)\n",
+               fprintf(stderr, "Total %u (delta %u), reused %u (delta %u)\n",
                        written, written_delta, reused, reused_delta);
        return 0;
 }