d94c79a3390ea458241e46a3c68ef1a3140736cf
   1#include "builtin.h"
   2#include "cache.h"
   3#include "object.h"
   4#include "blob.h"
   5#include "commit.h"
   6#include "tag.h"
   7#include "tree.h"
   8#include "delta.h"
   9#include "pack.h"
  10#include "csum-file.h"
  11#include "tree-walk.h"
  12#include "diff.h"
  13#include "revision.h"
  14#include "list-objects.h"
  15#include "progress.h"
  16
  17static const char pack_usage[] = "\
  18git-pack-objects [{ -q | --progress | --all-progress }] \n\
  19        [--local] [--incremental] [--window=N] [--depth=N] \n\
  20        [--no-reuse-delta] [--no-reuse-object] [--delta-base-offset] \n\
  21        [--non-empty] [--revs [--unpacked | --all]*] [--reflog] \n\
  22        [--stdout | base-name] [<ref-list | <object-list]";
  23
  24struct object_entry {
  25        unsigned char sha1[20];
  26        uint32_t crc32;         /* crc of raw pack data for this object */
  27        off_t offset;           /* offset into the final pack file */
  28        unsigned long size;     /* uncompressed size */
  29        unsigned int hash;      /* name hint hash */
  30        unsigned int depth;     /* delta depth */
  31        struct packed_git *in_pack;     /* already in pack */
  32        off_t in_pack_offset;
  33        struct object_entry *delta;     /* delta base object */
  34        struct object_entry *delta_child; /* deltified objects who bases me */
  35        struct object_entry *delta_sibling; /* other deltified objects who
  36                                             * uses the same base as me
  37                                             */
  38        unsigned long delta_size;       /* delta data size (uncompressed) */
  39        enum object_type type;
  40        enum object_type in_pack_type;  /* could be delta */
  41        unsigned char in_pack_header_size;
  42        unsigned char preferred_base; /* we do not pack this, but is available
  43                                       * to be used as the base objectto delta
  44                                       * objects against.
  45                                       */
  46};
  47
  48/*
  49 * Objects we are going to pack are collected in objects array (dynamically
  50 * expanded).  nr_objects & nr_alloc controls this array.  They are stored
  51 * in the order we see -- typically rev-list --objects order that gives us
  52 * nice "minimum seek" order.
  53 */
  54static struct object_entry *objects;
  55static uint32_t nr_objects, nr_alloc, nr_result;
  56
  57static int non_empty;
  58static int no_reuse_delta, no_reuse_object;
  59static int local;
  60static int incremental;
  61static int allow_ofs_delta;
  62static const char *pack_tmp_name, *idx_tmp_name;
  63static char tmpname[PATH_MAX];
  64static unsigned char pack_file_sha1[20];
  65static int progress = 1;
  66static int window = 10;
  67static int depth = 50;
  68static int pack_to_stdout;
  69static int num_preferred_base;
  70static struct progress progress_state;
  71
  72/*
  73 * The object names in objects array are hashed with this hashtable,
  74 * to help looking up the entry by object name.
  75 * This hashtable is built after all the objects are seen.
  76 */
  77static int *object_ix;
  78static int object_ix_hashsz;
  79
  80/*
  81 * Pack index for existing packs give us easy access to the offsets into
  82 * corresponding pack file where each object's data starts, but the entries
  83 * do not store the size of the compressed representation (uncompressed
  84 * size is easily available by examining the pack entry header).  It is
  85 * also rather expensive to find the sha1 for an object given its offset.
  86 *
  87 * We build a hashtable of existing packs (pack_revindex), and keep reverse
  88 * index here -- pack index file is sorted by object name mapping to offset;
  89 * this pack_revindex[].revindex array is a list of offset/index_nr pairs
  90 * ordered by offset, so if you know the offset of an object, next offset
  91 * is where its packed representation ends and the index_nr can be used to
  92 * get the object sha1 from the main index.
  93 */
  94struct revindex_entry {
  95        off_t offset;
  96        unsigned int nr;
  97};
  98struct pack_revindex {
  99        struct packed_git *p;
 100        struct revindex_entry *revindex;
 101};
 102static struct  pack_revindex *pack_revindex;
 103static int pack_revindex_hashsz;
 104
 105/*
 106 * stats
 107 */
 108static uint32_t written, written_delta;
 109static uint32_t reused, reused_delta;
 110
 111static int pack_revindex_ix(struct packed_git *p)
 112{
 113        unsigned long ui = (unsigned long)p;
 114        int i;
 115
 116        ui = ui ^ (ui >> 16); /* defeat structure alignment */
 117        i = (int)(ui % pack_revindex_hashsz);
 118        while (pack_revindex[i].p) {
 119                if (pack_revindex[i].p == p)
 120                        return i;
 121                if (++i == pack_revindex_hashsz)
 122                        i = 0;
 123        }
 124        return -1 - i;
 125}
 126
 127static void prepare_pack_ix(void)
 128{
 129        int num;
 130        struct packed_git *p;
 131        for (num = 0, p = packed_git; p; p = p->next)
 132                num++;
 133        if (!num)
 134                return;
 135        pack_revindex_hashsz = num * 11;
 136        pack_revindex = xcalloc(sizeof(*pack_revindex), pack_revindex_hashsz);
 137        for (p = packed_git; p; p = p->next) {
 138                num = pack_revindex_ix(p);
 139                num = - 1 - num;
 140                pack_revindex[num].p = p;
 141        }
 142        /* revindex elements are lazily initialized */
 143}
 144
 145static int cmp_offset(const void *a_, const void *b_)
 146{
 147        const struct revindex_entry *a = a_;
 148        const struct revindex_entry *b = b_;
 149        return (a->offset < b->offset) ? -1 : (a->offset > b->offset) ? 1 : 0;
 150}
 151
 152/*
 153 * Ordered list of offsets of objects in the pack.
 154 */
 155static void prepare_pack_revindex(struct pack_revindex *rix)
 156{
 157        struct packed_git *p = rix->p;
 158        int num_ent = p->num_objects;
 159        int i;
 160        const char *index = p->index_data;
 161
 162        rix->revindex = xmalloc(sizeof(*rix->revindex) * (num_ent + 1));
 163        index += 4 * 256;
 164
 165        if (p->index_version > 1) {
 166                const uint32_t *off_32 =
 167                        (uint32_t *)(index + 8 + p->num_objects * (20 + 4));
 168                const uint32_t *off_64 = off_32 + p->num_objects;
 169                for (i = 0; i < num_ent; i++) {
 170                        uint32_t off = ntohl(*off_32++);
 171                        if (!(off & 0x80000000)) {
 172                                rix->revindex[i].offset = off;
 173                        } else {
 174                                rix->revindex[i].offset =
 175                                        ((uint64_t)ntohl(*off_64++)) << 32;
 176                                rix->revindex[i].offset |=
 177                                        ntohl(*off_64++);
 178                        }
 179                        rix->revindex[i].nr = i;
 180                }
 181        } else {
 182                for (i = 0; i < num_ent; i++) {
 183                        uint32_t hl = *((uint32_t *)(index + 24 * i));
 184                        rix->revindex[i].offset = ntohl(hl);
 185                        rix->revindex[i].nr = i;
 186                }
 187        }
 188
 189        /* This knows the pack format -- the 20-byte trailer
 190         * follows immediately after the last object data.
 191         */
 192        rix->revindex[num_ent].offset = p->pack_size - 20;
 193        rix->revindex[num_ent].nr = -1;
 194        qsort(rix->revindex, num_ent, sizeof(*rix->revindex), cmp_offset);
 195}
 196
 197static struct revindex_entry * find_packed_object(struct packed_git *p,
 198                                                  off_t ofs)
 199{
 200        int num;
 201        int lo, hi;
 202        struct pack_revindex *rix;
 203        struct revindex_entry *revindex;
 204        num = pack_revindex_ix(p);
 205        if (num < 0)
 206                die("internal error: pack revindex uninitialized");
 207        rix = &pack_revindex[num];
 208        if (!rix->revindex)
 209                prepare_pack_revindex(rix);
 210        revindex = rix->revindex;
 211        lo = 0;
 212        hi = p->num_objects + 1;
 213        do {
 214                int mi = (lo + hi) / 2;
 215                if (revindex[mi].offset == ofs) {
 216                        return revindex + mi;
 217                }
 218                else if (ofs < revindex[mi].offset)
 219                        hi = mi;
 220                else
 221                        lo = mi + 1;
 222        } while (lo < hi);
 223        die("internal error: pack revindex corrupt");
 224}
 225
 226static const unsigned char *find_packed_object_name(struct packed_git *p,
 227                                                    off_t ofs)
 228{
 229        struct revindex_entry *entry = find_packed_object(p, ofs);
 230        return nth_packed_object_sha1(p, entry->nr);
 231}
 232
 233static void *delta_against(void *buf, unsigned long size, struct object_entry *entry)
 234{
 235        unsigned long othersize, delta_size;
 236        enum object_type type;
 237        void *otherbuf = read_sha1_file(entry->delta->sha1, &type, &othersize);
 238        void *delta_buf;
 239
 240        if (!otherbuf)
 241                die("unable to read %s", sha1_to_hex(entry->delta->sha1));
 242        delta_buf = diff_delta(otherbuf, othersize,
 243                               buf, size, &delta_size, 0);
 244        if (!delta_buf || delta_size != entry->delta_size)
 245                die("delta size changed");
 246        free(buf);
 247        free(otherbuf);
 248        return delta_buf;
 249}
 250
 251/*
 252 * The per-object header is a pretty dense thing, which is
 253 *  - first byte: low four bits are "size", then three bits of "type",
 254 *    and the high bit is "size continues".
 255 *  - each byte afterwards: low seven bits are size continuation,
 256 *    with the high bit being "size continues"
 257 */
 258static int encode_header(enum object_type type, unsigned long size, unsigned char *hdr)
 259{
 260        int n = 1;
 261        unsigned char c;
 262
 263        if (type < OBJ_COMMIT || type > OBJ_REF_DELTA)
 264                die("bad type %d", type);
 265
 266        c = (type << 4) | (size & 15);
 267        size >>= 4;
 268        while (size) {
 269                *hdr++ = c | 0x80;
 270                c = size & 0x7f;
 271                size >>= 7;
 272                n++;
 273        }
 274        *hdr = c;
 275        return n;
 276}
 277
 278/*
 279 * we are going to reuse the existing object data as is.  make
 280 * sure it is not corrupt.
 281 */
 282static int check_pack_inflate(struct packed_git *p,
 283                struct pack_window **w_curs,
 284                off_t offset,
 285                off_t len,
 286                unsigned long expect)
 287{
 288        z_stream stream;
 289        unsigned char fakebuf[4096], *in;
 290        int st;
 291
 292        memset(&stream, 0, sizeof(stream));
 293        inflateInit(&stream);
 294        do {
 295                in = use_pack(p, w_curs, offset, &stream.avail_in);
 296                stream.next_in = in;
 297                stream.next_out = fakebuf;
 298                stream.avail_out = sizeof(fakebuf);
 299                st = inflate(&stream, Z_FINISH);
 300                offset += stream.next_in - in;
 301        } while (st == Z_OK || st == Z_BUF_ERROR);
 302        inflateEnd(&stream);
 303        return (st == Z_STREAM_END &&
 304                stream.total_out == expect &&
 305                stream.total_in == len) ? 0 : -1;
 306}
 307
 308static int check_pack_crc(struct packed_git *p, struct pack_window **w_curs,
 309                          off_t offset, off_t len, unsigned int nr)
 310{
 311        const uint32_t *index_crc;
 312        uint32_t data_crc = crc32(0, Z_NULL, 0);
 313
 314        do {
 315                unsigned int avail;
 316                void *data = use_pack(p, w_curs, offset, &avail);
 317                if (avail > len)
 318                        avail = len;
 319                data_crc = crc32(data_crc, data, avail);
 320                offset += avail;
 321                len -= avail;
 322        } while (len);
 323
 324        index_crc = p->index_data;
 325        index_crc += 2 + 256 + p->num_objects * (20/4) + nr;
 326
 327        return data_crc != ntohl(*index_crc);
 328}
 329
 330static void copy_pack_data(struct sha1file *f,
 331                struct packed_git *p,
 332                struct pack_window **w_curs,
 333                off_t offset,
 334                off_t len)
 335{
 336        unsigned char *in;
 337        unsigned int avail;
 338
 339        while (len) {
 340                in = use_pack(p, w_curs, offset, &avail);
 341                if (avail > len)
 342                        avail = (unsigned int)len;
 343                sha1write(f, in, avail);
 344                offset += avail;
 345                len -= avail;
 346        }
 347}
 348
 349static int check_loose_inflate(unsigned char *data, unsigned long len, unsigned long expect)
 350{
 351        z_stream stream;
 352        unsigned char fakebuf[4096];
 353        int st;
 354
 355        memset(&stream, 0, sizeof(stream));
 356        stream.next_in = data;
 357        stream.avail_in = len;
 358        stream.next_out = fakebuf;
 359        stream.avail_out = sizeof(fakebuf);
 360        inflateInit(&stream);
 361
 362        while (1) {
 363                st = inflate(&stream, Z_FINISH);
 364                if (st == Z_STREAM_END || st == Z_OK) {
 365                        st = (stream.total_out == expect &&
 366                              stream.total_in == len) ? 0 : -1;
 367                        break;
 368                }
 369                if (st != Z_BUF_ERROR) {
 370                        st = -1;
 371                        break;
 372                }
 373                stream.next_out = fakebuf;
 374                stream.avail_out = sizeof(fakebuf);
 375        }
 376        inflateEnd(&stream);
 377        return st;
 378}
 379
 380static int revalidate_loose_object(struct object_entry *entry,
 381                                   unsigned char *map,
 382                                   unsigned long mapsize)
 383{
 384        /* we already know this is a loose object with new type header. */
 385        enum object_type type;
 386        unsigned long size, used;
 387
 388        if (pack_to_stdout)
 389                return 0;
 390
 391        used = unpack_object_header_gently(map, mapsize, &type, &size);
 392        if (!used)
 393                return -1;
 394        map += used;
 395        mapsize -= used;
 396        return check_loose_inflate(map, mapsize, size);
 397}
 398
 399static unsigned long write_object(struct sha1file *f,
 400                                  struct object_entry *entry)
 401{
 402        unsigned long size;
 403        enum object_type type;
 404        void *buf;
 405        unsigned char header[10];
 406        unsigned hdrlen;
 407        off_t datalen;
 408        enum object_type obj_type;
 409        int to_reuse = 0;
 410
 411        if (!pack_to_stdout)
 412                crc32_begin(f);
 413
 414        obj_type = entry->type;
 415        if (no_reuse_object)
 416                to_reuse = 0;   /* explicit */
 417        else if (!entry->in_pack)
 418                to_reuse = 0;   /* can't reuse what we don't have */
 419        else if (obj_type == OBJ_REF_DELTA || obj_type == OBJ_OFS_DELTA)
 420                to_reuse = 1;   /* check_object() decided it for us */
 421        else if (obj_type != entry->in_pack_type)
 422                to_reuse = 0;   /* pack has delta which is unusable */
 423        else if (entry->delta)
 424                to_reuse = 0;   /* we want to pack afresh */
 425        else
 426                to_reuse = 1;   /* we have it in-pack undeltified,
 427                                 * and we do not need to deltify it.
 428                                 */
 429
 430        if (!no_reuse_object && !entry->in_pack && !entry->delta) {
 431                unsigned char *map;
 432                unsigned long mapsize;
 433                map = map_sha1_file(entry->sha1, &mapsize);
 434                if (map && !legacy_loose_object(map)) {
 435                        /* We can copy straight into the pack file */
 436                        if (revalidate_loose_object(entry, map, mapsize))
 437                                die("corrupt loose object %s",
 438                                    sha1_to_hex(entry->sha1));
 439                        sha1write(f, map, mapsize);
 440                        munmap(map, mapsize);
 441                        written++;
 442                        reused++;
 443                        return mapsize;
 444                }
 445                if (map)
 446                        munmap(map, mapsize);
 447        }
 448
 449        if (!to_reuse) {
 450                buf = read_sha1_file(entry->sha1, &type, &size);
 451                if (!buf)
 452                        die("unable to read %s", sha1_to_hex(entry->sha1));
 453                if (size != entry->size)
 454                        die("object %s size inconsistency (%lu vs %lu)",
 455                            sha1_to_hex(entry->sha1), size, entry->size);
 456                if (entry->delta) {
 457                        buf = delta_against(buf, size, entry);
 458                        size = entry->delta_size;
 459                        obj_type = (allow_ofs_delta && entry->delta->offset) ?
 460                                OBJ_OFS_DELTA : OBJ_REF_DELTA;
 461                }
 462                /*
 463                 * The object header is a byte of 'type' followed by zero or
 464                 * more bytes of length.
 465                 */
 466                hdrlen = encode_header(obj_type, size, header);
 467                sha1write(f, header, hdrlen);
 468
 469                if (obj_type == OBJ_OFS_DELTA) {
 470                        /*
 471                         * Deltas with relative base contain an additional
 472                         * encoding of the relative offset for the delta
 473                         * base from this object's position in the pack.
 474                         */
 475                        off_t ofs = entry->offset - entry->delta->offset;
 476                        unsigned pos = sizeof(header) - 1;
 477                        header[pos] = ofs & 127;
 478                        while (ofs >>= 7)
 479                                header[--pos] = 128 | (--ofs & 127);
 480                        sha1write(f, header + pos, sizeof(header) - pos);
 481                        hdrlen += sizeof(header) - pos;
 482                } else if (obj_type == OBJ_REF_DELTA) {
 483                        /*
 484                         * Deltas with a base reference contain
 485                         * an additional 20 bytes for the base sha1.
 486                         */
 487                        sha1write(f, entry->delta->sha1, 20);
 488                        hdrlen += 20;
 489                }
 490                datalen = sha1write_compressed(f, buf, size);
 491                free(buf);
 492        }
 493        else {
 494                struct packed_git *p = entry->in_pack;
 495                struct pack_window *w_curs = NULL;
 496                struct revindex_entry *revidx;
 497                off_t offset;
 498
 499                if (entry->delta) {
 500                        obj_type = (allow_ofs_delta && entry->delta->offset) ?
 501                                OBJ_OFS_DELTA : OBJ_REF_DELTA;
 502                        reused_delta++;
 503                }
 504                hdrlen = encode_header(obj_type, entry->size, header);
 505                sha1write(f, header, hdrlen);
 506                if (obj_type == OBJ_OFS_DELTA) {
 507                        off_t ofs = entry->offset - entry->delta->offset;
 508                        unsigned pos = sizeof(header) - 1;
 509                        header[pos] = ofs & 127;
 510                        while (ofs >>= 7)
 511                                header[--pos] = 128 | (--ofs & 127);
 512                        sha1write(f, header + pos, sizeof(header) - pos);
 513                        hdrlen += sizeof(header) - pos;
 514                } else if (obj_type == OBJ_REF_DELTA) {
 515                        sha1write(f, entry->delta->sha1, 20);
 516                        hdrlen += 20;
 517                }
 518
 519                offset = entry->in_pack_offset;
 520                revidx = find_packed_object(p, offset);
 521                datalen = revidx[1].offset - offset;
 522                if (!pack_to_stdout && p->index_version > 1 &&
 523                    check_pack_crc(p, &w_curs, offset, datalen, revidx->nr))
 524                        die("bad packed object CRC for %s", sha1_to_hex(entry->sha1));
 525                offset += entry->in_pack_header_size;
 526                datalen -= entry->in_pack_header_size;
 527                if (!pack_to_stdout && p->index_version == 1 &&
 528                    check_pack_inflate(p, &w_curs, offset, datalen, entry->size))
 529                        die("corrupt packed object for %s", sha1_to_hex(entry->sha1));
 530                copy_pack_data(f, p, &w_curs, offset, datalen);
 531                unuse_pack(&w_curs);
 532                reused++;
 533        }
 534        if (entry->delta)
 535                written_delta++;
 536        written++;
 537        if (!pack_to_stdout)
 538                entry->crc32 = crc32_end(f);
 539        return hdrlen + datalen;
 540}
 541
 542static off_t write_one(struct sha1file *f,
 543                               struct object_entry *e,
 544                               off_t offset)
 545{
 546        unsigned long size;
 547
 548        /* offset is non zero if object is written already. */
 549        if (e->offset || e->preferred_base)
 550                return offset;
 551
 552        /* if we are deltified, write out base object first. */
 553        if (e->delta)
 554                offset = write_one(f, e->delta, offset);
 555
 556        e->offset = offset;
 557        size = write_object(f, e);
 558
 559        /* make sure off_t is sufficiently large not to wrap */
 560        if (offset > offset + size)
 561                die("pack too large for current definition of off_t");
 562        return offset + size;
 563}
 564
 565static int open_object_dir_tmp(const char *path)
 566{
 567    snprintf(tmpname, sizeof(tmpname), "%s/%s", get_object_directory(), path);
 568    return mkstemp(tmpname);
 569}
 570
 571static off_t write_pack_file(void)
 572{
 573        uint32_t i;
 574        struct sha1file *f;
 575        off_t offset, last_obj_offset = 0;
 576        struct pack_header hdr;
 577        int do_progress = progress;
 578
 579        if (pack_to_stdout) {
 580                f = sha1fd(1, "<stdout>");
 581                do_progress >>= 1;
 582        } else {
 583                int fd = open_object_dir_tmp("tmp_pack_XXXXXX");
 584                if (fd < 0)
 585                        die("unable to create %s: %s\n", tmpname, strerror(errno));
 586                pack_tmp_name = xstrdup(tmpname);
 587                f = sha1fd(fd, pack_tmp_name);
 588        }
 589
 590        if (do_progress)
 591                start_progress(&progress_state, "Writing %u objects...", "", nr_result);
 592
 593        hdr.hdr_signature = htonl(PACK_SIGNATURE);
 594        hdr.hdr_version = htonl(PACK_VERSION);
 595        hdr.hdr_entries = htonl(nr_result);
 596        sha1write(f, &hdr, sizeof(hdr));
 597        offset = sizeof(hdr);
 598        if (!nr_result)
 599                goto done;
 600        for (i = 0; i < nr_objects; i++) {
 601                last_obj_offset = offset;
 602                offset = write_one(f, objects + i, offset);
 603                if (do_progress)
 604                        display_progress(&progress_state, written);
 605        }
 606        if (do_progress)
 607                stop_progress(&progress_state);
 608 done:
 609        if (written != nr_result)
 610                die("wrote %u objects while expecting %u", written, nr_result);
 611        sha1close(f, pack_file_sha1, 1);
 612
 613        return last_obj_offset;
 614}
 615
 616static int sha1_sort(const void *_a, const void *_b)
 617{
 618        const struct object_entry *a = *(struct object_entry **)_a;
 619        const struct object_entry *b = *(struct object_entry **)_b;
 620        return hashcmp(a->sha1, b->sha1);
 621}
 622
 623static uint32_t index_default_version = 1;
 624static uint32_t index_off32_limit = 0x7fffffff;
 625
 626static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
 627{
 628        struct sha1file *f;
 629        struct object_entry **sorted_by_sha, **list, **last;
 630        uint32_t array[256];
 631        uint32_t i, index_version;
 632        SHA_CTX ctx;
 633
 634        int fd = open_object_dir_tmp("tmp_idx_XXXXXX");
 635        if (fd < 0)
 636                die("unable to create %s: %s\n", tmpname, strerror(errno));
 637        idx_tmp_name = xstrdup(tmpname);
 638        f = sha1fd(fd, idx_tmp_name);
 639
 640        if (nr_result) {
 641                uint32_t j = 0;
 642                sorted_by_sha =
 643                        xcalloc(nr_result, sizeof(struct object_entry *));
 644                for (i = 0; i < nr_objects; i++)
 645                        if (!objects[i].preferred_base)
 646                                sorted_by_sha[j++] = objects + i;
 647                if (j != nr_result)
 648                        die("listed %u objects while expecting %u", j, nr_result);
 649                qsort(sorted_by_sha, nr_result, sizeof(*sorted_by_sha), sha1_sort);
 650                list = sorted_by_sha;
 651                last = sorted_by_sha + nr_result;
 652        } else
 653                sorted_by_sha = list = last = NULL;
 654
 655        /* if last object's offset is >= 2^31 we should use index V2 */
 656        index_version = (last_obj_offset >> 31) ? 2 : index_default_version;
 657
 658        /* index versions 2 and above need a header */
 659        if (index_version >= 2) {
 660                struct pack_idx_header hdr;
 661                hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
 662                hdr.idx_version = htonl(index_version);
 663                sha1write(f, &hdr, sizeof(hdr));
 664        }
 665
 666        /*
 667         * Write the first-level table (the list is sorted,
 668         * but we use a 256-entry lookup to be able to avoid
 669         * having to do eight extra binary search iterations).
 670         */
 671        for (i = 0; i < 256; i++) {
 672                struct object_entry **next = list;
 673                while (next < last) {
 674                        struct object_entry *entry = *next;
 675                        if (entry->sha1[0] != i)
 676                                break;
 677                        next++;
 678                }
 679                array[i] = htonl(next - sorted_by_sha);
 680                list = next;
 681        }
 682        sha1write(f, array, 256 * 4);
 683
 684        /* Compute the SHA1 hash of sorted object names. */
 685        SHA1_Init(&ctx);
 686
 687        /* Write the actual SHA1 entries. */
 688        list = sorted_by_sha;
 689        for (i = 0; i < nr_result; i++) {
 690                struct object_entry *entry = *list++;
 691                if (index_version < 2) {
 692                        uint32_t offset = htonl(entry->offset);
 693                        sha1write(f, &offset, 4);
 694                }
 695                sha1write(f, entry->sha1, 20);
 696                SHA1_Update(&ctx, entry->sha1, 20);
 697        }
 698
 699        if (index_version >= 2) {
 700                unsigned int nr_large_offset = 0;
 701
 702                /* write the crc32 table */
 703                list = sorted_by_sha;
 704                for (i = 0; i < nr_objects; i++) {
 705                        struct object_entry *entry = *list++;
 706                        uint32_t crc32_val = htonl(entry->crc32);
 707                        sha1write(f, &crc32_val, 4);
 708                }
 709
 710                /* write the 32-bit offset table */
 711                list = sorted_by_sha;
 712                for (i = 0; i < nr_objects; i++) {
 713                        struct object_entry *entry = *list++;
 714                        uint32_t offset = (entry->offset <= index_off32_limit) ?
 715                                entry->offset : (0x80000000 | nr_large_offset++);
 716                        offset = htonl(offset);
 717                        sha1write(f, &offset, 4);
 718                }
 719
 720                /* write the large offset table */
 721                list = sorted_by_sha;
 722                while (nr_large_offset) {
 723                        struct object_entry *entry = *list++;
 724                        uint64_t offset = entry->offset;
 725                        if (offset > index_off32_limit) {
 726                                uint32_t split[2];
 727                                split[0]        = htonl(offset >> 32);
 728                                split[1] = htonl(offset & 0xffffffff);
 729                                sha1write(f, split, 8);
 730                                nr_large_offset--;
 731                        }
 732                }
 733        }
 734
 735        sha1write(f, pack_file_sha1, 20);
 736        sha1close(f, NULL, 1);
 737        free(sorted_by_sha);
 738        SHA1_Final(sha1, &ctx);
 739}
 740
 741static int locate_object_entry_hash(const unsigned char *sha1)
 742{
 743        int i;
 744        unsigned int ui;
 745        memcpy(&ui, sha1, sizeof(unsigned int));
 746        i = ui % object_ix_hashsz;
 747        while (0 < object_ix[i]) {
 748                if (!hashcmp(sha1, objects[object_ix[i] - 1].sha1))
 749                        return i;
 750                if (++i == object_ix_hashsz)
 751                        i = 0;
 752        }
 753        return -1 - i;
 754}
 755
 756static struct object_entry *locate_object_entry(const unsigned char *sha1)
 757{
 758        int i;
 759
 760        if (!object_ix_hashsz)
 761                return NULL;
 762
 763        i = locate_object_entry_hash(sha1);
 764        if (0 <= i)
 765                return &objects[object_ix[i]-1];
 766        return NULL;
 767}
 768
 769static void rehash_objects(void)
 770{
 771        uint32_t i;
 772        struct object_entry *oe;
 773
 774        object_ix_hashsz = nr_objects * 3;
 775        if (object_ix_hashsz < 1024)
 776                object_ix_hashsz = 1024;
 777        object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
 778        memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
 779        for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
 780                int ix = locate_object_entry_hash(oe->sha1);
 781                if (0 <= ix)
 782                        continue;
 783                ix = -1 - ix;
 784                object_ix[ix] = i + 1;
 785        }
 786}
 787
 788static unsigned name_hash(const char *name)
 789{
 790        unsigned char c;
 791        unsigned hash = 0;
 792
 793        /*
 794         * This effectively just creates a sortable number from the
 795         * last sixteen non-whitespace characters. Last characters
 796         * count "most", so things that end in ".c" sort together.
 797         */
 798        while ((c = *name++) != 0) {
 799                if (isspace(c))
 800                        continue;
 801                hash = (hash >> 2) + (c << 24);
 802        }
 803        return hash;
 804}
 805
 806static int add_object_entry(const unsigned char *sha1, enum object_type type,
 807                            unsigned hash, int exclude)
 808{
 809        struct object_entry *entry;
 810        struct packed_git *p, *found_pack = NULL;
 811        off_t found_offset = 0;
 812        int ix;
 813
 814        ix = nr_objects ? locate_object_entry_hash(sha1) : -1;
 815        if (ix >= 0) {
 816                if (exclude) {
 817                        entry = objects + object_ix[ix] - 1;
 818                        if (!entry->preferred_base)
 819                                nr_result--;
 820                        entry->preferred_base = 1;
 821                }
 822                return 0;
 823        }
 824
 825        for (p = packed_git; p; p = p->next) {
 826                off_t offset = find_pack_entry_one(sha1, p);
 827                if (offset) {
 828                        if (!found_pack) {
 829                                found_offset = offset;
 830                                found_pack = p;
 831                        }
 832                        if (exclude)
 833                                break;
 834                        if (incremental)
 835                                return 0;
 836                        if (local && !p->pack_local)
 837                                return 0;
 838                }
 839        }
 840
 841        if (nr_objects >= nr_alloc) {
 842                nr_alloc = (nr_alloc  + 1024) * 3 / 2;
 843                objects = xrealloc(objects, nr_alloc * sizeof(*entry));
 844        }
 845
 846        entry = objects + nr_objects++;
 847        memset(entry, 0, sizeof(*entry));
 848        hashcpy(entry->sha1, sha1);
 849        entry->hash = hash;
 850        if (type)
 851                entry->type = type;
 852        if (exclude)
 853                entry->preferred_base = 1;
 854        else
 855                nr_result++;
 856        if (found_pack) {
 857                entry->in_pack = found_pack;
 858                entry->in_pack_offset = found_offset;
 859        }
 860
 861        if (object_ix_hashsz * 3 <= nr_objects * 4)
 862                rehash_objects();
 863        else
 864                object_ix[-1 - ix] = nr_objects;
 865
 866        if (progress)
 867                display_progress(&progress_state, nr_objects);
 868
 869        return 1;
 870}
 871
 872struct pbase_tree_cache {
 873        unsigned char sha1[20];
 874        int ref;
 875        int temporary;
 876        void *tree_data;
 877        unsigned long tree_size;
 878};
 879
 880static struct pbase_tree_cache *(pbase_tree_cache[256]);
 881static int pbase_tree_cache_ix(const unsigned char *sha1)
 882{
 883        return sha1[0] % ARRAY_SIZE(pbase_tree_cache);
 884}
 885static int pbase_tree_cache_ix_incr(int ix)
 886{
 887        return (ix+1) % ARRAY_SIZE(pbase_tree_cache);
 888}
 889
 890static struct pbase_tree {
 891        struct pbase_tree *next;
 892        /* This is a phony "cache" entry; we are not
 893         * going to evict it nor find it through _get()
 894         * mechanism -- this is for the toplevel node that
 895         * would almost always change with any commit.
 896         */
 897        struct pbase_tree_cache pcache;
 898} *pbase_tree;
 899
 900static struct pbase_tree_cache *pbase_tree_get(const unsigned char *sha1)
 901{
 902        struct pbase_tree_cache *ent, *nent;
 903        void *data;
 904        unsigned long size;
 905        enum object_type type;
 906        int neigh;
 907        int my_ix = pbase_tree_cache_ix(sha1);
 908        int available_ix = -1;
 909
 910        /* pbase-tree-cache acts as a limited hashtable.
 911         * your object will be found at your index or within a few
 912         * slots after that slot if it is cached.
 913         */
 914        for (neigh = 0; neigh < 8; neigh++) {
 915                ent = pbase_tree_cache[my_ix];
 916                if (ent && !hashcmp(ent->sha1, sha1)) {
 917                        ent->ref++;
 918                        return ent;
 919                }
 920                else if (((available_ix < 0) && (!ent || !ent->ref)) ||
 921                         ((0 <= available_ix) &&
 922                          (!ent && pbase_tree_cache[available_ix])))
 923                        available_ix = my_ix;
 924                if (!ent)
 925                        break;
 926                my_ix = pbase_tree_cache_ix_incr(my_ix);
 927        }
 928
 929        /* Did not find one.  Either we got a bogus request or
 930         * we need to read and perhaps cache.
 931         */
 932        data = read_sha1_file(sha1, &type, &size);
 933        if (!data)
 934                return NULL;
 935        if (type != OBJ_TREE) {
 936                free(data);
 937                return NULL;
 938        }
 939
 940        /* We need to either cache or return a throwaway copy */
 941
 942        if (available_ix < 0)
 943                ent = NULL;
 944        else {
 945                ent = pbase_tree_cache[available_ix];
 946                my_ix = available_ix;
 947        }
 948
 949        if (!ent) {
 950                nent = xmalloc(sizeof(*nent));
 951                nent->temporary = (available_ix < 0);
 952        }
 953        else {
 954                /* evict and reuse */
 955                free(ent->tree_data);
 956                nent = ent;
 957        }
 958        hashcpy(nent->sha1, sha1);
 959        nent->tree_data = data;
 960        nent->tree_size = size;
 961        nent->ref = 1;
 962        if (!nent->temporary)
 963                pbase_tree_cache[my_ix] = nent;
 964        return nent;
 965}
 966
 967static void pbase_tree_put(struct pbase_tree_cache *cache)
 968{
 969        if (!cache->temporary) {
 970                cache->ref--;
 971                return;
 972        }
 973        free(cache->tree_data);
 974        free(cache);
 975}
 976
 977static int name_cmp_len(const char *name)
 978{
 979        int i;
 980        for (i = 0; name[i] && name[i] != '\n' && name[i] != '/'; i++)
 981                ;
 982        return i;
 983}
 984
 985static void add_pbase_object(struct tree_desc *tree,
 986                             const char *name,
 987                             int cmplen,
 988                             const char *fullname)
 989{
 990        struct name_entry entry;
 991        int cmp;
 992
 993        while (tree_entry(tree,&entry)) {
 994                cmp = tree_entry_len(entry.path, entry.sha1) != cmplen ? 1 :
 995                      memcmp(name, entry.path, cmplen);
 996                if (cmp > 0)
 997                        continue;
 998                if (cmp < 0)
 999                        return;
1000                if (name[cmplen] != '/') {
1001                        unsigned hash = name_hash(fullname);
1002                        add_object_entry(entry.sha1,
1003                                         S_ISDIR(entry.mode) ? OBJ_TREE : OBJ_BLOB,
1004                                         hash, 1);
1005                        return;
1006                }
1007                if (S_ISDIR(entry.mode)) {
1008                        struct tree_desc sub;
1009                        struct pbase_tree_cache *tree;
1010                        const char *down = name+cmplen+1;
1011                        int downlen = name_cmp_len(down);
1012
1013                        tree = pbase_tree_get(entry.sha1);
1014                        if (!tree)
1015                                return;
1016                        init_tree_desc(&sub, tree->tree_data, tree->tree_size);
1017
1018                        add_pbase_object(&sub, down, downlen, fullname);
1019                        pbase_tree_put(tree);
1020                }
1021        }
1022}
1023
1024static unsigned *done_pbase_paths;
1025static int done_pbase_paths_num;
1026static int done_pbase_paths_alloc;
1027static int done_pbase_path_pos(unsigned hash)
1028{
1029        int lo = 0;
1030        int hi = done_pbase_paths_num;
1031        while (lo < hi) {
1032                int mi = (hi + lo) / 2;
1033                if (done_pbase_paths[mi] == hash)
1034                        return mi;
1035                if (done_pbase_paths[mi] < hash)
1036                        hi = mi;
1037                else
1038                        lo = mi + 1;
1039        }
1040        return -lo-1;
1041}
1042
1043static int check_pbase_path(unsigned hash)
1044{
1045        int pos = (!done_pbase_paths) ? -1 : done_pbase_path_pos(hash);
1046        if (0 <= pos)
1047                return 1;
1048        pos = -pos - 1;
1049        if (done_pbase_paths_alloc <= done_pbase_paths_num) {
1050                done_pbase_paths_alloc = alloc_nr(done_pbase_paths_alloc);
1051                done_pbase_paths = xrealloc(done_pbase_paths,
1052                                            done_pbase_paths_alloc *
1053                                            sizeof(unsigned));
1054        }
1055        done_pbase_paths_num++;
1056        if (pos < done_pbase_paths_num)
1057                memmove(done_pbase_paths + pos + 1,
1058                        done_pbase_paths + pos,
1059                        (done_pbase_paths_num - pos - 1) * sizeof(unsigned));
1060        done_pbase_paths[pos] = hash;
1061        return 0;
1062}
1063
1064static void add_preferred_base_object(const char *name, unsigned hash)
1065{
1066        struct pbase_tree *it;
1067        int cmplen;
1068
1069        if (!num_preferred_base || check_pbase_path(hash))
1070                return;
1071
1072        cmplen = name_cmp_len(name);
1073        for (it = pbase_tree; it; it = it->next) {
1074                if (cmplen == 0) {
1075                        add_object_entry(it->pcache.sha1, OBJ_TREE, 0, 1);
1076                }
1077                else {
1078                        struct tree_desc tree;
1079                        init_tree_desc(&tree, it->pcache.tree_data, it->pcache.tree_size);
1080                        add_pbase_object(&tree, name, cmplen, name);
1081                }
1082        }
1083}
1084
1085static void add_preferred_base(unsigned char *sha1)
1086{
1087        struct pbase_tree *it;
1088        void *data;
1089        unsigned long size;
1090        unsigned char tree_sha1[20];
1091
1092        if (window <= num_preferred_base++)
1093                return;
1094
1095        data = read_object_with_reference(sha1, tree_type, &size, tree_sha1);
1096        if (!data)
1097                return;
1098
1099        for (it = pbase_tree; it; it = it->next) {
1100                if (!hashcmp(it->pcache.sha1, tree_sha1)) {
1101                        free(data);
1102                        return;
1103                }
1104        }
1105
1106        it = xcalloc(1, sizeof(*it));
1107        it->next = pbase_tree;
1108        pbase_tree = it;
1109
1110        hashcpy(it->pcache.sha1, tree_sha1);
1111        it->pcache.tree_data = data;
1112        it->pcache.tree_size = size;
1113}
1114
1115static void check_object(struct object_entry *entry)
1116{
1117        if (entry->in_pack) {
1118                struct packed_git *p = entry->in_pack;
1119                struct pack_window *w_curs = NULL;
1120                const unsigned char *base_ref = NULL;
1121                struct object_entry *base_entry;
1122                unsigned long used, used_0;
1123                unsigned int avail;
1124                off_t ofs;
1125                unsigned char *buf, c;
1126
1127                buf = use_pack(p, &w_curs, entry->in_pack_offset, &avail);
1128
1129                /*
1130                 * We want in_pack_type even if we do not reuse delta
1131                 * since non-delta representations could still be reused.
1132                 */
1133                used = unpack_object_header_gently(buf, avail,
1134                                                   &entry->in_pack_type,
1135                                                   &entry->size);
1136
1137                /*
1138                 * Determine if this is a delta and if so whether we can
1139                 * reuse it or not.  Otherwise let's find out as cheaply as
1140                 * possible what the actual type and size for this object is.
1141                 */
1142                switch (entry->in_pack_type) {
1143                default:
1144                        /* Not a delta hence we've already got all we need. */
1145                        entry->type = entry->in_pack_type;
1146                        entry->in_pack_header_size = used;
1147                        unuse_pack(&w_curs);
1148                        return;
1149                case OBJ_REF_DELTA:
1150                        if (!no_reuse_delta && !entry->preferred_base)
1151                                base_ref = use_pack(p, &w_curs,
1152                                                entry->in_pack_offset + used, NULL);
1153                        entry->in_pack_header_size = used + 20;
1154                        break;
1155                case OBJ_OFS_DELTA:
1156                        buf = use_pack(p, &w_curs,
1157                                       entry->in_pack_offset + used, NULL);
1158                        used_0 = 0;
1159                        c = buf[used_0++];
1160                        ofs = c & 127;
1161                        while (c & 128) {
1162                                ofs += 1;
1163                                if (!ofs || MSB(ofs, 7))
1164                                        die("delta base offset overflow in pack for %s",
1165                                            sha1_to_hex(entry->sha1));
1166                                c = buf[used_0++];
1167                                ofs = (ofs << 7) + (c & 127);
1168                        }
1169                        if (ofs >= entry->in_pack_offset)
1170                                die("delta base offset out of bound for %s",
1171                                    sha1_to_hex(entry->sha1));
1172                        ofs = entry->in_pack_offset - ofs;
1173                        if (!no_reuse_delta && !entry->preferred_base)
1174                                base_ref = find_packed_object_name(p, ofs);
1175                        entry->in_pack_header_size = used + used_0;
1176                        break;
1177                }
1178
1179                if (base_ref && (base_entry = locate_object_entry(base_ref))) {
1180                        /*
1181                         * If base_ref was set above that means we wish to
1182                         * reuse delta data, and we even found that base
1183                         * in the list of objects we want to pack. Goodie!
1184                         *
1185                         * Depth value does not matter - find_deltas() will
1186                         * never consider reused delta as the base object to
1187                         * deltify other objects against, in order to avoid
1188                         * circular deltas.
1189                         */
1190                        entry->type = entry->in_pack_type;
1191                        entry->delta = base_entry;
1192                        entry->delta_sibling = base_entry->delta_child;
1193                        base_entry->delta_child = entry;
1194                        unuse_pack(&w_curs);
1195                        return;
1196                }
1197
1198                if (entry->type) {
1199                        /*
1200                         * This must be a delta and we already know what the
1201                         * final object type is.  Let's extract the actual
1202                         * object size from the delta header.
1203                         */
1204                        entry->size = get_size_from_delta(p, &w_curs,
1205                                        entry->in_pack_offset + entry->in_pack_header_size);
1206                        unuse_pack(&w_curs);
1207                        return;
1208                }
1209
1210                /*
1211                 * No choice but to fall back to the recursive delta walk
1212                 * with sha1_object_info() to find about the object type
1213                 * at this point...
1214                 */
1215                unuse_pack(&w_curs);
1216        }
1217
1218        entry->type = sha1_object_info(entry->sha1, &entry->size);
1219        if (entry->type < 0)
1220                die("unable to get type of object %s",
1221                    sha1_to_hex(entry->sha1));
1222}
1223
1224static int pack_offset_sort(const void *_a, const void *_b)
1225{
1226        const struct object_entry *a = *(struct object_entry **)_a;
1227        const struct object_entry *b = *(struct object_entry **)_b;
1228
1229        /* avoid filesystem trashing with loose objects */
1230        if (!a->in_pack && !b->in_pack)
1231                return hashcmp(a->sha1, b->sha1);
1232
1233        if (a->in_pack < b->in_pack)
1234                return -1;
1235        if (a->in_pack > b->in_pack)
1236                return 1;
1237        return a->in_pack_offset < b->in_pack_offset ? -1 :
1238                        (a->in_pack_offset > b->in_pack_offset);
1239}
1240
1241static void get_object_details(void)
1242{
1243        uint32_t i;
1244        struct object_entry **sorted_by_offset;
1245
1246        sorted_by_offset = xcalloc(nr_objects, sizeof(struct object_entry *));
1247        for (i = 0; i < nr_objects; i++)
1248                sorted_by_offset[i] = objects + i;
1249        qsort(sorted_by_offset, nr_objects, sizeof(*sorted_by_offset), pack_offset_sort);
1250
1251        prepare_pack_ix();
1252        for (i = 0; i < nr_objects; i++)
1253                check_object(sorted_by_offset[i]);
1254        free(sorted_by_offset);
1255}
1256
1257static int type_size_sort(const void *_a, const void *_b)
1258{
1259        const struct object_entry *a = *(struct object_entry **)_a;
1260        const struct object_entry *b = *(struct object_entry **)_b;
1261
1262        if (a->type < b->type)
1263                return -1;
1264        if (a->type > b->type)
1265                return 1;
1266        if (a->hash < b->hash)
1267                return -1;
1268        if (a->hash > b->hash)
1269                return 1;
1270        if (a->preferred_base < b->preferred_base)
1271                return -1;
1272        if (a->preferred_base > b->preferred_base)
1273                return 1;
1274        if (a->size < b->size)
1275                return -1;
1276        if (a->size > b->size)
1277                return 1;
1278        return a > b ? -1 : (a < b);  /* newest last */
1279}
1280
1281struct unpacked {
1282        struct object_entry *entry;
1283        void *data;
1284        struct delta_index *index;
1285};
1286
1287/*
1288 * We search for deltas _backwards_ in a list sorted by type and
1289 * by size, so that we see progressively smaller and smaller files.
1290 * That's because we prefer deltas to be from the bigger file
1291 * to the smaller - deletes are potentially cheaper, but perhaps
1292 * more importantly, the bigger file is likely the more recent
1293 * one.
1294 */
1295static int try_delta(struct unpacked *trg, struct unpacked *src,
1296                     unsigned max_depth)
1297{
1298        struct object_entry *trg_entry = trg->entry;
1299        struct object_entry *src_entry = src->entry;
1300        unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;
1301        enum object_type type;
1302        void *delta_buf;
1303
1304        /* Don't bother doing diffs between different types */
1305        if (trg_entry->type != src_entry->type)
1306                return -1;
1307
1308        /* We do not compute delta to *create* objects we are not
1309         * going to pack.
1310         */
1311        if (trg_entry->preferred_base)
1312                return -1;
1313
1314        /*
1315         * We do not bother to try a delta that we discarded
1316         * on an earlier try, but only when reusing delta data.
1317         */
1318        if (!no_reuse_delta && trg_entry->in_pack &&
1319            trg_entry->in_pack == src_entry->in_pack &&
1320            trg_entry->in_pack_type != OBJ_REF_DELTA &&
1321            trg_entry->in_pack_type != OBJ_OFS_DELTA)
1322                return 0;
1323
1324        /* Let's not bust the allowed depth. */
1325        if (src_entry->depth >= max_depth)
1326                return 0;
1327
1328        /* Now some size filtering heuristics. */
1329        trg_size = trg_entry->size;
1330        max_size = trg_size/2 - 20;
1331        max_size = max_size * (max_depth - src_entry->depth) / max_depth;
1332        if (max_size == 0)
1333                return 0;
1334        if (trg_entry->delta && trg_entry->delta_size <= max_size)
1335                max_size = trg_entry->delta_size-1;
1336        src_size = src_entry->size;
1337        sizediff = src_size < trg_size ? trg_size - src_size : 0;
1338        if (sizediff >= max_size)
1339                return 0;
1340
1341        /* Load data if not already done */
1342        if (!trg->data) {
1343                trg->data = read_sha1_file(trg_entry->sha1, &type, &sz);
1344                if (sz != trg_size)
1345                        die("object %s inconsistent object length (%lu vs %lu)",
1346                            sha1_to_hex(trg_entry->sha1), sz, trg_size);
1347        }
1348        if (!src->data) {
1349                src->data = read_sha1_file(src_entry->sha1, &type, &sz);
1350                if (sz != src_size)
1351                        die("object %s inconsistent object length (%lu vs %lu)",
1352                            sha1_to_hex(src_entry->sha1), sz, src_size);
1353        }
1354        if (!src->index) {
1355                src->index = create_delta_index(src->data, src_size);
1356                if (!src->index)
1357                        die("out of memory");
1358        }
1359
1360        delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size);
1361        if (!delta_buf)
1362                return 0;
1363
1364        trg_entry->delta = src_entry;
1365        trg_entry->delta_size = delta_size;
1366        trg_entry->depth = src_entry->depth + 1;
1367        free(delta_buf);
1368        return 1;
1369}
1370
1371static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
1372{
1373        struct object_entry *child = me->delta_child;
1374        unsigned int m = n;
1375        while (child) {
1376                unsigned int c = check_delta_limit(child, n + 1);
1377                if (m < c)
1378                        m = c;
1379                child = child->delta_sibling;
1380        }
1381        return m;
1382}
1383
1384static void find_deltas(struct object_entry **list, int window, int depth)
1385{
1386        uint32_t i = nr_objects, idx = 0, processed = 0;
1387        unsigned int array_size = window * sizeof(struct unpacked);
1388        struct unpacked *array;
1389        int max_depth;
1390
1391        if (!nr_objects)
1392                return;
1393        array = xmalloc(array_size);
1394        memset(array, 0, array_size);
1395        if (progress)
1396                start_progress(&progress_state, "Deltifying %u objects...", "", nr_result);
1397
1398        do {
1399                struct object_entry *entry = list[--i];
1400                struct unpacked *n = array + idx;
1401                int j;
1402
1403                if (!entry->preferred_base)
1404                        processed++;
1405
1406                if (progress)
1407                        display_progress(&progress_state, processed);
1408
1409                if (entry->delta)
1410                        /* This happens if we decided to reuse existing
1411                         * delta from a pack.  "!no_reuse_delta &&" is implied.
1412                         */
1413                        continue;
1414
1415                if (entry->size < 50)
1416                        continue;
1417                free_delta_index(n->index);
1418                n->index = NULL;
1419                free(n->data);
1420                n->data = NULL;
1421                n->entry = entry;
1422
1423                /*
1424                 * If the current object is at pack edge, take the depth the
1425                 * objects that depend on the current object into account
1426                 * otherwise they would become too deep.
1427                 */
1428                max_depth = depth;
1429                if (entry->delta_child) {
1430                        max_depth -= check_delta_limit(entry, 0);
1431                        if (max_depth <= 0)
1432                                goto next;
1433                }
1434
1435                j = window;
1436                while (--j > 0) {
1437                        uint32_t other_idx = idx + j;
1438                        struct unpacked *m;
1439                        if (other_idx >= window)
1440                                other_idx -= window;
1441                        m = array + other_idx;
1442                        if (!m->entry)
1443                                break;
1444                        if (try_delta(n, m, max_depth) < 0)
1445                                break;
1446                }
1447
1448                /* if we made n a delta, and if n is already at max
1449                 * depth, leaving it in the window is pointless.  we
1450                 * should evict it first.
1451                 */
1452                if (entry->delta && depth <= entry->depth)
1453                        continue;
1454
1455                next:
1456                idx++;
1457                if (idx >= window)
1458                        idx = 0;
1459        } while (i > 0);
1460
1461        if (progress)
1462                stop_progress(&progress_state);
1463
1464        for (i = 0; i < window; ++i) {
1465                free_delta_index(array[i].index);
1466                free(array[i].data);
1467        }
1468        free(array);
1469}
1470
1471static void prepare_pack(int window, int depth)
1472{
1473        struct object_entry **delta_list;
1474        uint32_t i;
1475
1476        get_object_details();
1477
1478        if (!window || !depth)
1479                return;
1480
1481        delta_list = xmalloc(nr_objects * sizeof(*delta_list));
1482        for (i = 0; i < nr_objects; i++)
1483                delta_list[i] = objects + i;
1484        qsort(delta_list, nr_objects, sizeof(*delta_list), type_size_sort);
1485        find_deltas(delta_list, window+1, depth);
1486        free(delta_list);
1487}
1488
1489static int git_pack_config(const char *k, const char *v)
1490{
1491        if(!strcmp(k, "pack.window")) {
1492                window = git_config_int(k, v);
1493                return 0;
1494        }
1495        if(!strcmp(k, "pack.depth")) {
1496                depth = git_config_int(k, v);
1497                return 0;
1498        }
1499        return git_default_config(k, v);
1500}
1501
1502static void read_object_list_from_stdin(void)
1503{
1504        char line[40 + 1 + PATH_MAX + 2];
1505        unsigned char sha1[20];
1506        unsigned hash;
1507
1508        for (;;) {
1509                if (!fgets(line, sizeof(line), stdin)) {
1510                        if (feof(stdin))
1511                                break;
1512                        if (!ferror(stdin))
1513                                die("fgets returned NULL, not EOF, not error!");
1514                        if (errno != EINTR)
1515                                die("fgets: %s", strerror(errno));
1516                        clearerr(stdin);
1517                        continue;
1518                }
1519                if (line[0] == '-') {
1520                        if (get_sha1_hex(line+1, sha1))
1521                                die("expected edge sha1, got garbage:\n %s",
1522                                    line);
1523                        add_preferred_base(sha1);
1524                        continue;
1525                }
1526                if (get_sha1_hex(line, sha1))
1527                        die("expected sha1, got garbage:\n %s", line);
1528
1529                hash = name_hash(line+41);
1530                add_preferred_base_object(line+41, hash);
1531                add_object_entry(sha1, 0, hash, 0);
1532        }
1533}
1534
1535static void show_commit(struct commit *commit)
1536{
1537        add_object_entry(commit->object.sha1, OBJ_COMMIT, 0, 0);
1538}
1539
1540static void show_object(struct object_array_entry *p)
1541{
1542        unsigned hash = name_hash(p->name);
1543        add_preferred_base_object(p->name, hash);
1544        add_object_entry(p->item->sha1, p->item->type, hash, 0);
1545}
1546
1547static void show_edge(struct commit *commit)
1548{
1549        add_preferred_base(commit->object.sha1);
1550}
1551
1552static void get_object_list(int ac, const char **av)
1553{
1554        struct rev_info revs;
1555        char line[1000];
1556        int flags = 0;
1557
1558        init_revisions(&revs, NULL);
1559        save_commit_buffer = 0;
1560        track_object_refs = 0;
1561        setup_revisions(ac, av, &revs, NULL);
1562
1563        while (fgets(line, sizeof(line), stdin) != NULL) {
1564                int len = strlen(line);
1565                if (line[len - 1] == '\n')
1566                        line[--len] = 0;
1567                if (!len)
1568                        break;
1569                if (*line == '-') {
1570                        if (!strcmp(line, "--not")) {
1571                                flags ^= UNINTERESTING;
1572                                continue;
1573                        }
1574                        die("not a rev '%s'", line);
1575                }
1576                if (handle_revision_arg(line, &revs, flags, 1))
1577                        die("bad revision '%s'", line);
1578        }
1579
1580        prepare_revision_walk(&revs);
1581        mark_edges_uninteresting(revs.commits, &revs, show_edge);
1582        traverse_commit_list(&revs, show_commit, show_object);
1583}
1584
1585static int adjust_perm(const char *path, mode_t mode)
1586{
1587        if (chmod(path, mode))
1588                return -1;
1589        return adjust_shared_perm(path);
1590}
1591
1592int cmd_pack_objects(int argc, const char **argv, const char *prefix)
1593{
1594        int use_internal_rev_list = 0;
1595        int thin = 0;
1596        uint32_t i;
1597        off_t last_obj_offset;
1598        const char *base_name = NULL;
1599        const char **rp_av;
1600        int rp_ac_alloc = 64;
1601        int rp_ac;
1602
1603        rp_av = xcalloc(rp_ac_alloc, sizeof(*rp_av));
1604
1605        rp_av[0] = "pack-objects";
1606        rp_av[1] = "--objects"; /* --thin will make it --objects-edge */
1607        rp_ac = 2;
1608
1609        git_config(git_pack_config);
1610
1611        progress = isatty(2);
1612        for (i = 1; i < argc; i++) {
1613                const char *arg = argv[i];
1614
1615                if (*arg != '-')
1616                        break;
1617
1618                if (!strcmp("--non-empty", arg)) {
1619                        non_empty = 1;
1620                        continue;
1621                }
1622                if (!strcmp("--local", arg)) {
1623                        local = 1;
1624                        continue;
1625                }
1626                if (!strcmp("--incremental", arg)) {
1627                        incremental = 1;
1628                        continue;
1629                }
1630                if (!prefixcmp(arg, "--window=")) {
1631                        char *end;
1632                        window = strtoul(arg+9, &end, 0);
1633                        if (!arg[9] || *end)
1634                                usage(pack_usage);
1635                        continue;
1636                }
1637                if (!prefixcmp(arg, "--depth=")) {
1638                        char *end;
1639                        depth = strtoul(arg+8, &end, 0);
1640                        if (!arg[8] || *end)
1641                                usage(pack_usage);
1642                        continue;
1643                }
1644                if (!strcmp("--progress", arg)) {
1645                        progress = 1;
1646                        continue;
1647                }
1648                if (!strcmp("--all-progress", arg)) {
1649                        progress = 2;
1650                        continue;
1651                }
1652                if (!strcmp("-q", arg)) {
1653                        progress = 0;
1654                        continue;
1655                }
1656                if (!strcmp("--no-reuse-delta", arg)) {
1657                        no_reuse_delta = 1;
1658                        continue;
1659                }
1660                if (!strcmp("--no-reuse-object", arg)) {
1661                        no_reuse_object = no_reuse_delta = 1;
1662                        continue;
1663                }
1664                if (!strcmp("--delta-base-offset", arg)) {
1665                        allow_ofs_delta = 1;
1666                        continue;
1667                }
1668                if (!strcmp("--stdout", arg)) {
1669                        pack_to_stdout = 1;
1670                        continue;
1671                }
1672                if (!strcmp("--revs", arg)) {
1673                        use_internal_rev_list = 1;
1674                        continue;
1675                }
1676                if (!strcmp("--unpacked", arg) ||
1677                    !prefixcmp(arg, "--unpacked=") ||
1678                    !strcmp("--reflog", arg) ||
1679                    !strcmp("--all", arg)) {
1680                        use_internal_rev_list = 1;
1681                        if (rp_ac >= rp_ac_alloc - 1) {
1682                                rp_ac_alloc = alloc_nr(rp_ac_alloc);
1683                                rp_av = xrealloc(rp_av,
1684                                                 rp_ac_alloc * sizeof(*rp_av));
1685                        }
1686                        rp_av[rp_ac++] = arg;
1687                        continue;
1688                }
1689                if (!strcmp("--thin", arg)) {
1690                        use_internal_rev_list = 1;
1691                        thin = 1;
1692                        rp_av[1] = "--objects-edge";
1693                        continue;
1694                }
1695                if (!prefixcmp(arg, "--index-version=")) {
1696                        char *c;
1697                        index_default_version = strtoul(arg + 16, &c, 10);
1698                        if (index_default_version > 2)
1699                                die("bad %s", arg);
1700                        if (*c == ',')
1701                                index_off32_limit = strtoul(c+1, &c, 0);
1702                        if (*c || index_off32_limit & 0x80000000)
1703                                die("bad %s", arg);
1704                        continue;
1705                }
1706                usage(pack_usage);
1707        }
1708
1709        /* Traditionally "pack-objects [options] base extra" failed;
1710         * we would however want to take refs parameter that would
1711         * have been given to upstream rev-list ourselves, which means
1712         * we somehow want to say what the base name is.  So the
1713         * syntax would be:
1714         *
1715         * pack-objects [options] base <refs...>
1716         *
1717         * in other words, we would treat the first non-option as the
1718         * base_name and send everything else to the internal revision
1719         * walker.
1720         */
1721
1722        if (!pack_to_stdout)
1723                base_name = argv[i++];
1724
1725        if (pack_to_stdout != !base_name)
1726                usage(pack_usage);
1727
1728        if (!pack_to_stdout && thin)
1729                die("--thin cannot be used to build an indexable pack.");
1730
1731        prepare_packed_git();
1732
1733        if (progress)
1734                start_progress(&progress_state, "Generating pack...",
1735                               "Counting objects: ", 0);
1736        if (!use_internal_rev_list)
1737                read_object_list_from_stdin();
1738        else {
1739                rp_av[rp_ac] = NULL;
1740                get_object_list(rp_ac, rp_av);
1741        }
1742        if (progress) {
1743                stop_progress(&progress_state);
1744                fprintf(stderr, "Done counting %u objects.\n", nr_objects);
1745        }
1746
1747        if (non_empty && !nr_result)
1748                return 0;
1749        if (progress && (nr_objects != nr_result))
1750                fprintf(stderr, "Result has %u objects.\n", nr_result);
1751        if (nr_result)
1752                prepare_pack(window, depth);
1753        last_obj_offset = write_pack_file();
1754        if (!pack_to_stdout) {
1755                unsigned char object_list_sha1[20];
1756                mode_t mode = umask(0);
1757
1758                umask(mode);
1759                mode = 0444 & ~mode;
1760
1761                write_index_file(last_obj_offset, object_list_sha1);
1762                snprintf(tmpname, sizeof(tmpname), "%s-%s.pack",
1763                         base_name, sha1_to_hex(object_list_sha1));
1764                if (adjust_perm(pack_tmp_name, mode))
1765                        die("unable to make temporary pack file readable: %s",
1766                            strerror(errno));
1767                if (rename(pack_tmp_name, tmpname))
1768                        die("unable to rename temporary pack file: %s",
1769                            strerror(errno));
1770                snprintf(tmpname, sizeof(tmpname), "%s-%s.idx",
1771                         base_name, sha1_to_hex(object_list_sha1));
1772                if (adjust_perm(idx_tmp_name, mode))
1773                        die("unable to make temporary index file readable: %s",
1774                            strerror(errno));
1775                if (rename(idx_tmp_name, tmpname))
1776                        die("unable to rename temporary index file: %s",
1777                            strerror(errno));
1778                puts(sha1_to_hex(object_list_sha1));
1779        }
1780        if (progress)
1781                fprintf(stderr, "Total %u (delta %u), reused %u (delta %u)\n",
1782                        written, written_delta, reused, reused_delta);
1783        return 0;
1784}