sha1_file.con commit git-svn: avoid filling up the disk with temp files. (29c70e0)
   1/*
   2 * GIT - The information manager from hell
   3 *
   4 * Copyright (C) Linus Torvalds, 2005
   5 *
   6 * This handles basic git sha1 object files - packing, unpacking,
   7 * creation etc.
   8 */
   9#include "cache.h"
  10#include "delta.h"
  11#include "pack.h"
  12#include "blob.h"
  13#include "commit.h"
  14#include "tag.h"
  15#include "tree.h"
  16#include "refs.h"
  17#include "pack-revindex.h"
  18#include "sha1-lookup.h"
  19
  20#ifndef O_NOATIME
  21#if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
  22#define O_NOATIME 01000000
  23#else
  24#define O_NOATIME 0
  25#endif
  26#endif
  27
  28#ifdef NO_C99_FORMAT
  29#define SZ_FMT "lu"
  30static unsigned long sz_fmt(size_t s) { return (unsigned long)s; }
  31#else
  32#define SZ_FMT "zu"
  33static size_t sz_fmt(size_t s) { return s; }
  34#endif
  35
  36const unsigned char null_sha1[20];
  37
  38const signed char hexval_table[256] = {
  39         -1, -1, -1, -1, -1, -1, -1, -1,                /* 00-07 */
  40         -1, -1, -1, -1, -1, -1, -1, -1,                /* 08-0f */
  41         -1, -1, -1, -1, -1, -1, -1, -1,                /* 10-17 */
  42         -1, -1, -1, -1, -1, -1, -1, -1,                /* 18-1f */
  43         -1, -1, -1, -1, -1, -1, -1, -1,                /* 20-27 */
  44         -1, -1, -1, -1, -1, -1, -1, -1,                /* 28-2f */
  45          0,  1,  2,  3,  4,  5,  6,  7,                /* 30-37 */
  46          8,  9, -1, -1, -1, -1, -1, -1,                /* 38-3f */
  47         -1, 10, 11, 12, 13, 14, 15, -1,                /* 40-47 */
  48         -1, -1, -1, -1, -1, -1, -1, -1,                /* 48-4f */
  49         -1, -1, -1, -1, -1, -1, -1, -1,                /* 50-57 */
  50         -1, -1, -1, -1, -1, -1, -1, -1,                /* 58-5f */
  51         -1, 10, 11, 12, 13, 14, 15, -1,                /* 60-67 */
  52         -1, -1, -1, -1, -1, -1, -1, -1,                /* 68-67 */
  53         -1, -1, -1, -1, -1, -1, -1, -1,                /* 70-77 */
  54         -1, -1, -1, -1, -1, -1, -1, -1,                /* 78-7f */
  55         -1, -1, -1, -1, -1, -1, -1, -1,                /* 80-87 */
  56         -1, -1, -1, -1, -1, -1, -1, -1,                /* 88-8f */
  57         -1, -1, -1, -1, -1, -1, -1, -1,                /* 90-97 */
  58         -1, -1, -1, -1, -1, -1, -1, -1,                /* 98-9f */
  59         -1, -1, -1, -1, -1, -1, -1, -1,                /* a0-a7 */
  60         -1, -1, -1, -1, -1, -1, -1, -1,                /* a8-af */
  61         -1, -1, -1, -1, -1, -1, -1, -1,                /* b0-b7 */
  62         -1, -1, -1, -1, -1, -1, -1, -1,                /* b8-bf */
  63         -1, -1, -1, -1, -1, -1, -1, -1,                /* c0-c7 */
  64         -1, -1, -1, -1, -1, -1, -1, -1,                /* c8-cf */
  65         -1, -1, -1, -1, -1, -1, -1, -1,                /* d0-d7 */
  66         -1, -1, -1, -1, -1, -1, -1, -1,                /* d8-df */
  67         -1, -1, -1, -1, -1, -1, -1, -1,                /* e0-e7 */
  68         -1, -1, -1, -1, -1, -1, -1, -1,                /* e8-ef */
  69         -1, -1, -1, -1, -1, -1, -1, -1,                /* f0-f7 */
  70         -1, -1, -1, -1, -1, -1, -1, -1,                /* f8-ff */
  71};
  72
  73int get_sha1_hex(const char *hex, unsigned char *sha1)
  74{
  75        int i;
  76        for (i = 0; i < 20; i++) {
  77                unsigned int val = (hexval(hex[0]) << 4) | hexval(hex[1]);
  78                if (val & ~0xff)
  79                        return -1;
  80                *sha1++ = val;
  81                hex += 2;
  82        }
  83        return 0;
  84}
  85
  86int safe_create_leading_directories(char *path)
  87{
  88        char *pos = path;
  89        struct stat st;
  90
  91        if (is_absolute_path(path))
  92                pos++;
  93
  94        while (pos) {
  95                pos = strchr(pos, '/');
  96                if (!pos)
  97                        break;
  98                *pos = 0;
  99                if (!stat(path, &st)) {
 100                        /* path exists */
 101                        if (!S_ISDIR(st.st_mode)) {
 102                                *pos = '/';
 103                                return -3;
 104                        }
 105                }
 106                else if (mkdir(path, 0777)) {
 107                        *pos = '/';
 108                        return -1;
 109                }
 110                else if (adjust_shared_perm(path)) {
 111                        *pos = '/';
 112                        return -2;
 113                }
 114                *pos++ = '/';
 115        }
 116        return 0;
 117}
 118
 119int safe_create_leading_directories_const(const char *path)
 120{
 121        /* path points to cache entries, so xstrdup before messing with it */
 122        char *buf = xstrdup(path);
 123        int result = safe_create_leading_directories(buf);
 124        free(buf);
 125        return result;
 126}
 127
 128char *sha1_to_hex(const unsigned char *sha1)
 129{
 130        static int bufno;
 131        static char hexbuffer[4][50];
 132        static const char hex[] = "0123456789abcdef";
 133        char *buffer = hexbuffer[3 & ++bufno], *buf = buffer;
 134        int i;
 135
 136        for (i = 0; i < 20; i++) {
 137                unsigned int val = *sha1++;
 138                *buf++ = hex[val >> 4];
 139                *buf++ = hex[val & 0xf];
 140        }
 141        *buf = '\0';
 142
 143        return buffer;
 144}
 145
 146static void fill_sha1_path(char *pathbuf, const unsigned char *sha1)
 147{
 148        int i;
 149        for (i = 0; i < 20; i++) {
 150                static char hex[] = "0123456789abcdef";
 151                unsigned int val = sha1[i];
 152                char *pos = pathbuf + i*2 + (i > 0);
 153                *pos++ = hex[val >> 4];
 154                *pos = hex[val & 0xf];
 155        }
 156}
 157
 158/*
 159 * NOTE! This returns a statically allocated buffer, so you have to be
 160 * careful about using it. Do an "xstrdup()" if you need to save the
 161 * filename.
 162 *
 163 * Also note that this returns the location for creating.  Reading
 164 * SHA1 file can happen from any alternate directory listed in the
 165 * DB_ENVIRONMENT environment variable if it is not found in
 166 * the primary object database.
 167 */
 168char *sha1_file_name(const unsigned char *sha1)
 169{
 170        static char *name, *base;
 171
 172        if (!base) {
 173                const char *sha1_file_directory = get_object_directory();
 174                int len = strlen(sha1_file_directory);
 175                base = xmalloc(len + 60);
 176                memcpy(base, sha1_file_directory, len);
 177                memset(base+len, 0, 60);
 178                base[len] = '/';
 179                base[len+3] = '/';
 180                name = base + len + 1;
 181        }
 182        fill_sha1_path(name, sha1);
 183        return base;
 184}
 185
 186static char *sha1_get_pack_name(const unsigned char *sha1,
 187                                char **name, char **base, const char *which)
 188{
 189        static const char hex[] = "0123456789abcdef";
 190        char *buf;
 191        int i;
 192
 193        if (!*base) {
 194                const char *sha1_file_directory = get_object_directory();
 195                int len = strlen(sha1_file_directory);
 196                *base = xmalloc(len + 60);
 197                sprintf(*base, "%s/pack/pack-1234567890123456789012345678901234567890.%s",
 198                        sha1_file_directory, which);
 199                *name = *base + len + 11;
 200        }
 201
 202        buf = *name;
 203
 204        for (i = 0; i < 20; i++) {
 205                unsigned int val = *sha1++;
 206                *buf++ = hex[val >> 4];
 207                *buf++ = hex[val & 0xf];
 208        }
 209
 210        return *base;
 211}
 212
 213char *sha1_pack_name(const unsigned char *sha1)
 214{
 215        static char *name, *base;
 216
 217        return sha1_get_pack_name(sha1, &name, &base, "pack");
 218}
 219
 220char *sha1_pack_index_name(const unsigned char *sha1)
 221{
 222        static char *name, *base;
 223
 224        return sha1_get_pack_name(sha1, &name, &base, "idx");
 225}
 226
 227struct alternate_object_database *alt_odb_list;
 228static struct alternate_object_database **alt_odb_tail;
 229
 230static void read_info_alternates(const char * alternates, int depth);
 231
 232/*
 233 * Prepare alternate object database registry.
 234 *
 235 * The variable alt_odb_list points at the list of struct
 236 * alternate_object_database.  The elements on this list come from
 237 * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
 238 * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
 239 * whose contents is similar to that environment variable but can be
 240 * LF separated.  Its base points at a statically allocated buffer that
 241 * contains "/the/directory/corresponding/to/.git/objects/...", while
 242 * its name points just after the slash at the end of ".git/objects/"
 243 * in the example above, and has enough space to hold 40-byte hex
 244 * SHA1, an extra slash for the first level indirection, and the
 245 * terminating NUL.
 246 */
 247static int link_alt_odb_entry(const char * entry, int len, const char * relative_base, int depth)
 248{
 249        struct stat st;
 250        const char *objdir = get_object_directory();
 251        struct alternate_object_database *ent;
 252        struct alternate_object_database *alt;
 253        /* 43 = 40-byte + 2 '/' + terminating NUL */
 254        int pfxlen = len;
 255        int entlen = pfxlen + 43;
 256        int base_len = -1;
 257
 258        if (!is_absolute_path(entry) && relative_base) {
 259                /* Relative alt-odb */
 260                if (base_len < 0)
 261                        base_len = strlen(relative_base) + 1;
 262                entlen += base_len;
 263                pfxlen += base_len;
 264        }
 265        ent = xmalloc(sizeof(*ent) + entlen);
 266
 267        if (!is_absolute_path(entry) && relative_base) {
 268                memcpy(ent->base, relative_base, base_len - 1);
 269                ent->base[base_len - 1] = '/';
 270                memcpy(ent->base + base_len, entry, len);
 271        }
 272        else
 273                memcpy(ent->base, entry, pfxlen);
 274
 275        ent->name = ent->base + pfxlen + 1;
 276        ent->base[pfxlen + 3] = '/';
 277        ent->base[pfxlen] = ent->base[entlen-1] = 0;
 278
 279        /* Detect cases where alternate disappeared */
 280        if (stat(ent->base, &st) || !S_ISDIR(st.st_mode)) {
 281                error("object directory %s does not exist; "
 282                      "check .git/objects/info/alternates.",
 283                      ent->base);
 284                free(ent);
 285                return -1;
 286        }
 287
 288        /* Prevent the common mistake of listing the same
 289         * thing twice, or object directory itself.
 290         */
 291        for (alt = alt_odb_list; alt; alt = alt->next) {
 292                if (!memcmp(ent->base, alt->base, pfxlen)) {
 293                        free(ent);
 294                        return -1;
 295                }
 296        }
 297        if (!memcmp(ent->base, objdir, pfxlen)) {
 298                free(ent);
 299                return -1;
 300        }
 301
 302        /* add the alternate entry */
 303        *alt_odb_tail = ent;
 304        alt_odb_tail = &(ent->next);
 305        ent->next = NULL;
 306
 307        /* recursively add alternates */
 308        read_info_alternates(ent->base, depth + 1);
 309
 310        ent->base[pfxlen] = '/';
 311
 312        return 0;
 313}
 314
 315static void link_alt_odb_entries(const char *alt, const char *ep, int sep,
 316                                 const char *relative_base, int depth)
 317{
 318        const char *cp, *last;
 319
 320        if (depth > 5) {
 321                error("%s: ignoring alternate object stores, nesting too deep.",
 322                                relative_base);
 323                return;
 324        }
 325
 326        last = alt;
 327        while (last < ep) {
 328                cp = last;
 329                if (cp < ep && *cp == '#') {
 330                        while (cp < ep && *cp != sep)
 331                                cp++;
 332                        last = cp + 1;
 333                        continue;
 334                }
 335                while (cp < ep && *cp != sep)
 336                        cp++;
 337                if (last != cp) {
 338                        if (!is_absolute_path(last) && depth) {
 339                                error("%s: ignoring relative alternate object store %s",
 340                                                relative_base, last);
 341                        } else {
 342                                link_alt_odb_entry(last, cp - last,
 343                                                relative_base, depth);
 344                        }
 345                }
 346                while (cp < ep && *cp == sep)
 347                        cp++;
 348                last = cp;
 349        }
 350}
 351
 352static void read_info_alternates(const char * relative_base, int depth)
 353{
 354        char *map;
 355        size_t mapsz;
 356        struct stat st;
 357        const char alt_file_name[] = "info/alternates";
 358        /* Given that relative_base is no longer than PATH_MAX,
 359           ensure that "path" has enough space to append "/", the
 360           file name, "info/alternates", and a trailing NUL.  */
 361        char path[PATH_MAX + 1 + sizeof alt_file_name];
 362        int fd;
 363
 364        sprintf(path, "%s/%s", relative_base, alt_file_name);
 365        fd = open(path, O_RDONLY);
 366        if (fd < 0)
 367                return;
 368        if (fstat(fd, &st) || (st.st_size == 0)) {
 369                close(fd);
 370                return;
 371        }
 372        mapsz = xsize_t(st.st_size);
 373        map = xmmap(NULL, mapsz, PROT_READ, MAP_PRIVATE, fd, 0);
 374        close(fd);
 375
 376        link_alt_odb_entries(map, map + mapsz, '\n', relative_base, depth);
 377
 378        munmap(map, mapsz);
 379}
 380
 381void add_to_alternates_file(const char *reference)
 382{
 383        struct lock_file *lock = xcalloc(1, sizeof(struct lock_file));
 384        int fd = hold_lock_file_for_append(lock, git_path("objects/info/alternates"), 1);
 385        char *alt = mkpath("%s/objects\n", reference);
 386        write_or_die(fd, alt, strlen(alt));
 387        if (commit_lock_file(lock))
 388                die("could not close alternates file");
 389        if (alt_odb_tail)
 390                link_alt_odb_entries(alt, alt + strlen(alt), '\n', NULL, 0);
 391}
 392
 393void prepare_alt_odb(void)
 394{
 395        const char *alt;
 396
 397        if (alt_odb_tail)
 398                return;
 399
 400        alt = getenv(ALTERNATE_DB_ENVIRONMENT);
 401        if (!alt) alt = "";
 402
 403        alt_odb_tail = &alt_odb_list;
 404        link_alt_odb_entries(alt, alt + strlen(alt), ':', NULL, 0);
 405
 406        read_info_alternates(get_object_directory(), 0);
 407}
 408
 409static int has_loose_object(const unsigned char *sha1)
 410{
 411        char *name = sha1_file_name(sha1);
 412        struct alternate_object_database *alt;
 413
 414        if (!access(name, F_OK))
 415                return 1;
 416        prepare_alt_odb();
 417        for (alt = alt_odb_list; alt; alt = alt->next) {
 418                name = alt->name;
 419                fill_sha1_path(name, sha1);
 420                if (!access(alt->base, F_OK))
 421                        return 1;
 422        }
 423        return 0;
 424}
 425
 426static unsigned int pack_used_ctr;
 427static unsigned int pack_mmap_calls;
 428static unsigned int peak_pack_open_windows;
 429static unsigned int pack_open_windows;
 430static size_t peak_pack_mapped;
 431static size_t pack_mapped;
 432struct packed_git *packed_git;
 433
 434void pack_report(void)
 435{
 436        fprintf(stderr,
 437                "pack_report: getpagesize()            = %10" SZ_FMT "\n"
 438                "pack_report: core.packedGitWindowSize = %10" SZ_FMT "\n"
 439                "pack_report: core.packedGitLimit      = %10" SZ_FMT "\n",
 440                sz_fmt(getpagesize()),
 441                sz_fmt(packed_git_window_size),
 442                sz_fmt(packed_git_limit));
 443        fprintf(stderr,
 444                "pack_report: pack_used_ctr            = %10u\n"
 445                "pack_report: pack_mmap_calls          = %10u\n"
 446                "pack_report: pack_open_windows        = %10u / %10u\n"
 447                "pack_report: pack_mapped              = "
 448                        "%10" SZ_FMT " / %10" SZ_FMT "\n",
 449                pack_used_ctr,
 450                pack_mmap_calls,
 451                pack_open_windows, peak_pack_open_windows,
 452                sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped));
 453}
 454
 455static int check_packed_git_idx(const char *path,  struct packed_git *p)
 456{
 457        void *idx_map;
 458        struct pack_idx_header *hdr;
 459        size_t idx_size;
 460        uint32_t version, nr, i, *index;
 461        int fd = open(path, O_RDONLY);
 462        struct stat st;
 463
 464        if (fd < 0)
 465                return -1;
 466        if (fstat(fd, &st)) {
 467                close(fd);
 468                return -1;
 469        }
 470        idx_size = xsize_t(st.st_size);
 471        if (idx_size < 4 * 256 + 20 + 20) {
 472                close(fd);
 473                return error("index file %s is too small", path);
 474        }
 475        idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
 476        close(fd);
 477
 478        hdr = idx_map;
 479        if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
 480                version = ntohl(hdr->idx_version);
 481                if (version < 2 || version > 2) {
 482                        munmap(idx_map, idx_size);
 483                        return error("index file %s is version %d"
 484                                     " and is not supported by this binary"
 485                                     " (try upgrading GIT to a newer version)",
 486                                     path, version);
 487                }
 488        } else
 489                version = 1;
 490
 491        nr = 0;
 492        index = idx_map;
 493        if (version > 1)
 494                index += 2;  /* skip index header */
 495        for (i = 0; i < 256; i++) {
 496                uint32_t n = ntohl(index[i]);
 497                if (n < nr) {
 498                        munmap(idx_map, idx_size);
 499                        return error("non-monotonic index %s", path);
 500                }
 501                nr = n;
 502        }
 503
 504        if (version == 1) {
 505                /*
 506                 * Total size:
 507                 *  - 256 index entries 4 bytes each
 508                 *  - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
 509                 *  - 20-byte SHA1 of the packfile
 510                 *  - 20-byte SHA1 file checksum
 511                 */
 512                if (idx_size != 4*256 + nr * 24 + 20 + 20) {
 513                        munmap(idx_map, idx_size);
 514                        return error("wrong index v1 file size in %s", path);
 515                }
 516        } else if (version == 2) {
 517                /*
 518                 * Minimum size:
 519                 *  - 8 bytes of header
 520                 *  - 256 index entries 4 bytes each
 521                 *  - 20-byte sha1 entry * nr
 522                 *  - 4-byte crc entry * nr
 523                 *  - 4-byte offset entry * nr
 524                 *  - 20-byte SHA1 of the packfile
 525                 *  - 20-byte SHA1 file checksum
 526                 * And after the 4-byte offset table might be a
 527                 * variable sized table containing 8-byte entries
 528                 * for offsets larger than 2^31.
 529                 */
 530                unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
 531                unsigned long max_size = min_size;
 532                if (nr)
 533                        max_size += (nr - 1)*8;
 534                if (idx_size < min_size || idx_size > max_size) {
 535                        munmap(idx_map, idx_size);
 536                        return error("wrong index v2 file size in %s", path);
 537                }
 538                if (idx_size != min_size &&
 539                    /*
 540                     * make sure we can deal with large pack offsets.
 541                     * 31-bit signed offset won't be enough, neither
 542                     * 32-bit unsigned one will be.
 543                     */
 544                    (sizeof(off_t) <= 4)) {
 545                        munmap(idx_map, idx_size);
 546                        return error("pack too large for current definition of off_t in %s", path);
 547                }
 548        }
 549
 550        p->index_version = version;
 551        p->index_data = idx_map;
 552        p->index_size = idx_size;
 553        p->num_objects = nr;
 554        return 0;
 555}
 556
 557int open_pack_index(struct packed_git *p)
 558{
 559        char *idx_name;
 560        int ret;
 561
 562        if (p->index_data)
 563                return 0;
 564
 565        idx_name = xstrdup(p->pack_name);
 566        strcpy(idx_name + strlen(idx_name) - strlen(".pack"), ".idx");
 567        ret = check_packed_git_idx(idx_name, p);
 568        free(idx_name);
 569        return ret;
 570}
 571
 572static void scan_windows(struct packed_git *p,
 573        struct packed_git **lru_p,
 574        struct pack_window **lru_w,
 575        struct pack_window **lru_l)
 576{
 577        struct pack_window *w, *w_l;
 578
 579        for (w_l = NULL, w = p->windows; w; w = w->next) {
 580                if (!w->inuse_cnt) {
 581                        if (!*lru_w || w->last_used < (*lru_w)->last_used) {
 582                                *lru_p = p;
 583                                *lru_w = w;
 584                                *lru_l = w_l;
 585                        }
 586                }
 587                w_l = w;
 588        }
 589}
 590
 591static int unuse_one_window(struct packed_git *current, int keep_fd)
 592{
 593        struct packed_git *p, *lru_p = NULL;
 594        struct pack_window *lru_w = NULL, *lru_l = NULL;
 595
 596        if (current)
 597                scan_windows(current, &lru_p, &lru_w, &lru_l);
 598        for (p = packed_git; p; p = p->next)
 599                scan_windows(p, &lru_p, &lru_w, &lru_l);
 600        if (lru_p) {
 601                munmap(lru_w->base, lru_w->len);
 602                pack_mapped -= lru_w->len;
 603                if (lru_l)
 604                        lru_l->next = lru_w->next;
 605                else {
 606                        lru_p->windows = lru_w->next;
 607                        if (!lru_p->windows && lru_p->pack_fd != keep_fd) {
 608                                close(lru_p->pack_fd);
 609                                lru_p->pack_fd = -1;
 610                        }
 611                }
 612                free(lru_w);
 613                pack_open_windows--;
 614                return 1;
 615        }
 616        return 0;
 617}
 618
 619void release_pack_memory(size_t need, int fd)
 620{
 621        size_t cur = pack_mapped;
 622        while (need >= (cur - pack_mapped) && unuse_one_window(NULL, fd))
 623                ; /* nothing */
 624}
 625
 626void close_pack_windows(struct packed_git *p)
 627{
 628        while (p->windows) {
 629                struct pack_window *w = p->windows;
 630
 631                if (w->inuse_cnt)
 632                        die("pack '%s' still has open windows to it",
 633                            p->pack_name);
 634                munmap(w->base, w->len);
 635                pack_mapped -= w->len;
 636                pack_open_windows--;
 637                p->windows = w->next;
 638                free(w);
 639        }
 640}
 641
 642void unuse_pack(struct pack_window **w_cursor)
 643{
 644        struct pack_window *w = *w_cursor;
 645        if (w) {
 646                w->inuse_cnt--;
 647                *w_cursor = NULL;
 648        }
 649}
 650
 651/*
 652 * Do not call this directly as this leaks p->pack_fd on error return;
 653 * call open_packed_git() instead.
 654 */
 655static int open_packed_git_1(struct packed_git *p)
 656{
 657        struct stat st;
 658        struct pack_header hdr;
 659        unsigned char sha1[20];
 660        unsigned char *idx_sha1;
 661        long fd_flag;
 662
 663        if (!p->index_data && open_pack_index(p))
 664                return error("packfile %s index unavailable", p->pack_name);
 665
 666        p->pack_fd = open(p->pack_name, O_RDONLY);
 667        if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
 668                return -1;
 669
 670        /* If we created the struct before we had the pack we lack size. */
 671        if (!p->pack_size) {
 672                if (!S_ISREG(st.st_mode))
 673                        return error("packfile %s not a regular file", p->pack_name);
 674                p->pack_size = st.st_size;
 675        } else if (p->pack_size != st.st_size)
 676                return error("packfile %s size changed", p->pack_name);
 677
 678        /* We leave these file descriptors open with sliding mmap;
 679         * there is no point keeping them open across exec(), though.
 680         */
 681        fd_flag = fcntl(p->pack_fd, F_GETFD, 0);
 682        if (fd_flag < 0)
 683                return error("cannot determine file descriptor flags");
 684        fd_flag |= FD_CLOEXEC;
 685        if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
 686                return error("cannot set FD_CLOEXEC");
 687
 688        /* Verify we recognize this pack file format. */
 689        if (read_in_full(p->pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
 690                return error("file %s is far too short to be a packfile", p->pack_name);
 691        if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
 692                return error("file %s is not a GIT packfile", p->pack_name);
 693        if (!pack_version_ok(hdr.hdr_version))
 694                return error("packfile %s is version %u and not supported"
 695                        " (try upgrading GIT to a newer version)",
 696                        p->pack_name, ntohl(hdr.hdr_version));
 697
 698        /* Verify the pack matches its index. */
 699        if (p->num_objects != ntohl(hdr.hdr_entries))
 700                return error("packfile %s claims to have %u objects"
 701                             " while index indicates %u objects",
 702                             p->pack_name, ntohl(hdr.hdr_entries),
 703                             p->num_objects);
 704        if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
 705                return error("end of packfile %s is unavailable", p->pack_name);
 706        if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1))
 707                return error("packfile %s signature is unavailable", p->pack_name);
 708        idx_sha1 = ((unsigned char *)p->index_data) + p->index_size - 40;
 709        if (hashcmp(sha1, idx_sha1))
 710                return error("packfile %s does not match index", p->pack_name);
 711        return 0;
 712}
 713
 714static int open_packed_git(struct packed_git *p)
 715{
 716        if (!open_packed_git_1(p))
 717                return 0;
 718        if (p->pack_fd != -1) {
 719                close(p->pack_fd);
 720                p->pack_fd = -1;
 721        }
 722        return -1;
 723}
 724
 725static int in_window(struct pack_window *win, off_t offset)
 726{
 727        /* We must promise at least 20 bytes (one hash) after the
 728         * offset is available from this window, otherwise the offset
 729         * is not actually in this window and a different window (which
 730         * has that one hash excess) must be used.  This is to support
 731         * the object header and delta base parsing routines below.
 732         */
 733        off_t win_off = win->offset;
 734        return win_off <= offset
 735                && (offset + 20) <= (win_off + win->len);
 736}
 737
 738unsigned char* use_pack(struct packed_git *p,
 739                struct pack_window **w_cursor,
 740                off_t offset,
 741                unsigned int *left)
 742{
 743        struct pack_window *win = *w_cursor;
 744
 745        if (p->pack_fd == -1 && open_packed_git(p))
 746                die("packfile %s cannot be accessed", p->pack_name);
 747
 748        /* Since packfiles end in a hash of their content and its
 749         * pointless to ask for an offset into the middle of that
 750         * hash, and the in_window function above wouldn't match
 751         * don't allow an offset too close to the end of the file.
 752         */
 753        if (offset > (p->pack_size - 20))
 754                die("offset beyond end of packfile (truncated pack?)");
 755
 756        if (!win || !in_window(win, offset)) {
 757                if (win)
 758                        win->inuse_cnt--;
 759                for (win = p->windows; win; win = win->next) {
 760                        if (in_window(win, offset))
 761                                break;
 762                }
 763                if (!win) {
 764                        size_t window_align = packed_git_window_size / 2;
 765                        off_t len;
 766                        win = xcalloc(1, sizeof(*win));
 767                        win->offset = (offset / window_align) * window_align;
 768                        len = p->pack_size - win->offset;
 769                        if (len > packed_git_window_size)
 770                                len = packed_git_window_size;
 771                        win->len = (size_t)len;
 772                        pack_mapped += win->len;
 773                        while (packed_git_limit < pack_mapped
 774                                && unuse_one_window(p, p->pack_fd))
 775                                ; /* nothing */
 776                        win->base = xmmap(NULL, win->len,
 777                                PROT_READ, MAP_PRIVATE,
 778                                p->pack_fd, win->offset);
 779                        if (win->base == MAP_FAILED)
 780                                die("packfile %s cannot be mapped: %s",
 781                                        p->pack_name,
 782                                        strerror(errno));
 783                        pack_mmap_calls++;
 784                        pack_open_windows++;
 785                        if (pack_mapped > peak_pack_mapped)
 786                                peak_pack_mapped = pack_mapped;
 787                        if (pack_open_windows > peak_pack_open_windows)
 788                                peak_pack_open_windows = pack_open_windows;
 789                        win->next = p->windows;
 790                        p->windows = win;
 791                }
 792        }
 793        if (win != *w_cursor) {
 794                win->last_used = pack_used_ctr++;
 795                win->inuse_cnt++;
 796                *w_cursor = win;
 797        }
 798        offset -= win->offset;
 799        if (left)
 800                *left = win->len - xsize_t(offset);
 801        return win->base + offset;
 802}
 803
 804struct packed_git *add_packed_git(const char *path, int path_len, int local)
 805{
 806        struct stat st;
 807        struct packed_git *p = xmalloc(sizeof(*p) + path_len + 2);
 808
 809        /*
 810         * Make sure a corresponding .pack file exists and that
 811         * the index looks sane.
 812         */
 813        path_len -= strlen(".idx");
 814        if (path_len < 1)
 815                return NULL;
 816        memcpy(p->pack_name, path, path_len);
 817        strcpy(p->pack_name + path_len, ".pack");
 818        if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
 819                free(p);
 820                return NULL;
 821        }
 822
 823        /* ok, it looks sane as far as we can check without
 824         * actually mapping the pack file.
 825         */
 826        p->index_version = 0;
 827        p->index_data = NULL;
 828        p->index_size = 0;
 829        p->num_objects = 0;
 830        p->pack_size = st.st_size;
 831        p->next = NULL;
 832        p->windows = NULL;
 833        p->pack_fd = -1;
 834        p->pack_local = local;
 835        p->mtime = st.st_mtime;
 836        if (path_len < 40 || get_sha1_hex(path + path_len - 40, p->sha1))
 837                hashclr(p->sha1);
 838        return p;
 839}
 840
 841struct packed_git *parse_pack_index(unsigned char *sha1)
 842{
 843        const char *idx_path = sha1_pack_index_name(sha1);
 844        const char *path = sha1_pack_name(sha1);
 845        struct packed_git *p = xmalloc(sizeof(*p) + strlen(path) + 2);
 846
 847        if (check_packed_git_idx(idx_path, p)) {
 848                free(p);
 849                return NULL;
 850        }
 851
 852        strcpy(p->pack_name, path);
 853        p->pack_size = 0;
 854        p->next = NULL;
 855        p->windows = NULL;
 856        p->pack_fd = -1;
 857        hashcpy(p->sha1, sha1);
 858        return p;
 859}
 860
 861void install_packed_git(struct packed_git *pack)
 862{
 863        pack->next = packed_git;
 864        packed_git = pack;
 865}
 866
 867static void prepare_packed_git_one(char *objdir, int local)
 868{
 869        /* Ensure that this buffer is large enough so that we can
 870           append "/pack/" without clobbering the stack even if
 871           strlen(objdir) were PATH_MAX.  */
 872        char path[PATH_MAX + 1 + 4 + 1 + 1];
 873        int len;
 874        DIR *dir;
 875        struct dirent *de;
 876
 877        sprintf(path, "%s/pack", objdir);
 878        len = strlen(path);
 879        dir = opendir(path);
 880        if (!dir) {
 881                if (errno != ENOENT)
 882                        error("unable to open object pack directory: %s: %s",
 883                              path, strerror(errno));
 884                return;
 885        }
 886        path[len++] = '/';
 887        while ((de = readdir(dir)) != NULL) {
 888                int namelen = strlen(de->d_name);
 889                struct packed_git *p;
 890
 891                if (!has_extension(de->d_name, ".idx"))
 892                        continue;
 893
 894                if (len + namelen + 1 > sizeof(path))
 895                        continue;
 896
 897                /* Don't reopen a pack we already have. */
 898                strcpy(path + len, de->d_name);
 899                for (p = packed_git; p; p = p->next) {
 900                        if (!memcmp(path, p->pack_name, len + namelen - 4))
 901                                break;
 902                }
 903                if (p)
 904                        continue;
 905                /* See if it really is a valid .idx file with corresponding
 906                 * .pack file that we can map.
 907                 */
 908                p = add_packed_git(path, len + namelen, local);
 909                if (!p)
 910                        continue;
 911                install_packed_git(p);
 912        }
 913        closedir(dir);
 914}
 915
 916static int sort_pack(const void *a_, const void *b_)
 917{
 918        struct packed_git *a = *((struct packed_git **)a_);
 919        struct packed_git *b = *((struct packed_git **)b_);
 920        int st;
 921
 922        /*
 923         * Local packs tend to contain objects specific to our
 924         * variant of the project than remote ones.  In addition,
 925         * remote ones could be on a network mounted filesystem.
 926         * Favor local ones for these reasons.
 927         */
 928        st = a->pack_local - b->pack_local;
 929        if (st)
 930                return -st;
 931
 932        /*
 933         * Younger packs tend to contain more recent objects,
 934         * and more recent objects tend to get accessed more
 935         * often.
 936         */
 937        if (a->mtime < b->mtime)
 938                return 1;
 939        else if (a->mtime == b->mtime)
 940                return 0;
 941        return -1;
 942}
 943
 944static void rearrange_packed_git(void)
 945{
 946        struct packed_git **ary, *p;
 947        int i, n;
 948
 949        for (n = 0, p = packed_git; p; p = p->next)
 950                n++;
 951        if (n < 2)
 952                return;
 953
 954        /* prepare an array of packed_git for easier sorting */
 955        ary = xcalloc(n, sizeof(struct packed_git *));
 956        for (n = 0, p = packed_git; p; p = p->next)
 957                ary[n++] = p;
 958
 959        qsort(ary, n, sizeof(struct packed_git *), sort_pack);
 960
 961        /* link them back again */
 962        for (i = 0; i < n - 1; i++)
 963                ary[i]->next = ary[i + 1];
 964        ary[n - 1]->next = NULL;
 965        packed_git = ary[0];
 966
 967        free(ary);
 968}
 969
 970static int prepare_packed_git_run_once = 0;
 971void prepare_packed_git(void)
 972{
 973        struct alternate_object_database *alt;
 974
 975        if (prepare_packed_git_run_once)
 976                return;
 977        prepare_packed_git_one(get_object_directory(), 1);
 978        prepare_alt_odb();
 979        for (alt = alt_odb_list; alt; alt = alt->next) {
 980                alt->name[-1] = 0;
 981                prepare_packed_git_one(alt->base, 0);
 982                alt->name[-1] = '/';
 983        }
 984        rearrange_packed_git();
 985        prepare_packed_git_run_once = 1;
 986}
 987
 988void reprepare_packed_git(void)
 989{
 990        prepare_packed_git_run_once = 0;
 991        prepare_packed_git();
 992}
 993
 994int check_sha1_signature(const unsigned char *sha1, void *map, unsigned long size, const char *type)
 995{
 996        unsigned char real_sha1[20];
 997        hash_sha1_file(map, size, type, real_sha1);
 998        return hashcmp(sha1, real_sha1) ? -1 : 0;
 999}
1000
1001static int git_open_noatime(const char *name)
1002{
1003        static int sha1_file_open_flag = O_NOATIME;
1004        int fd = open(name, O_RDONLY | sha1_file_open_flag);
1005
1006        /* Might the failure be due to O_NOATIME? */
1007        if (fd < 0 && errno != ENOENT && sha1_file_open_flag) {
1008                fd = open(name, O_RDONLY);
1009                if (fd >= 0)
1010                        sha1_file_open_flag = 0;
1011        }
1012        return fd;
1013}
1014
1015static int open_sha1_file(const unsigned char *sha1)
1016{
1017        int fd;
1018        char *name = sha1_file_name(sha1);
1019        struct alternate_object_database *alt;
1020
1021        fd = git_open_noatime(name);
1022        if (fd >= 0)
1023                return fd;
1024
1025        prepare_alt_odb();
1026        errno = ENOENT;
1027        for (alt = alt_odb_list; alt; alt = alt->next) {
1028                name = alt->name;
1029                fill_sha1_path(name, sha1);
1030                fd = git_open_noatime(alt->base);
1031                if (fd >= 0)
1032                        return fd;
1033        }
1034        return -1;
1035}
1036
1037static void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
1038{
1039        void *map;
1040        int fd;
1041
1042        fd = open_sha1_file(sha1);
1043        map = NULL;
1044        if (fd >= 0) {
1045                struct stat st;
1046
1047                if (!fstat(fd, &st)) {
1048                        *size = xsize_t(st.st_size);
1049                        map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
1050                }
1051                close(fd);
1052        }
1053        return map;
1054}
1055
1056static int legacy_loose_object(unsigned char *map)
1057{
1058        unsigned int word;
1059
1060        /*
1061         * Is it a zlib-compressed buffer? If so, the first byte
1062         * must be 0x78 (15-bit window size, deflated), and the
1063         * first 16-bit word is evenly divisible by 31
1064         */
1065        word = (map[0] << 8) + map[1];
1066        if (map[0] == 0x78 && !(word % 31))
1067                return 1;
1068        else
1069                return 0;
1070}
1071
1072unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep)
1073{
1074        unsigned shift;
1075        unsigned char c;
1076        unsigned long size;
1077        unsigned long used = 0;
1078
1079        c = buf[used++];
1080        *type = (c >> 4) & 7;
1081        size = c & 15;
1082        shift = 4;
1083        while (c & 0x80) {
1084                if (len <= used)
1085                        return 0;
1086                if (sizeof(long) * 8 <= shift)
1087                        return 0;
1088                c = buf[used++];
1089                size += (c & 0x7f) << shift;
1090                shift += 7;
1091        }
1092        *sizep = size;
1093        return used;
1094}
1095
1096static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
1097{
1098        unsigned long size, used;
1099        static const char valid_loose_object_type[8] = {
1100                0, /* OBJ_EXT */
1101                1, 1, 1, 1, /* "commit", "tree", "blob", "tag" */
1102                0, /* "delta" and others are invalid in a loose object */
1103        };
1104        enum object_type type;
1105
1106        /* Get the data stream */
1107        memset(stream, 0, sizeof(*stream));
1108        stream->next_in = map;
1109        stream->avail_in = mapsize;
1110        stream->next_out = buffer;
1111        stream->avail_out = bufsiz;
1112
1113        if (legacy_loose_object(map)) {
1114                inflateInit(stream);
1115                return inflate(stream, 0);
1116        }
1117
1118
1119        /*
1120         * There used to be a second loose object header format which
1121         * was meant to mimic the in-pack format, allowing for direct
1122         * copy of the object data.  This format turned up not to be
1123         * really worth it and we don't write it any longer.  But we
1124         * can still read it.
1125         */
1126        used = unpack_object_header_gently(map, mapsize, &type, &size);
1127        if (!used || !valid_loose_object_type[type])
1128                return -1;
1129        map += used;
1130        mapsize -= used;
1131
1132        /* Set up the stream for the rest.. */
1133        stream->next_in = map;
1134        stream->avail_in = mapsize;
1135        inflateInit(stream);
1136
1137        /* And generate the fake traditional header */
1138        stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu",
1139                                         typename(type), size);
1140        return 0;
1141}
1142
1143static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
1144{
1145        int bytes = strlen(buffer) + 1;
1146        unsigned char *buf = xmalloc(1+size);
1147        unsigned long n;
1148        int status = Z_OK;
1149
1150        n = stream->total_out - bytes;
1151        if (n > size)
1152                n = size;
1153        memcpy(buf, (char *) buffer + bytes, n);
1154        bytes = n;
1155        if (bytes <= size) {
1156                /*
1157                 * The above condition must be (bytes <= size), not
1158                 * (bytes < size).  In other words, even though we
1159                 * expect no more output and set avail_out to zer0,
1160                 * the input zlib stream may have bytes that express
1161                 * "this concludes the stream", and we *do* want to
1162                 * eat that input.
1163                 *
1164                 * Otherwise we would not be able to test that we
1165                 * consumed all the input to reach the expected size;
1166                 * we also want to check that zlib tells us that all
1167                 * went well with status == Z_STREAM_END at the end.
1168                 */
1169                stream->next_out = buf + bytes;
1170                stream->avail_out = size - bytes;
1171                while (status == Z_OK)
1172                        status = inflate(stream, Z_FINISH);
1173        }
1174        buf[size] = 0;
1175        if (status == Z_STREAM_END && !stream->avail_in) {
1176                inflateEnd(stream);
1177                return buf;
1178        }
1179
1180        if (status < 0)
1181                error("corrupt loose object '%s'", sha1_to_hex(sha1));
1182        else if (stream->avail_in)
1183                error("garbage at end of loose object '%s'",
1184                      sha1_to_hex(sha1));
1185        free(buf);
1186        return NULL;
1187}
1188
1189/*
1190 * We used to just use "sscanf()", but that's actually way
1191 * too permissive for what we want to check. So do an anal
1192 * object header parse by hand.
1193 */
1194static int parse_sha1_header(const char *hdr, unsigned long *sizep)
1195{
1196        char type[10];
1197        int i;
1198        unsigned long size;
1199
1200        /*
1201         * The type can be at most ten bytes (including the
1202         * terminating '\0' that we add), and is followed by
1203         * a space.
1204         */
1205        i = 0;
1206        for (;;) {
1207                char c = *hdr++;
1208                if (c == ' ')
1209                        break;
1210                type[i++] = c;
1211                if (i >= sizeof(type))
1212                        return -1;
1213        }
1214        type[i] = 0;
1215
1216        /*
1217         * The length must follow immediately, and be in canonical
1218         * decimal format (ie "010" is not valid).
1219         */
1220        size = *hdr++ - '0';
1221        if (size > 9)
1222                return -1;
1223        if (size) {
1224                for (;;) {
1225                        unsigned long c = *hdr - '0';
1226                        if (c > 9)
1227                                break;
1228                        hdr++;
1229                        size = size * 10 + c;
1230                }
1231        }
1232        *sizep = size;
1233
1234        /*
1235         * The length must be followed by a zero byte
1236         */
1237        return *hdr ? -1 : type_from_string(type);
1238}
1239
1240static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
1241{
1242        int ret;
1243        z_stream stream;
1244        char hdr[8192];
1245
1246        ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr));
1247        if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0)
1248                return NULL;
1249
1250        return unpack_sha1_rest(&stream, hdr, *size, sha1);
1251}
1252
1253unsigned long get_size_from_delta(struct packed_git *p,
1254                                  struct pack_window **w_curs,
1255                                  off_t curpos)
1256{
1257        const unsigned char *data;
1258        unsigned char delta_head[20], *in;
1259        z_stream stream;
1260        int st;
1261
1262        memset(&stream, 0, sizeof(stream));
1263        stream.next_out = delta_head;
1264        stream.avail_out = sizeof(delta_head);
1265
1266        inflateInit(&stream);
1267        do {
1268                in = use_pack(p, w_curs, curpos, &stream.avail_in);
1269                stream.next_in = in;
1270                st = inflate(&stream, Z_FINISH);
1271                curpos += stream.next_in - in;
1272        } while ((st == Z_OK || st == Z_BUF_ERROR) &&
1273                 stream.total_out < sizeof(delta_head));
1274        inflateEnd(&stream);
1275        if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head))
1276                die("delta data unpack-initial failed");
1277
1278        /* Examine the initial part of the delta to figure out
1279         * the result size.
1280         */
1281        data = delta_head;
1282
1283        /* ignore base size */
1284        get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
1285
1286        /* Read the result size */
1287        return get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
1288}
1289
1290static off_t get_delta_base(struct packed_git *p,
1291                                    struct pack_window **w_curs,
1292                                    off_t *curpos,
1293                                    enum object_type type,
1294                                    off_t delta_obj_offset)
1295{
1296        unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL);
1297        off_t base_offset;
1298
1299        /* use_pack() assured us we have [base_info, base_info + 20)
1300         * as a range that we can look at without walking off the
1301         * end of the mapped window.  Its actually the hash size
1302         * that is assured.  An OFS_DELTA longer than the hash size
1303         * is stupid, as then a REF_DELTA would be smaller to store.
1304         */
1305        if (type == OBJ_OFS_DELTA) {
1306                unsigned used = 0;
1307                unsigned char c = base_info[used++];
1308                base_offset = c & 127;
1309                while (c & 128) {
1310                        base_offset += 1;
1311                        if (!base_offset || MSB(base_offset, 7))
1312                                die("offset value overflow for delta base object");
1313                        c = base_info[used++];
1314                        base_offset = (base_offset << 7) + (c & 127);
1315                }
1316                base_offset = delta_obj_offset - base_offset;
1317                if (base_offset >= delta_obj_offset)
1318                        die("delta base offset out of bound");
1319                *curpos += used;
1320        } else if (type == OBJ_REF_DELTA) {
1321                /* The base entry _must_ be in the same pack */
1322                base_offset = find_pack_entry_one(base_info, p);
1323                if (!base_offset)
1324                        die("failed to find delta-pack base object %s",
1325                                sha1_to_hex(base_info));
1326                *curpos += 20;
1327        } else
1328                die("I am totally screwed");
1329        return base_offset;
1330}
1331
1332/* forward declaration for a mutually recursive function */
1333static int packed_object_info(struct packed_git *p, off_t offset,
1334                              unsigned long *sizep);
1335
1336static int packed_delta_info(struct packed_git *p,
1337                             struct pack_window **w_curs,
1338                             off_t curpos,
1339                             enum object_type type,
1340                             off_t obj_offset,
1341                             unsigned long *sizep)
1342{
1343        off_t base_offset;
1344
1345        base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
1346        type = packed_object_info(p, base_offset, NULL);
1347
1348        /* We choose to only get the type of the base object and
1349         * ignore potentially corrupt pack file that expects the delta
1350         * based on a base with a wrong size.  This saves tons of
1351         * inflate() calls.
1352         */
1353        if (sizep)
1354                *sizep = get_size_from_delta(p, w_curs, curpos);
1355
1356        return type;
1357}
1358
1359static int unpack_object_header(struct packed_git *p,
1360                                struct pack_window **w_curs,
1361                                off_t *curpos,
1362                                unsigned long *sizep)
1363{
1364        unsigned char *base;
1365        unsigned int left;
1366        unsigned long used;
1367        enum object_type type;
1368
1369        /* use_pack() assures us we have [base, base + 20) available
1370         * as a range that we can look at at.  (Its actually the hash
1371         * size that is assured.)  With our object header encoding
1372         * the maximum deflated object size is 2^137, which is just
1373         * insane, so we know won't exceed what we have been given.
1374         */
1375        base = use_pack(p, w_curs, *curpos, &left);
1376        used = unpack_object_header_gently(base, left, &type, sizep);
1377        if (!used)
1378                die("object offset outside of pack file");
1379        *curpos += used;
1380
1381        return type;
1382}
1383
1384const char *packed_object_info_detail(struct packed_git *p,
1385                                      off_t obj_offset,
1386                                      unsigned long *size,
1387                                      unsigned long *store_size,
1388                                      unsigned int *delta_chain_length,
1389                                      unsigned char *base_sha1)
1390{
1391        struct pack_window *w_curs = NULL;
1392        off_t curpos;
1393        unsigned long dummy;
1394        unsigned char *next_sha1;
1395        enum object_type type;
1396        struct revindex_entry *revidx;
1397
1398        *delta_chain_length = 0;
1399        curpos = obj_offset;
1400        type = unpack_object_header(p, &w_curs, &curpos, size);
1401
1402        revidx = find_pack_revindex(p, obj_offset);
1403        *store_size = revidx[1].offset - obj_offset;
1404
1405        for (;;) {
1406                switch (type) {
1407                default:
1408                        die("pack %s contains unknown object type %d",
1409                            p->pack_name, type);
1410                case OBJ_COMMIT:
1411                case OBJ_TREE:
1412                case OBJ_BLOB:
1413                case OBJ_TAG:
1414                        unuse_pack(&w_curs);
1415                        return typename(type);
1416                case OBJ_OFS_DELTA:
1417                        obj_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
1418                        if (*delta_chain_length == 0) {
1419                                revidx = find_pack_revindex(p, obj_offset);
1420                                hashcpy(base_sha1, nth_packed_object_sha1(p, revidx->nr));
1421                        }
1422                        break;
1423                case OBJ_REF_DELTA:
1424                        next_sha1 = use_pack(p, &w_curs, curpos, NULL);
1425                        if (*delta_chain_length == 0)
1426                                hashcpy(base_sha1, next_sha1);
1427                        obj_offset = find_pack_entry_one(next_sha1, p);
1428                        break;
1429                }
1430                (*delta_chain_length)++;
1431                curpos = obj_offset;
1432                type = unpack_object_header(p, &w_curs, &curpos, &dummy);
1433        }
1434}
1435
1436static int packed_object_info(struct packed_git *p, off_t obj_offset,
1437                              unsigned long *sizep)
1438{
1439        struct pack_window *w_curs = NULL;
1440        unsigned long size;
1441        off_t curpos = obj_offset;
1442        enum object_type type;
1443
1444        type = unpack_object_header(p, &w_curs, &curpos, &size);
1445
1446        switch (type) {
1447        case OBJ_OFS_DELTA:
1448        case OBJ_REF_DELTA:
1449                type = packed_delta_info(p, &w_curs, curpos,
1450                                         type, obj_offset, sizep);
1451                break;
1452        case OBJ_COMMIT:
1453        case OBJ_TREE:
1454        case OBJ_BLOB:
1455        case OBJ_TAG:
1456                if (sizep)
1457                        *sizep = size;
1458                break;
1459        default:
1460                die("pack %s contains unknown object type %d",
1461                    p->pack_name, type);
1462        }
1463        unuse_pack(&w_curs);
1464        return type;
1465}
1466
1467static void *unpack_compressed_entry(struct packed_git *p,
1468                                    struct pack_window **w_curs,
1469                                    off_t curpos,
1470                                    unsigned long size)
1471{
1472        int st;
1473        z_stream stream;
1474        unsigned char *buffer, *in;
1475
1476        buffer = xmalloc(size + 1);
1477        buffer[size] = 0;
1478        memset(&stream, 0, sizeof(stream));
1479        stream.next_out = buffer;
1480        stream.avail_out = size;
1481
1482        inflateInit(&stream);
1483        do {
1484                in = use_pack(p, w_curs, curpos, &stream.avail_in);
1485                stream.next_in = in;
1486                st = inflate(&stream, Z_FINISH);
1487                curpos += stream.next_in - in;
1488        } while (st == Z_OK || st == Z_BUF_ERROR);
1489        inflateEnd(&stream);
1490        if ((st != Z_STREAM_END) || stream.total_out != size) {
1491                free(buffer);
1492                return NULL;
1493        }
1494
1495        return buffer;
1496}
1497
1498#define MAX_DELTA_CACHE (256)
1499
1500static size_t delta_base_cached;
1501
1502static struct delta_base_cache_lru_list {
1503        struct delta_base_cache_lru_list *prev;
1504        struct delta_base_cache_lru_list *next;
1505} delta_base_cache_lru = { &delta_base_cache_lru, &delta_base_cache_lru };
1506
1507static struct delta_base_cache_entry {
1508        struct delta_base_cache_lru_list lru;
1509        void *data;
1510        struct packed_git *p;
1511        off_t base_offset;
1512        unsigned long size;
1513        enum object_type type;
1514} delta_base_cache[MAX_DELTA_CACHE];
1515
1516static unsigned long pack_entry_hash(struct packed_git *p, off_t base_offset)
1517{
1518        unsigned long hash;
1519
1520        hash = (unsigned long)p + (unsigned long)base_offset;
1521        hash += (hash >> 8) + (hash >> 16);
1522        return hash % MAX_DELTA_CACHE;
1523}
1524
1525static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
1526        unsigned long *base_size, enum object_type *type, int keep_cache)
1527{
1528        void *ret;
1529        unsigned long hash = pack_entry_hash(p, base_offset);
1530        struct delta_base_cache_entry *ent = delta_base_cache + hash;
1531
1532        ret = ent->data;
1533        if (ret && ent->p == p && ent->base_offset == base_offset)
1534                goto found_cache_entry;
1535        return unpack_entry(p, base_offset, type, base_size);
1536
1537found_cache_entry:
1538        if (!keep_cache) {
1539                ent->data = NULL;
1540                ent->lru.next->prev = ent->lru.prev;
1541                ent->lru.prev->next = ent->lru.next;
1542                delta_base_cached -= ent->size;
1543        } else {
1544                ret = xmemdupz(ent->data, ent->size);
1545        }
1546        *type = ent->type;
1547        *base_size = ent->size;
1548        return ret;
1549}
1550
1551static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
1552{
1553        if (ent->data) {
1554                free(ent->data);
1555                ent->data = NULL;
1556                ent->lru.next->prev = ent->lru.prev;
1557                ent->lru.prev->next = ent->lru.next;
1558                delta_base_cached -= ent->size;
1559        }
1560}
1561
1562static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
1563        void *base, unsigned long base_size, enum object_type type)
1564{
1565        unsigned long hash = pack_entry_hash(p, base_offset);
1566        struct delta_base_cache_entry *ent = delta_base_cache + hash;
1567        struct delta_base_cache_lru_list *lru;
1568
1569        release_delta_base_cache(ent);
1570        delta_base_cached += base_size;
1571
1572        for (lru = delta_base_cache_lru.next;
1573             delta_base_cached > delta_base_cache_limit
1574             && lru != &delta_base_cache_lru;
1575             lru = lru->next) {
1576                struct delta_base_cache_entry *f = (void *)lru;
1577                if (f->type == OBJ_BLOB)
1578                        release_delta_base_cache(f);
1579        }
1580        for (lru = delta_base_cache_lru.next;
1581             delta_base_cached > delta_base_cache_limit
1582             && lru != &delta_base_cache_lru;
1583             lru = lru->next) {
1584                struct delta_base_cache_entry *f = (void *)lru;
1585                release_delta_base_cache(f);
1586        }
1587
1588        ent->p = p;
1589        ent->base_offset = base_offset;
1590        ent->type = type;
1591        ent->data = base;
1592        ent->size = base_size;
1593        ent->lru.next = &delta_base_cache_lru;
1594        ent->lru.prev = delta_base_cache_lru.prev;
1595        delta_base_cache_lru.prev->next = &ent->lru;
1596        delta_base_cache_lru.prev = &ent->lru;
1597}
1598
1599static void *unpack_delta_entry(struct packed_git *p,
1600                                struct pack_window **w_curs,
1601                                off_t curpos,
1602                                unsigned long delta_size,
1603                                off_t obj_offset,
1604                                enum object_type *type,
1605                                unsigned long *sizep)
1606{
1607        void *delta_data, *result, *base;
1608        unsigned long base_size;
1609        off_t base_offset;
1610
1611        base_offset = get_delta_base(p, w_curs, &curpos, *type, obj_offset);
1612        base = cache_or_unpack_entry(p, base_offset, &base_size, type, 0);
1613        if (!base)
1614                die("failed to read delta base object"
1615                    " at %"PRIuMAX" from %s",
1616                    (uintmax_t)base_offset, p->pack_name);
1617
1618        delta_data = unpack_compressed_entry(p, w_curs, curpos, delta_size);
1619        if (!delta_data)
1620                die("failed to unpack compressed delta"
1621                    " at %"PRIuMAX" from %s",
1622                    (uintmax_t)curpos, p->pack_name);
1623        result = patch_delta(base, base_size,
1624                             delta_data, delta_size,
1625                             sizep);
1626        if (!result)
1627                die("failed to apply delta");
1628        free(delta_data);
1629        add_delta_base_cache(p, base_offset, base, base_size, *type);
1630        return result;
1631}
1632
1633void *unpack_entry(struct packed_git *p, off_t obj_offset,
1634                   enum object_type *type, unsigned long *sizep)
1635{
1636        struct pack_window *w_curs = NULL;
1637        off_t curpos = obj_offset;
1638        void *data;
1639
1640        *type = unpack_object_header(p, &w_curs, &curpos, sizep);
1641        switch (*type) {
1642        case OBJ_OFS_DELTA:
1643        case OBJ_REF_DELTA:
1644                data = unpack_delta_entry(p, &w_curs, curpos, *sizep,
1645                                          obj_offset, type, sizep);
1646                break;
1647        case OBJ_COMMIT:
1648        case OBJ_TREE:
1649        case OBJ_BLOB:
1650        case OBJ_TAG:
1651                data = unpack_compressed_entry(p, &w_curs, curpos, *sizep);
1652                break;
1653        default:
1654                die("unknown object type %i in %s", *type, p->pack_name);
1655        }
1656        unuse_pack(&w_curs);
1657        return data;
1658}
1659
1660const unsigned char *nth_packed_object_sha1(struct packed_git *p,
1661                                            uint32_t n)
1662{
1663        const unsigned char *index = p->index_data;
1664        if (!index) {
1665                if (open_pack_index(p))
1666                        return NULL;
1667                index = p->index_data;
1668        }
1669        if (n >= p->num_objects)
1670                return NULL;
1671        index += 4 * 256;
1672        if (p->index_version == 1) {
1673                return index + 24 * n + 4;
1674        } else {
1675                index += 8;
1676                return index + 20 * n;
1677        }
1678}
1679
1680static off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
1681{
1682        const unsigned char *index = p->index_data;
1683        index += 4 * 256;
1684        if (p->index_version == 1) {
1685                return ntohl(*((uint32_t *)(index + 24 * n)));
1686        } else {
1687                uint32_t off;
1688                index += 8 + p->num_objects * (20 + 4);
1689                off = ntohl(*((uint32_t *)(index + 4 * n)));
1690                if (!(off & 0x80000000))
1691                        return off;
1692                index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
1693                return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
1694                                   ntohl(*((uint32_t *)(index + 4)));
1695        }
1696}
1697
1698off_t find_pack_entry_one(const unsigned char *sha1,
1699                                  struct packed_git *p)
1700{
1701        const uint32_t *level1_ofs = p->index_data;
1702        const unsigned char *index = p->index_data;
1703        unsigned hi, lo, stride;
1704        static int use_lookup = -1;
1705        static int debug_lookup = -1;
1706
1707        if (debug_lookup < 0)
1708                debug_lookup = !!getenv("GIT_DEBUG_LOOKUP");
1709
1710        if (!index) {
1711                if (open_pack_index(p))
1712                        return 0;
1713                level1_ofs = p->index_data;
1714                index = p->index_data;
1715        }
1716        if (p->index_version > 1) {
1717                level1_ofs += 2;
1718                index += 8;
1719        }
1720        index += 4 * 256;
1721        hi = ntohl(level1_ofs[*sha1]);
1722        lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
1723        if (p->index_version > 1) {
1724                stride = 20;
1725        } else {
1726                stride = 24;
1727                index += 4;
1728        }
1729
1730        if (debug_lookup)
1731                printf("%02x%02x%02x... lo %u hi %u nr %u\n",
1732                       sha1[0], sha1[1], sha1[2], lo, hi, p->num_objects);
1733
1734        if (use_lookup < 0)
1735                use_lookup = !!getenv("GIT_USE_LOOKUP");
1736        if (use_lookup) {
1737                int pos = sha1_entry_pos(index, stride, 0,
1738                                         lo, hi, p->num_objects, sha1);
1739                if (pos < 0)
1740                        return 0;
1741                return nth_packed_object_offset(p, pos);
1742        }
1743
1744        do {
1745                unsigned mi = (lo + hi) / 2;
1746                int cmp = hashcmp(index + mi * stride, sha1);
1747
1748                if (debug_lookup)
1749                        printf("lo %u hi %u rg %u mi %u\n",
1750                               lo, hi, hi - lo, mi);
1751                if (!cmp)
1752                        return nth_packed_object_offset(p, mi);
1753                if (cmp > 0)
1754                        hi = mi;
1755                else
1756                        lo = mi+1;
1757        } while (lo < hi);
1758        return 0;
1759}
1760
1761int matches_pack_name(struct packed_git *p, const char *name)
1762{
1763        const char *last_c, *c;
1764
1765        if (!strcmp(p->pack_name, name))
1766                return 1;
1767
1768        for (c = p->pack_name, last_c = c; *c;)
1769                if (*c == '/')
1770                        last_c = ++c;
1771                else
1772                        ++c;
1773        if (!strcmp(last_c, name))
1774                return 1;
1775
1776        return 0;
1777}
1778
1779static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e, const char **ignore_packed)
1780{
1781        static struct packed_git *last_found = (void *)1;
1782        struct packed_git *p;
1783        off_t offset;
1784
1785        prepare_packed_git();
1786        if (!packed_git)
1787                return 0;
1788        p = (last_found == (void *)1) ? packed_git : last_found;
1789
1790        do {
1791                if (ignore_packed) {
1792                        const char **ig;
1793                        for (ig = ignore_packed; *ig; ig++)
1794                                if (matches_pack_name(p, *ig))
1795                                        break;
1796                        if (*ig)
1797                                goto next;
1798                }
1799
1800                offset = find_pack_entry_one(sha1, p);
1801                if (offset) {
1802                        /*
1803                         * We are about to tell the caller where they can
1804                         * locate the requested object.  We better make
1805                         * sure the packfile is still here and can be
1806                         * accessed before supplying that answer, as
1807                         * it may have been deleted since the index
1808                         * was loaded!
1809                         */
1810                        if (p->pack_fd == -1 && open_packed_git(p)) {
1811                                error("packfile %s cannot be accessed", p->pack_name);
1812                                goto next;
1813                        }
1814                        e->offset = offset;
1815                        e->p = p;
1816                        hashcpy(e->sha1, sha1);
1817                        last_found = p;
1818                        return 1;
1819                }
1820
1821                next:
1822                if (p == last_found)
1823                        p = packed_git;
1824                else
1825                        p = p->next;
1826                if (p == last_found)
1827                        p = p->next;
1828        } while (p);
1829        return 0;
1830}
1831
1832struct packed_git *find_sha1_pack(const unsigned char *sha1,
1833                                  struct packed_git *packs)
1834{
1835        struct packed_git *p;
1836
1837        for (p = packs; p; p = p->next) {
1838                if (find_pack_entry_one(sha1, p))
1839                        return p;
1840        }
1841        return NULL;
1842
1843}
1844
1845static int sha1_loose_object_info(const unsigned char *sha1, unsigned long *sizep)
1846{
1847        int status;
1848        unsigned long mapsize, size;
1849        void *map;
1850        z_stream stream;
1851        char hdr[32];
1852
1853        map = map_sha1_file(sha1, &mapsize);
1854        if (!map)
1855                return error("unable to find %s", sha1_to_hex(sha1));
1856        if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
1857                status = error("unable to unpack %s header",
1858                               sha1_to_hex(sha1));
1859        else if ((status = parse_sha1_header(hdr, &size)) < 0)
1860                status = error("unable to parse %s header", sha1_to_hex(sha1));
1861        else if (sizep)
1862                *sizep = size;
1863        inflateEnd(&stream);
1864        munmap(map, mapsize);
1865        return status;
1866}
1867
1868int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
1869{
1870        struct pack_entry e;
1871
1872        if (!find_pack_entry(sha1, &e, NULL)) {
1873                reprepare_packed_git();
1874                if (!find_pack_entry(sha1, &e, NULL))
1875                        return sha1_loose_object_info(sha1, sizep);
1876        }
1877        return packed_object_info(e.p, e.offset, sizep);
1878}
1879
1880static void *read_packed_sha1(const unsigned char *sha1,
1881                              enum object_type *type, unsigned long *size)
1882{
1883        struct pack_entry e;
1884
1885        if (!find_pack_entry(sha1, &e, NULL))
1886                return NULL;
1887        else
1888                return cache_or_unpack_entry(e.p, e.offset, size, type, 1);
1889}
1890
1891/*
1892 * This is meant to hold a *small* number of objects that you would
1893 * want read_sha1_file() to be able to return, but yet you do not want
1894 * to write them into the object store (e.g. a browse-only
1895 * application).
1896 */
1897static struct cached_object {
1898        unsigned char sha1[20];
1899        enum object_type type;
1900        void *buf;
1901        unsigned long size;
1902} *cached_objects;
1903static int cached_object_nr, cached_object_alloc;
1904
1905static struct cached_object empty_tree = {
1906        /* empty tree sha1: 4b825dc642cb6eb9a060e54bf8d69288fbee4904 */
1907        "\x4b\x82\x5d\xc6\x42\xcb\x6e\xb9\xa0\x60"
1908        "\xe5\x4b\xf8\xd6\x92\x88\xfb\xee\x49\x04",
1909        OBJ_TREE,
1910        "",
1911        0
1912};
1913
1914static struct cached_object *find_cached_object(const unsigned char *sha1)
1915{
1916        int i;
1917        struct cached_object *co = cached_objects;
1918
1919        for (i = 0; i < cached_object_nr; i++, co++) {
1920                if (!hashcmp(co->sha1, sha1))
1921                        return co;
1922        }
1923        if (!hashcmp(sha1, empty_tree.sha1))
1924                return &empty_tree;
1925        return NULL;
1926}
1927
1928int pretend_sha1_file(void *buf, unsigned long len, enum object_type type,
1929                      unsigned char *sha1)
1930{
1931        struct cached_object *co;
1932
1933        hash_sha1_file(buf, len, typename(type), sha1);
1934        if (has_sha1_file(sha1) || find_cached_object(sha1))
1935                return 0;
1936        if (cached_object_alloc <= cached_object_nr) {
1937                cached_object_alloc = alloc_nr(cached_object_alloc);
1938                cached_objects = xrealloc(cached_objects,
1939                                          sizeof(*cached_objects) *
1940                                          cached_object_alloc);
1941        }
1942        co = &cached_objects[cached_object_nr++];
1943        co->size = len;
1944        co->type = type;
1945        co->buf = xmalloc(len);
1946        memcpy(co->buf, buf, len);
1947        hashcpy(co->sha1, sha1);
1948        return 0;
1949}
1950
1951void *read_sha1_file(const unsigned char *sha1, enum object_type *type,
1952                     unsigned long *size)
1953{
1954        unsigned long mapsize;
1955        void *map, *buf;
1956        struct cached_object *co;
1957
1958        co = find_cached_object(sha1);
1959        if (co) {
1960                *type = co->type;
1961                *size = co->size;
1962                return xmemdupz(co->buf, co->size);
1963        }
1964
1965        buf = read_packed_sha1(sha1, type, size);
1966        if (buf)
1967                return buf;
1968        map = map_sha1_file(sha1, &mapsize);
1969        if (map) {
1970                buf = unpack_sha1_file(map, mapsize, type, size, sha1);
1971                munmap(map, mapsize);
1972                return buf;
1973        }
1974        reprepare_packed_git();
1975        return read_packed_sha1(sha1, type, size);
1976}
1977
1978void *read_object_with_reference(const unsigned char *sha1,
1979                                 const char *required_type_name,
1980                                 unsigned long *size,
1981                                 unsigned char *actual_sha1_return)
1982{
1983        enum object_type type, required_type;
1984        void *buffer;
1985        unsigned long isize;
1986        unsigned char actual_sha1[20];
1987
1988        required_type = type_from_string(required_type_name);
1989        hashcpy(actual_sha1, sha1);
1990        while (1) {
1991                int ref_length = -1;
1992                const char *ref_type = NULL;
1993
1994                buffer = read_sha1_file(actual_sha1, &type, &isize);
1995                if (!buffer)
1996                        return NULL;
1997                if (type == required_type) {
1998                        *size = isize;
1999                        if (actual_sha1_return)
2000                                hashcpy(actual_sha1_return, actual_sha1);
2001                        return buffer;
2002                }
2003                /* Handle references */
2004                else if (type == OBJ_COMMIT)
2005                        ref_type = "tree ";
2006                else if (type == OBJ_TAG)
2007                        ref_type = "object ";
2008                else {
2009                        free(buffer);
2010                        return NULL;
2011                }
2012                ref_length = strlen(ref_type);
2013
2014                if (ref_length + 40 > isize ||
2015                    memcmp(buffer, ref_type, ref_length) ||
2016                    get_sha1_hex((char *) buffer + ref_length, actual_sha1)) {
2017                        free(buffer);
2018                        return NULL;
2019                }
2020                free(buffer);
2021                /* Now we have the ID of the referred-to object in
2022                 * actual_sha1.  Check again. */
2023        }
2024}
2025
2026static void write_sha1_file_prepare(const void *buf, unsigned long len,
2027                                    const char *type, unsigned char *sha1,
2028                                    char *hdr, int *hdrlen)
2029{
2030        SHA_CTX c;
2031
2032        /* Generate the header */
2033        *hdrlen = sprintf(hdr, "%s %lu", type, len)+1;
2034
2035        /* Sha1.. */
2036        SHA1_Init(&c);
2037        SHA1_Update(&c, hdr, *hdrlen);
2038        SHA1_Update(&c, buf, len);
2039        SHA1_Final(sha1, &c);
2040}
2041
2042/*
2043 * Move the just written object into its final resting place
2044 */
2045int move_temp_to_file(const char *tmpfile, const char *filename)
2046{
2047        int ret = link(tmpfile, filename);
2048
2049        /*
2050         * Coda hack - coda doesn't like cross-directory links,
2051         * so we fall back to a rename, which will mean that it
2052         * won't be able to check collisions, but that's not a
2053         * big deal.
2054         *
2055         * The same holds for FAT formatted media.
2056         *
2057         * When this succeeds, we just return 0. We have nothing
2058         * left to unlink.
2059         */
2060        if (ret && ret != EEXIST) {
2061                if (!rename(tmpfile, filename))
2062                        return 0;
2063                ret = errno;
2064        }
2065        unlink(tmpfile);
2066        if (ret) {
2067                if (ret != EEXIST) {
2068                        return error("unable to write sha1 filename %s: %s\n", filename, strerror(ret));
2069                }
2070                /* FIXME!!! Collision check here ? */
2071        }
2072
2073        return 0;
2074}
2075
2076static int write_buffer(int fd, const void *buf, size_t len)
2077{
2078        if (write_in_full(fd, buf, len) < 0)
2079                return error("file write error (%s)", strerror(errno));
2080        return 0;
2081}
2082
2083int hash_sha1_file(const void *buf, unsigned long len, const char *type,
2084                   unsigned char *sha1)
2085{
2086        char hdr[32];
2087        int hdrlen;
2088        write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
2089        return 0;
2090}
2091
2092/* Finalize a file on disk, and close it. */
2093static void close_sha1_file(int fd)
2094{
2095        /* For safe-mode, we could fsync_or_die(fd, "sha1 file") here */
2096        fchmod(fd, 0444);
2097        if (close(fd) != 0)
2098                die("unable to write sha1 file");
2099}
2100
2101/* Size of directory component, including the ending '/' */
2102static inline int directory_size(const char *filename)
2103{
2104        const char *s = strrchr(filename, '/');
2105        if (!s)
2106                return 0;
2107        return s - filename + 1;
2108}
2109
2110/*
2111 * This creates a temporary file in the same directory as the final
2112 * 'filename'
2113 *
2114 * We want to avoid cross-directory filename renames, because those
2115 * can have problems on various filesystems (FAT, NFS, Coda).
2116 */
2117static int create_tmpfile(char *buffer, size_t bufsiz, const char *filename)
2118{
2119        int fd, dirlen = directory_size(filename);
2120
2121        if (dirlen + 20 > bufsiz) {
2122                errno = ENAMETOOLONG;
2123                return -1;
2124        }
2125        memcpy(buffer, filename, dirlen);
2126        strcpy(buffer + dirlen, "tmp_obj_XXXXXX");
2127        fd = mkstemp(buffer);
2128        if (fd < 0 && dirlen) {
2129                /* Make sure the directory exists */
2130                memcpy(buffer, filename, dirlen);
2131                buffer[dirlen-1] = 0;
2132                if (mkdir(buffer, 0777) || adjust_shared_perm(buffer))
2133                        return -1;
2134
2135                /* Try again */
2136                strcpy(buffer + dirlen - 1, "/tmp_obj_XXXXXX");
2137                fd = mkstemp(buffer);
2138        }
2139        return fd;
2140}
2141
2142static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
2143                              void *buf, unsigned long len, time_t mtime)
2144{
2145        int fd, size, ret;
2146        unsigned char *compressed;
2147        z_stream stream;
2148        char *filename;
2149        static char tmpfile[PATH_MAX];
2150
2151        filename = sha1_file_name(sha1);
2152        fd = create_tmpfile(tmpfile, sizeof(tmpfile), filename);
2153        if (fd < 0) {
2154                if (errno == EPERM)
2155                        return error("insufficient permission for adding an object to repository database %s\n", get_object_directory());
2156                else
2157                        return error("unable to create temporary sha1 filename %s: %s\n", tmpfile, strerror(errno));
2158        }
2159
2160        /* Set it up */
2161        memset(&stream, 0, sizeof(stream));
2162        deflateInit(&stream, zlib_compression_level);
2163        size = 8 + deflateBound(&stream, len+hdrlen);
2164        compressed = xmalloc(size);
2165
2166        /* Compress it */
2167        stream.next_out = compressed;
2168        stream.avail_out = size;
2169
2170        /* First header.. */
2171        stream.next_in = (unsigned char *)hdr;
2172        stream.avail_in = hdrlen;
2173        while (deflate(&stream, 0) == Z_OK)
2174                /* nothing */;
2175
2176        /* Then the data itself.. */
2177        stream.next_in = buf;
2178        stream.avail_in = len;
2179        ret = deflate(&stream, Z_FINISH);
2180        if (ret != Z_STREAM_END)
2181                die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret);
2182
2183        ret = deflateEnd(&stream);
2184        if (ret != Z_OK)
2185                die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret);
2186
2187        size = stream.total_out;
2188
2189        if (write_buffer(fd, compressed, size) < 0)
2190                die("unable to write sha1 file");
2191        close_sha1_file(fd);
2192        free(compressed);
2193
2194        if (mtime) {
2195                struct utimbuf utb;
2196                utb.actime = mtime;
2197                utb.modtime = mtime;
2198                if (utime(tmpfile, &utb) < 0)
2199                        warning("failed utime() on %s: %s",
2200                                tmpfile, strerror(errno));
2201        }
2202
2203        return move_temp_to_file(tmpfile, filename);
2204}
2205
2206int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
2207{
2208        unsigned char sha1[20];
2209        char hdr[32];
2210        int hdrlen;
2211
2212        /* Normally if we have it in the pack then we do not bother writing
2213         * it out into .git/objects/??/?{38} file.
2214         */
2215        write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
2216        if (returnsha1)
2217                hashcpy(returnsha1, sha1);
2218        if (has_sha1_file(sha1))
2219                return 0;
2220        return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
2221}
2222
2223int force_object_loose(const unsigned char *sha1, time_t mtime)
2224{
2225        void *buf;
2226        unsigned long len;
2227        enum object_type type;
2228        char hdr[32];
2229        int hdrlen;
2230
2231        if (has_loose_object(sha1))
2232                return 0;
2233        buf = read_packed_sha1(sha1, &type, &len);
2234        if (!buf)
2235                return error("cannot read sha1_file for %s", sha1_to_hex(sha1));
2236        hdrlen = sprintf(hdr, "%s %lu", typename(type), len) + 1;
2237        return write_loose_object(sha1, hdr, hdrlen, buf, len, mtime);
2238}
2239
2240int has_pack_index(const unsigned char *sha1)
2241{
2242        struct stat st;
2243        if (stat(sha1_pack_index_name(sha1), &st))
2244                return 0;
2245        return 1;
2246}
2247
2248int has_pack_file(const unsigned char *sha1)
2249{
2250        struct stat st;
2251        if (stat(sha1_pack_name(sha1), &st))
2252                return 0;
2253        return 1;
2254}
2255
2256int has_sha1_pack(const unsigned char *sha1, const char **ignore_packed)
2257{
2258        struct pack_entry e;
2259        return find_pack_entry(sha1, &e, ignore_packed);
2260}
2261
2262int has_sha1_file(const unsigned char *sha1)
2263{
2264        struct pack_entry e;
2265
2266        if (find_pack_entry(sha1, &e, NULL))
2267                return 1;
2268        return has_loose_object(sha1);
2269}
2270
2271int index_pipe(unsigned char *sha1, int fd, const char *type, int write_object)
2272{
2273        struct strbuf buf;
2274        int ret;
2275
2276        strbuf_init(&buf, 0);
2277        if (strbuf_read(&buf, fd, 4096) < 0) {
2278                strbuf_release(&buf);
2279                return -1;
2280        }
2281
2282        if (!type)
2283                type = blob_type;
2284        if (write_object)
2285                ret = write_sha1_file(buf.buf, buf.len, type, sha1);
2286        else
2287                ret = hash_sha1_file(buf.buf, buf.len, type, sha1);
2288        strbuf_release(&buf);
2289
2290        return ret;
2291}
2292
2293int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object,
2294             enum object_type type, const char *path)
2295{
2296        size_t size = xsize_t(st->st_size);
2297        void *buf = NULL;
2298        int ret, re_allocated = 0;
2299
2300        if (size)
2301                buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
2302        close(fd);
2303
2304        if (!type)
2305                type = OBJ_BLOB;
2306
2307        /*
2308         * Convert blobs to git internal format
2309         */
2310        if ((type == OBJ_BLOB) && S_ISREG(st->st_mode)) {
2311                struct strbuf nbuf;
2312                strbuf_init(&nbuf, 0);
2313                if (convert_to_git(path, buf, size, &nbuf,
2314                                   write_object ? safe_crlf : 0)) {
2315                        munmap(buf, size);
2316                        buf = strbuf_detach(&nbuf, &size);
2317                        re_allocated = 1;
2318                }
2319        }
2320
2321        if (write_object)
2322                ret = write_sha1_file(buf, size, typename(type), sha1);
2323        else
2324                ret = hash_sha1_file(buf, size, typename(type), sha1);
2325        if (re_allocated) {
2326                free(buf);
2327                return ret;
2328        }
2329        if (size)
2330                munmap(buf, size);
2331        return ret;
2332}
2333
2334int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object)
2335{
2336        int fd;
2337        char *target;
2338        size_t len;
2339
2340        switch (st->st_mode & S_IFMT) {
2341        case S_IFREG:
2342                fd = open(path, O_RDONLY);
2343                if (fd < 0)
2344                        return error("open(\"%s\"): %s", path,
2345                                     strerror(errno));
2346                if (index_fd(sha1, fd, st, write_object, OBJ_BLOB, path) < 0)
2347                        return error("%s: failed to insert into database",
2348                                     path);
2349                break;
2350        case S_IFLNK:
2351                len = xsize_t(st->st_size);
2352                target = xmalloc(len + 1);
2353                if (readlink(path, target, len + 1) != st->st_size) {
2354                        char *errstr = strerror(errno);
2355                        free(target);
2356                        return error("readlink(\"%s\"): %s", path,
2357                                     errstr);
2358                }
2359                if (!write_object)
2360                        hash_sha1_file(target, len, blob_type, sha1);
2361                else if (write_sha1_file(target, len, blob_type, sha1))
2362                        return error("%s: failed to insert into database",
2363                                     path);
2364                free(target);
2365                break;
2366        case S_IFDIR:
2367                return resolve_gitlink_ref(path, "HEAD", sha1);
2368        default:
2369                return error("%s: unsupported file type", path);
2370        }
2371        return 0;
2372}
2373
2374int read_pack_header(int fd, struct pack_header *header)
2375{
2376        if (read_in_full(fd, header, sizeof(*header)) < sizeof(*header))
2377                /* "eof before pack header was fully read" */
2378                return PH_ERROR_EOF;
2379
2380        if (header->hdr_signature != htonl(PACK_SIGNATURE))
2381                /* "protocol error (pack signature mismatch detected)" */
2382                return PH_ERROR_PACK_SIGNATURE;
2383        if (!pack_version_ok(header->hdr_version))
2384                /* "protocol error (pack version unsupported)" */
2385                return PH_ERROR_PROTOCOL;
2386        return 0;
2387}