sha1_file.con commit for-each-ref: fix off by one read. (a74fa11)
   1/*
   2 * GIT - The information manager from hell
   3 *
   4 * Copyright (C) Linus Torvalds, 2005
   5 *
   6 * This handles basic git sha1 object files - packing, unpacking,
   7 * creation etc.
   8 */
   9#include "cache.h"
  10#include "delta.h"
  11#include "pack.h"
  12#include "blob.h"
  13#include "commit.h"
  14#include "tag.h"
  15#include "tree.h"
  16#include "refs.h"
  17
  18#ifndef O_NOATIME
  19#if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
  20#define O_NOATIME 01000000
  21#else
  22#define O_NOATIME 0
  23#endif
  24#endif
  25
  26#ifdef NO_C99_FORMAT
  27#define SZ_FMT "lu"
  28#else
  29#define SZ_FMT "zu"
  30#endif
  31
  32const unsigned char null_sha1[20];
  33
  34static unsigned int sha1_file_open_flag = O_NOATIME;
  35
  36const signed char hexval_table[256] = {
  37         -1, -1, -1, -1, -1, -1, -1, -1,                /* 00-07 */
  38         -1, -1, -1, -1, -1, -1, -1, -1,                /* 08-0f */
  39         -1, -1, -1, -1, -1, -1, -1, -1,                /* 10-17 */
  40         -1, -1, -1, -1, -1, -1, -1, -1,                /* 18-1f */
  41         -1, -1, -1, -1, -1, -1, -1, -1,                /* 20-27 */
  42         -1, -1, -1, -1, -1, -1, -1, -1,                /* 28-2f */
  43          0,  1,  2,  3,  4,  5,  6,  7,                /* 30-37 */
  44          8,  9, -1, -1, -1, -1, -1, -1,                /* 38-3f */
  45         -1, 10, 11, 12, 13, 14, 15, -1,                /* 40-47 */
  46         -1, -1, -1, -1, -1, -1, -1, -1,                /* 48-4f */
  47         -1, -1, -1, -1, -1, -1, -1, -1,                /* 50-57 */
  48         -1, -1, -1, -1, -1, -1, -1, -1,                /* 58-5f */
  49         -1, 10, 11, 12, 13, 14, 15, -1,                /* 60-67 */
  50         -1, -1, -1, -1, -1, -1, -1, -1,                /* 68-67 */
  51         -1, -1, -1, -1, -1, -1, -1, -1,                /* 70-77 */
  52         -1, -1, -1, -1, -1, -1, -1, -1,                /* 78-7f */
  53         -1, -1, -1, -1, -1, -1, -1, -1,                /* 80-87 */
  54         -1, -1, -1, -1, -1, -1, -1, -1,                /* 88-8f */
  55         -1, -1, -1, -1, -1, -1, -1, -1,                /* 90-97 */
  56         -1, -1, -1, -1, -1, -1, -1, -1,                /* 98-9f */
  57         -1, -1, -1, -1, -1, -1, -1, -1,                /* a0-a7 */
  58         -1, -1, -1, -1, -1, -1, -1, -1,                /* a8-af */
  59         -1, -1, -1, -1, -1, -1, -1, -1,                /* b0-b7 */
  60         -1, -1, -1, -1, -1, -1, -1, -1,                /* b8-bf */
  61         -1, -1, -1, -1, -1, -1, -1, -1,                /* c0-c7 */
  62         -1, -1, -1, -1, -1, -1, -1, -1,                /* c8-cf */
  63         -1, -1, -1, -1, -1, -1, -1, -1,                /* d0-d7 */
  64         -1, -1, -1, -1, -1, -1, -1, -1,                /* d8-df */
  65         -1, -1, -1, -1, -1, -1, -1, -1,                /* e0-e7 */
  66         -1, -1, -1, -1, -1, -1, -1, -1,                /* e8-ef */
  67         -1, -1, -1, -1, -1, -1, -1, -1,                /* f0-f7 */
  68         -1, -1, -1, -1, -1, -1, -1, -1,                /* f8-ff */
  69};
  70
  71int get_sha1_hex(const char *hex, unsigned char *sha1)
  72{
  73        int i;
  74        for (i = 0; i < 20; i++) {
  75                unsigned int val = (hexval(hex[0]) << 4) | hexval(hex[1]);
  76                if (val & ~0xff)
  77                        return -1;
  78                *sha1++ = val;
  79                hex += 2;
  80        }
  81        return 0;
  82}
  83
  84int safe_create_leading_directories(char *path)
  85{
  86        char *pos = path;
  87        struct stat st;
  88
  89        if (*pos == '/')
  90                pos++;
  91
  92        while (pos) {
  93                pos = strchr(pos, '/');
  94                if (!pos)
  95                        break;
  96                *pos = 0;
  97                if (!stat(path, &st)) {
  98                        /* path exists */
  99                        if (!S_ISDIR(st.st_mode)) {
 100                                *pos = '/';
 101                                return -3;
 102                        }
 103                }
 104                else if (mkdir(path, 0777)) {
 105                        *pos = '/';
 106                        return -1;
 107                }
 108                else if (adjust_shared_perm(path)) {
 109                        *pos = '/';
 110                        return -2;
 111                }
 112                *pos++ = '/';
 113        }
 114        return 0;
 115}
 116
 117char * sha1_to_hex(const unsigned char *sha1)
 118{
 119        static int bufno;
 120        static char hexbuffer[4][50];
 121        static const char hex[] = "0123456789abcdef";
 122        char *buffer = hexbuffer[3 & ++bufno], *buf = buffer;
 123        int i;
 124
 125        for (i = 0; i < 20; i++) {
 126                unsigned int val = *sha1++;
 127                *buf++ = hex[val >> 4];
 128                *buf++ = hex[val & 0xf];
 129        }
 130        *buf = '\0';
 131
 132        return buffer;
 133}
 134
 135static void fill_sha1_path(char *pathbuf, const unsigned char *sha1)
 136{
 137        int i;
 138        for (i = 0; i < 20; i++) {
 139                static char hex[] = "0123456789abcdef";
 140                unsigned int val = sha1[i];
 141                char *pos = pathbuf + i*2 + (i > 0);
 142                *pos++ = hex[val >> 4];
 143                *pos = hex[val & 0xf];
 144        }
 145}
 146
 147/*
 148 * NOTE! This returns a statically allocated buffer, so you have to be
 149 * careful about using it. Do a "xstrdup()" if you need to save the
 150 * filename.
 151 *
 152 * Also note that this returns the location for creating.  Reading
 153 * SHA1 file can happen from any alternate directory listed in the
 154 * DB_ENVIRONMENT environment variable if it is not found in
 155 * the primary object database.
 156 */
 157char *sha1_file_name(const unsigned char *sha1)
 158{
 159        static char *name, *base;
 160
 161        if (!base) {
 162                const char *sha1_file_directory = get_object_directory();
 163                int len = strlen(sha1_file_directory);
 164                base = xmalloc(len + 60);
 165                memcpy(base, sha1_file_directory, len);
 166                memset(base+len, 0, 60);
 167                base[len] = '/';
 168                base[len+3] = '/';
 169                name = base + len + 1;
 170        }
 171        fill_sha1_path(name, sha1);
 172        return base;
 173}
 174
 175char *sha1_pack_name(const unsigned char *sha1)
 176{
 177        static const char hex[] = "0123456789abcdef";
 178        static char *name, *base, *buf;
 179        int i;
 180
 181        if (!base) {
 182                const char *sha1_file_directory = get_object_directory();
 183                int len = strlen(sha1_file_directory);
 184                base = xmalloc(len + 60);
 185                sprintf(base, "%s/pack/pack-1234567890123456789012345678901234567890.pack", sha1_file_directory);
 186                name = base + len + 11;
 187        }
 188
 189        buf = name;
 190
 191        for (i = 0; i < 20; i++) {
 192                unsigned int val = *sha1++;
 193                *buf++ = hex[val >> 4];
 194                *buf++ = hex[val & 0xf];
 195        }
 196
 197        return base;
 198}
 199
 200char *sha1_pack_index_name(const unsigned char *sha1)
 201{
 202        static const char hex[] = "0123456789abcdef";
 203        static char *name, *base, *buf;
 204        int i;
 205
 206        if (!base) {
 207                const char *sha1_file_directory = get_object_directory();
 208                int len = strlen(sha1_file_directory);
 209                base = xmalloc(len + 60);
 210                sprintf(base, "%s/pack/pack-1234567890123456789012345678901234567890.idx", sha1_file_directory);
 211                name = base + len + 11;
 212        }
 213
 214        buf = name;
 215
 216        for (i = 0; i < 20; i++) {
 217                unsigned int val = *sha1++;
 218                *buf++ = hex[val >> 4];
 219                *buf++ = hex[val & 0xf];
 220        }
 221
 222        return base;
 223}
 224
 225struct alternate_object_database *alt_odb_list;
 226static struct alternate_object_database **alt_odb_tail;
 227
 228static void read_info_alternates(const char * alternates, int depth);
 229
 230/*
 231 * Prepare alternate object database registry.
 232 *
 233 * The variable alt_odb_list points at the list of struct
 234 * alternate_object_database.  The elements on this list come from
 235 * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
 236 * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
 237 * whose contents is similar to that environment variable but can be
 238 * LF separated.  Its base points at a statically allocated buffer that
 239 * contains "/the/directory/corresponding/to/.git/objects/...", while
 240 * its name points just after the slash at the end of ".git/objects/"
 241 * in the example above, and has enough space to hold 40-byte hex
 242 * SHA1, an extra slash for the first level indirection, and the
 243 * terminating NUL.
 244 */
 245static int link_alt_odb_entry(const char * entry, int len, const char * relative_base, int depth)
 246{
 247        struct stat st;
 248        const char *objdir = get_object_directory();
 249        struct alternate_object_database *ent;
 250        struct alternate_object_database *alt;
 251        /* 43 = 40-byte + 2 '/' + terminating NUL */
 252        int pfxlen = len;
 253        int entlen = pfxlen + 43;
 254        int base_len = -1;
 255
 256        if (*entry != '/' && relative_base) {
 257                /* Relative alt-odb */
 258                if (base_len < 0)
 259                        base_len = strlen(relative_base) + 1;
 260                entlen += base_len;
 261                pfxlen += base_len;
 262        }
 263        ent = xmalloc(sizeof(*ent) + entlen);
 264
 265        if (*entry != '/' && relative_base) {
 266                memcpy(ent->base, relative_base, base_len - 1);
 267                ent->base[base_len - 1] = '/';
 268                memcpy(ent->base + base_len, entry, len);
 269        }
 270        else
 271                memcpy(ent->base, entry, pfxlen);
 272
 273        ent->name = ent->base + pfxlen + 1;
 274        ent->base[pfxlen + 3] = '/';
 275        ent->base[pfxlen] = ent->base[entlen-1] = 0;
 276
 277        /* Detect cases where alternate disappeared */
 278        if (stat(ent->base, &st) || !S_ISDIR(st.st_mode)) {
 279                error("object directory %s does not exist; "
 280                      "check .git/objects/info/alternates.",
 281                      ent->base);
 282                free(ent);
 283                return -1;
 284        }
 285
 286        /* Prevent the common mistake of listing the same
 287         * thing twice, or object directory itself.
 288         */
 289        for (alt = alt_odb_list; alt; alt = alt->next) {
 290                if (!memcmp(ent->base, alt->base, pfxlen)) {
 291                        free(ent);
 292                        return -1;
 293                }
 294        }
 295        if (!memcmp(ent->base, objdir, pfxlen)) {
 296                free(ent);
 297                return -1;
 298        }
 299
 300        /* add the alternate entry */
 301        *alt_odb_tail = ent;
 302        alt_odb_tail = &(ent->next);
 303        ent->next = NULL;
 304
 305        /* recursively add alternates */
 306        read_info_alternates(ent->base, depth + 1);
 307
 308        ent->base[pfxlen] = '/';
 309
 310        return 0;
 311}
 312
 313static void link_alt_odb_entries(const char *alt, const char *ep, int sep,
 314                                 const char *relative_base, int depth)
 315{
 316        const char *cp, *last;
 317
 318        if (depth > 5) {
 319                error("%s: ignoring alternate object stores, nesting too deep.",
 320                                relative_base);
 321                return;
 322        }
 323
 324        last = alt;
 325        while (last < ep) {
 326                cp = last;
 327                if (cp < ep && *cp == '#') {
 328                        while (cp < ep && *cp != sep)
 329                                cp++;
 330                        last = cp + 1;
 331                        continue;
 332                }
 333                while (cp < ep && *cp != sep)
 334                        cp++;
 335                if (last != cp) {
 336                        if ((*last != '/') && depth) {
 337                                error("%s: ignoring relative alternate object store %s",
 338                                                relative_base, last);
 339                        } else {
 340                                link_alt_odb_entry(last, cp - last,
 341                                                relative_base, depth);
 342                        }
 343                }
 344                while (cp < ep && *cp == sep)
 345                        cp++;
 346                last = cp;
 347        }
 348}
 349
 350static void read_info_alternates(const char * relative_base, int depth)
 351{
 352        char *map;
 353        size_t mapsz;
 354        struct stat st;
 355        const char alt_file_name[] = "info/alternates";
 356        /* Given that relative_base is no longer than PATH_MAX,
 357           ensure that "path" has enough space to append "/", the
 358           file name, "info/alternates", and a trailing NUL.  */
 359        char path[PATH_MAX + 1 + sizeof alt_file_name];
 360        int fd;
 361
 362        sprintf(path, "%s/%s", relative_base, alt_file_name);
 363        fd = open(path, O_RDONLY);
 364        if (fd < 0)
 365                return;
 366        if (fstat(fd, &st) || (st.st_size == 0)) {
 367                close(fd);
 368                return;
 369        }
 370        mapsz = xsize_t(st.st_size);
 371        map = xmmap(NULL, mapsz, PROT_READ, MAP_PRIVATE, fd, 0);
 372        close(fd);
 373
 374        link_alt_odb_entries(map, map + mapsz, '\n', relative_base, depth);
 375
 376        munmap(map, mapsz);
 377}
 378
 379void prepare_alt_odb(void)
 380{
 381        const char *alt;
 382
 383        if (alt_odb_tail)
 384                return;
 385
 386        alt = getenv(ALTERNATE_DB_ENVIRONMENT);
 387        if (!alt) alt = "";
 388
 389        alt_odb_tail = &alt_odb_list;
 390        link_alt_odb_entries(alt, alt + strlen(alt), ':', NULL, 0);
 391
 392        read_info_alternates(get_object_directory(), 0);
 393}
 394
 395static char *find_sha1_file(const unsigned char *sha1, struct stat *st)
 396{
 397        char *name = sha1_file_name(sha1);
 398        struct alternate_object_database *alt;
 399
 400        if (!stat(name, st))
 401                return name;
 402        prepare_alt_odb();
 403        for (alt = alt_odb_list; alt; alt = alt->next) {
 404                name = alt->name;
 405                fill_sha1_path(name, sha1);
 406                if (!stat(alt->base, st))
 407                        return alt->base;
 408        }
 409        return NULL;
 410}
 411
 412static unsigned int pack_used_ctr;
 413static unsigned int pack_mmap_calls;
 414static unsigned int peak_pack_open_windows;
 415static unsigned int pack_open_windows;
 416static size_t peak_pack_mapped;
 417static size_t pack_mapped;
 418struct packed_git *packed_git;
 419
 420void pack_report(void)
 421{
 422        fprintf(stderr,
 423                "pack_report: getpagesize()            = %10" SZ_FMT "\n"
 424                "pack_report: core.packedGitWindowSize = %10" SZ_FMT "\n"
 425                "pack_report: core.packedGitLimit      = %10" SZ_FMT "\n",
 426                (size_t) getpagesize(),
 427                packed_git_window_size,
 428                packed_git_limit);
 429        fprintf(stderr,
 430                "pack_report: pack_used_ctr            = %10u\n"
 431                "pack_report: pack_mmap_calls          = %10u\n"
 432                "pack_report: pack_open_windows        = %10u / %10u\n"
 433                "pack_report: pack_mapped              = "
 434                        "%10" SZ_FMT " / %10" SZ_FMT "\n",
 435                pack_used_ctr,
 436                pack_mmap_calls,
 437                pack_open_windows, peak_pack_open_windows,
 438                pack_mapped, peak_pack_mapped);
 439}
 440
 441static int check_packed_git_idx(const char *path,  struct packed_git *p)
 442{
 443        void *idx_map;
 444        struct pack_idx_header *hdr;
 445        size_t idx_size;
 446        uint32_t version, nr, i, *index;
 447        int fd = open(path, O_RDONLY);
 448        struct stat st;
 449
 450        if (fd < 0)
 451                return -1;
 452        if (fstat(fd, &st)) {
 453                close(fd);
 454                return -1;
 455        }
 456        idx_size = xsize_t(st.st_size);
 457        if (idx_size < 4 * 256 + 20 + 20) {
 458                close(fd);
 459                return error("index file %s is too small", path);
 460        }
 461        idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
 462        close(fd);
 463
 464        hdr = idx_map;
 465        if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
 466                version = ntohl(hdr->idx_version);
 467                if (version < 2 || version > 2) {
 468                        munmap(idx_map, idx_size);
 469                        return error("index file %s is version %d"
 470                                     " and is not supported by this binary"
 471                                     " (try upgrading GIT to a newer version)",
 472                                     path, version);
 473                }
 474        } else
 475                version = 1;
 476
 477        nr = 0;
 478        index = idx_map;
 479        if (version > 1)
 480                index += 2;  /* skip index header */
 481        for (i = 0; i < 256; i++) {
 482                uint32_t n = ntohl(index[i]);
 483                if (n < nr) {
 484                        munmap(idx_map, idx_size);
 485                        return error("non-monotonic index %s", path);
 486                }
 487                nr = n;
 488        }
 489
 490        if (version == 1) {
 491                /*
 492                 * Total size:
 493                 *  - 256 index entries 4 bytes each
 494                 *  - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
 495                 *  - 20-byte SHA1 of the packfile
 496                 *  - 20-byte SHA1 file checksum
 497                 */
 498                if (idx_size != 4*256 + nr * 24 + 20 + 20) {
 499                        munmap(idx_map, idx_size);
 500                        return error("wrong index v1 file size in %s", path);
 501                }
 502        } else if (version == 2) {
 503                /*
 504                 * Minimum size:
 505                 *  - 8 bytes of header
 506                 *  - 256 index entries 4 bytes each
 507                 *  - 20-byte sha1 entry * nr
 508                 *  - 4-byte crc entry * nr
 509                 *  - 4-byte offset entry * nr
 510                 *  - 20-byte SHA1 of the packfile
 511                 *  - 20-byte SHA1 file checksum
 512                 * And after the 4-byte offset table might be a
 513                 * variable sized table containing 8-byte entries
 514                 * for offsets larger than 2^31.
 515                 */
 516                unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
 517                unsigned long max_size = min_size;
 518                if (nr)
 519                        max_size += (nr - 1)*8;
 520                if (idx_size < min_size || idx_size > max_size) {
 521                        munmap(idx_map, idx_size);
 522                        return error("wrong index v2 file size in %s", path);
 523                }
 524                if (idx_size != min_size &&
 525                    /*
 526                     * make sure we can deal with large pack offsets.
 527                     * 31-bit signed offset won't be enough, neither
 528                     * 32-bit unsigned one will be.
 529                     */
 530                    (sizeof(off_t) <= 4)) {
 531                        munmap(idx_map, idx_size);
 532                        return error("pack too large for current definition of off_t in %s", path);
 533                }
 534        }
 535
 536        p->index_version = version;
 537        p->index_data = idx_map;
 538        p->index_size = idx_size;
 539        p->num_objects = nr;
 540        return 0;
 541}
 542
 543int open_pack_index(struct packed_git *p)
 544{
 545        char *idx_name;
 546        int ret;
 547
 548        if (p->index_data)
 549                return 0;
 550
 551        idx_name = xstrdup(p->pack_name);
 552        strcpy(idx_name + strlen(idx_name) - strlen(".pack"), ".idx");
 553        ret = check_packed_git_idx(idx_name, p);
 554        free(idx_name);
 555        return ret;
 556}
 557
 558static void scan_windows(struct packed_git *p,
 559        struct packed_git **lru_p,
 560        struct pack_window **lru_w,
 561        struct pack_window **lru_l)
 562{
 563        struct pack_window *w, *w_l;
 564
 565        for (w_l = NULL, w = p->windows; w; w = w->next) {
 566                if (!w->inuse_cnt) {
 567                        if (!*lru_w || w->last_used < (*lru_w)->last_used) {
 568                                *lru_p = p;
 569                                *lru_w = w;
 570                                *lru_l = w_l;
 571                        }
 572                }
 573                w_l = w;
 574        }
 575}
 576
 577static int unuse_one_window(struct packed_git *current, int keep_fd)
 578{
 579        struct packed_git *p, *lru_p = NULL;
 580        struct pack_window *lru_w = NULL, *lru_l = NULL;
 581
 582        if (current)
 583                scan_windows(current, &lru_p, &lru_w, &lru_l);
 584        for (p = packed_git; p; p = p->next)
 585                scan_windows(p, &lru_p, &lru_w, &lru_l);
 586        if (lru_p) {
 587                munmap(lru_w->base, lru_w->len);
 588                pack_mapped -= lru_w->len;
 589                if (lru_l)
 590                        lru_l->next = lru_w->next;
 591                else {
 592                        lru_p->windows = lru_w->next;
 593                        if (!lru_p->windows && lru_p->pack_fd != keep_fd) {
 594                                close(lru_p->pack_fd);
 595                                lru_p->pack_fd = -1;
 596                        }
 597                }
 598                free(lru_w);
 599                pack_open_windows--;
 600                return 1;
 601        }
 602        return 0;
 603}
 604
 605void release_pack_memory(size_t need, int fd)
 606{
 607        size_t cur = pack_mapped;
 608        while (need >= (cur - pack_mapped) && unuse_one_window(NULL, fd))
 609                ; /* nothing */
 610}
 611
 612void unuse_pack(struct pack_window **w_cursor)
 613{
 614        struct pack_window *w = *w_cursor;
 615        if (w) {
 616                w->inuse_cnt--;
 617                *w_cursor = NULL;
 618        }
 619}
 620
 621/*
 622 * Do not call this directly as this leaks p->pack_fd on error return;
 623 * call open_packed_git() instead.
 624 */
 625static int open_packed_git_1(struct packed_git *p)
 626{
 627        struct stat st;
 628        struct pack_header hdr;
 629        unsigned char sha1[20];
 630        unsigned char *idx_sha1;
 631        long fd_flag;
 632
 633        if (!p->index_data && open_pack_index(p))
 634                return error("packfile %s index unavailable", p->pack_name);
 635
 636        p->pack_fd = open(p->pack_name, O_RDONLY);
 637        if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
 638                return -1;
 639
 640        /* If we created the struct before we had the pack we lack size. */
 641        if (!p->pack_size) {
 642                if (!S_ISREG(st.st_mode))
 643                        return error("packfile %s not a regular file", p->pack_name);
 644                p->pack_size = st.st_size;
 645        } else if (p->pack_size != st.st_size)
 646                return error("packfile %s size changed", p->pack_name);
 647
 648        /* We leave these file descriptors open with sliding mmap;
 649         * there is no point keeping them open across exec(), though.
 650         */
 651        fd_flag = fcntl(p->pack_fd, F_GETFD, 0);
 652        if (fd_flag < 0)
 653                return error("cannot determine file descriptor flags");
 654        fd_flag |= FD_CLOEXEC;
 655        if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
 656                return error("cannot set FD_CLOEXEC");
 657
 658        /* Verify we recognize this pack file format. */
 659        if (read_in_full(p->pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
 660                return error("file %s is far too short to be a packfile", p->pack_name);
 661        if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
 662                return error("file %s is not a GIT packfile", p->pack_name);
 663        if (!pack_version_ok(hdr.hdr_version))
 664                return error("packfile %s is version %u and not supported"
 665                        " (try upgrading GIT to a newer version)",
 666                        p->pack_name, ntohl(hdr.hdr_version));
 667
 668        /* Verify the pack matches its index. */
 669        if (p->num_objects != ntohl(hdr.hdr_entries))
 670                return error("packfile %s claims to have %u objects"
 671                             " while index indicates %u objects",
 672                             p->pack_name, ntohl(hdr.hdr_entries),
 673                             p->num_objects);
 674        if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
 675                return error("end of packfile %s is unavailable", p->pack_name);
 676        if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1))
 677                return error("packfile %s signature is unavailable", p->pack_name);
 678        idx_sha1 = ((unsigned char *)p->index_data) + p->index_size - 40;
 679        if (hashcmp(sha1, idx_sha1))
 680                return error("packfile %s does not match index", p->pack_name);
 681        return 0;
 682}
 683
 684static int open_packed_git(struct packed_git *p)
 685{
 686        if (!open_packed_git_1(p))
 687                return 0;
 688        if (p->pack_fd != -1) {
 689                close(p->pack_fd);
 690                p->pack_fd = -1;
 691        }
 692        return -1;
 693}
 694
 695static int in_window(struct pack_window *win, off_t offset)
 696{
 697        /* We must promise at least 20 bytes (one hash) after the
 698         * offset is available from this window, otherwise the offset
 699         * is not actually in this window and a different window (which
 700         * has that one hash excess) must be used.  This is to support
 701         * the object header and delta base parsing routines below.
 702         */
 703        off_t win_off = win->offset;
 704        return win_off <= offset
 705                && (offset + 20) <= (win_off + win->len);
 706}
 707
 708unsigned char* use_pack(struct packed_git *p,
 709                struct pack_window **w_cursor,
 710                off_t offset,
 711                unsigned int *left)
 712{
 713        struct pack_window *win = *w_cursor;
 714
 715        if (p->pack_fd == -1 && open_packed_git(p))
 716                die("packfile %s cannot be accessed", p->pack_name);
 717
 718        /* Since packfiles end in a hash of their content and its
 719         * pointless to ask for an offset into the middle of that
 720         * hash, and the in_window function above wouldn't match
 721         * don't allow an offset too close to the end of the file.
 722         */
 723        if (offset > (p->pack_size - 20))
 724                die("offset beyond end of packfile (truncated pack?)");
 725
 726        if (!win || !in_window(win, offset)) {
 727                if (win)
 728                        win->inuse_cnt--;
 729                for (win = p->windows; win; win = win->next) {
 730                        if (in_window(win, offset))
 731                                break;
 732                }
 733                if (!win) {
 734                        size_t window_align = packed_git_window_size / 2;
 735                        off_t len;
 736                        win = xcalloc(1, sizeof(*win));
 737                        win->offset = (offset / window_align) * window_align;
 738                        len = p->pack_size - win->offset;
 739                        if (len > packed_git_window_size)
 740                                len = packed_git_window_size;
 741                        win->len = (size_t)len;
 742                        pack_mapped += win->len;
 743                        while (packed_git_limit < pack_mapped
 744                                && unuse_one_window(p, p->pack_fd))
 745                                ; /* nothing */
 746                        win->base = xmmap(NULL, win->len,
 747                                PROT_READ, MAP_PRIVATE,
 748                                p->pack_fd, win->offset);
 749                        if (win->base == MAP_FAILED)
 750                                die("packfile %s cannot be mapped: %s",
 751                                        p->pack_name,
 752                                        strerror(errno));
 753                        pack_mmap_calls++;
 754                        pack_open_windows++;
 755                        if (pack_mapped > peak_pack_mapped)
 756                                peak_pack_mapped = pack_mapped;
 757                        if (pack_open_windows > peak_pack_open_windows)
 758                                peak_pack_open_windows = pack_open_windows;
 759                        win->next = p->windows;
 760                        p->windows = win;
 761                }
 762        }
 763        if (win != *w_cursor) {
 764                win->last_used = pack_used_ctr++;
 765                win->inuse_cnt++;
 766                *w_cursor = win;
 767        }
 768        offset -= win->offset;
 769        if (left)
 770                *left = win->len - xsize_t(offset);
 771        return win->base + offset;
 772}
 773
 774struct packed_git *add_packed_git(const char *path, int path_len, int local)
 775{
 776        struct stat st;
 777        struct packed_git *p = xmalloc(sizeof(*p) + path_len + 2);
 778
 779        /*
 780         * Make sure a corresponding .pack file exists and that
 781         * the index looks sane.
 782         */
 783        path_len -= strlen(".idx");
 784        if (path_len < 1)
 785                return NULL;
 786        memcpy(p->pack_name, path, path_len);
 787        strcpy(p->pack_name + path_len, ".pack");
 788        if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
 789                free(p);
 790                return NULL;
 791        }
 792
 793        /* ok, it looks sane as far as we can check without
 794         * actually mapping the pack file.
 795         */
 796        p->index_version = 0;
 797        p->index_data = NULL;
 798        p->index_size = 0;
 799        p->num_objects = 0;
 800        p->pack_size = st.st_size;
 801        p->next = NULL;
 802        p->windows = NULL;
 803        p->pack_fd = -1;
 804        p->pack_local = local;
 805        p->mtime = st.st_mtime;
 806        if (path_len < 40 || get_sha1_hex(path + path_len - 40, p->sha1))
 807                hashclr(p->sha1);
 808        return p;
 809}
 810
 811struct packed_git *parse_pack_index(unsigned char *sha1)
 812{
 813        char *path = sha1_pack_index_name(sha1);
 814        return parse_pack_index_file(sha1, path);
 815}
 816
 817struct packed_git *parse_pack_index_file(const unsigned char *sha1,
 818                                         const char *idx_path)
 819{
 820        const char *path = sha1_pack_name(sha1);
 821        struct packed_git *p = xmalloc(sizeof(*p) + strlen(path) + 2);
 822
 823        if (check_packed_git_idx(idx_path, p)) {
 824                free(p);
 825                return NULL;
 826        }
 827
 828        strcpy(p->pack_name, path);
 829        p->pack_size = 0;
 830        p->next = NULL;
 831        p->windows = NULL;
 832        p->pack_fd = -1;
 833        hashcpy(p->sha1, sha1);
 834        return p;
 835}
 836
 837void install_packed_git(struct packed_git *pack)
 838{
 839        pack->next = packed_git;
 840        packed_git = pack;
 841}
 842
 843static void prepare_packed_git_one(char *objdir, int local)
 844{
 845        /* Ensure that this buffer is large enough so that we can
 846           append "/pack/" without clobbering the stack even if
 847           strlen(objdir) were PATH_MAX.  */
 848        char path[PATH_MAX + 1 + 4 + 1 + 1];
 849        int len;
 850        DIR *dir;
 851        struct dirent *de;
 852
 853        sprintf(path, "%s/pack", objdir);
 854        len = strlen(path);
 855        dir = opendir(path);
 856        if (!dir) {
 857                if (errno != ENOENT)
 858                        error("unable to open object pack directory: %s: %s",
 859                              path, strerror(errno));
 860                return;
 861        }
 862        path[len++] = '/';
 863        while ((de = readdir(dir)) != NULL) {
 864                int namelen = strlen(de->d_name);
 865                struct packed_git *p;
 866
 867                if (!has_extension(de->d_name, ".idx"))
 868                        continue;
 869
 870                if (len + namelen + 1 > sizeof(path))
 871                        continue;
 872
 873                /* Don't reopen a pack we already have. */
 874                strcpy(path + len, de->d_name);
 875                for (p = packed_git; p; p = p->next) {
 876                        if (!memcmp(path, p->pack_name, len + namelen - 4))
 877                                break;
 878                }
 879                if (p)
 880                        continue;
 881                /* See if it really is a valid .idx file with corresponding
 882                 * .pack file that we can map.
 883                 */
 884                p = add_packed_git(path, len + namelen, local);
 885                if (!p)
 886                        continue;
 887                install_packed_git(p);
 888        }
 889        closedir(dir);
 890}
 891
 892static int sort_pack(const void *a_, const void *b_)
 893{
 894        struct packed_git *a = *((struct packed_git **)a_);
 895        struct packed_git *b = *((struct packed_git **)b_);
 896        int st;
 897
 898        /*
 899         * Local packs tend to contain objects specific to our
 900         * variant of the project than remote ones.  In addition,
 901         * remote ones could be on a network mounted filesystem.
 902         * Favor local ones for these reasons.
 903         */
 904        st = a->pack_local - b->pack_local;
 905        if (st)
 906                return -st;
 907
 908        /*
 909         * Younger packs tend to contain more recent objects,
 910         * and more recent objects tend to get accessed more
 911         * often.
 912         */
 913        if (a->mtime < b->mtime)
 914                return 1;
 915        else if (a->mtime == b->mtime)
 916                return 0;
 917        return -1;
 918}
 919
 920static void rearrange_packed_git(void)
 921{
 922        struct packed_git **ary, *p;
 923        int i, n;
 924
 925        for (n = 0, p = packed_git; p; p = p->next)
 926                n++;
 927        if (n < 2)
 928                return;
 929
 930        /* prepare an array of packed_git for easier sorting */
 931        ary = xcalloc(n, sizeof(struct packed_git *));
 932        for (n = 0, p = packed_git; p; p = p->next)
 933                ary[n++] = p;
 934
 935        qsort(ary, n, sizeof(struct packed_git *), sort_pack);
 936
 937        /* link them back again */
 938        for (i = 0; i < n - 1; i++)
 939                ary[i]->next = ary[i + 1];
 940        ary[n - 1]->next = NULL;
 941        packed_git = ary[0];
 942
 943        free(ary);
 944}
 945
 946static int prepare_packed_git_run_once = 0;
 947void prepare_packed_git(void)
 948{
 949        struct alternate_object_database *alt;
 950
 951        if (prepare_packed_git_run_once)
 952                return;
 953        prepare_packed_git_one(get_object_directory(), 1);
 954        prepare_alt_odb();
 955        for (alt = alt_odb_list; alt; alt = alt->next) {
 956                alt->name[-1] = 0;
 957                prepare_packed_git_one(alt->base, 0);
 958                alt->name[-1] = '/';
 959        }
 960        rearrange_packed_git();
 961        prepare_packed_git_run_once = 1;
 962}
 963
 964void reprepare_packed_git(void)
 965{
 966        prepare_packed_git_run_once = 0;
 967        prepare_packed_git();
 968}
 969
 970int check_sha1_signature(const unsigned char *sha1, void *map, unsigned long size, const char *type)
 971{
 972        unsigned char real_sha1[20];
 973        hash_sha1_file(map, size, type, real_sha1);
 974        return hashcmp(sha1, real_sha1) ? -1 : 0;
 975}
 976
 977static void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
 978{
 979        struct stat st;
 980        void *map;
 981        int fd;
 982        char *filename = find_sha1_file(sha1, &st);
 983
 984        if (!filename) {
 985                return NULL;
 986        }
 987
 988        fd = open(filename, O_RDONLY | sha1_file_open_flag);
 989        if (fd < 0) {
 990                /* See if it works without O_NOATIME */
 991                switch (sha1_file_open_flag) {
 992                default:
 993                        fd = open(filename, O_RDONLY);
 994                        if (fd >= 0)
 995                                break;
 996                /* Fallthrough */
 997                case 0:
 998                        return NULL;
 999                }
1000
1001                /* If it failed once, it will probably fail again.
1002                 * Stop using O_NOATIME
1003                 */
1004                sha1_file_open_flag = 0;
1005        }
1006        *size = xsize_t(st.st_size);
1007        map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
1008        close(fd);
1009        return map;
1010}
1011
1012static int legacy_loose_object(unsigned char *map)
1013{
1014        unsigned int word;
1015
1016        /*
1017         * Is it a zlib-compressed buffer? If so, the first byte
1018         * must be 0x78 (15-bit window size, deflated), and the
1019         * first 16-bit word is evenly divisible by 31
1020         */
1021        word = (map[0] << 8) + map[1];
1022        if (map[0] == 0x78 && !(word % 31))
1023                return 1;
1024        else
1025                return 0;
1026}
1027
1028unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep)
1029{
1030        unsigned shift;
1031        unsigned char c;
1032        unsigned long size;
1033        unsigned long used = 0;
1034
1035        c = buf[used++];
1036        *type = (c >> 4) & 7;
1037        size = c & 15;
1038        shift = 4;
1039        while (c & 0x80) {
1040                if (len <= used)
1041                        return 0;
1042                if (sizeof(long) * 8 <= shift)
1043                        return 0;
1044                c = buf[used++];
1045                size += (c & 0x7f) << shift;
1046                shift += 7;
1047        }
1048        *sizep = size;
1049        return used;
1050}
1051
1052static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
1053{
1054        unsigned long size, used;
1055        static const char valid_loose_object_type[8] = {
1056                0, /* OBJ_EXT */
1057                1, 1, 1, 1, /* "commit", "tree", "blob", "tag" */
1058                0, /* "delta" and others are invalid in a loose object */
1059        };
1060        enum object_type type;
1061
1062        /* Get the data stream */
1063        memset(stream, 0, sizeof(*stream));
1064        stream->next_in = map;
1065        stream->avail_in = mapsize;
1066        stream->next_out = buffer;
1067        stream->avail_out = bufsiz;
1068
1069        if (legacy_loose_object(map)) {
1070                inflateInit(stream);
1071                return inflate(stream, 0);
1072        }
1073
1074
1075        /*
1076         * There used to be a second loose object header format which
1077         * was meant to mimic the in-pack format, allowing for direct
1078         * copy of the object data.  This format turned up not to be
1079         * really worth it and we don't write it any longer.  But we
1080         * can still read it.
1081         */
1082        used = unpack_object_header_gently(map, mapsize, &type, &size);
1083        if (!used || !valid_loose_object_type[type])
1084                return -1;
1085        map += used;
1086        mapsize -= used;
1087
1088        /* Set up the stream for the rest.. */
1089        stream->next_in = map;
1090        stream->avail_in = mapsize;
1091        inflateInit(stream);
1092
1093        /* And generate the fake traditional header */
1094        stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu",
1095                                         typename(type), size);
1096        return 0;
1097}
1098
1099static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
1100{
1101        int bytes = strlen(buffer) + 1;
1102        unsigned char *buf = xmalloc(1+size);
1103        unsigned long n;
1104        int status = Z_OK;
1105
1106        n = stream->total_out - bytes;
1107        if (n > size)
1108                n = size;
1109        memcpy(buf, (char *) buffer + bytes, n);
1110        bytes = n;
1111        if (bytes <= size) {
1112                /*
1113                 * The above condition must be (bytes <= size), not
1114                 * (bytes < size).  In other words, even though we
1115                 * expect no more output and set avail_out to zer0,
1116                 * the input zlib stream may have bytes that express
1117                 * "this concludes the stream", and we *do* want to
1118                 * eat that input.
1119                 *
1120                 * Otherwise we would not be able to test that we
1121                 * consumed all the input to reach the expected size;
1122                 * we also want to check that zlib tells us that all
1123                 * went well with status == Z_STREAM_END at the end.
1124                 */
1125                stream->next_out = buf + bytes;
1126                stream->avail_out = size - bytes;
1127                while (status == Z_OK)
1128                        status = inflate(stream, Z_FINISH);
1129        }
1130        buf[size] = 0;
1131        if (status == Z_STREAM_END && !stream->avail_in) {
1132                inflateEnd(stream);
1133                return buf;
1134        }
1135
1136        if (status < 0)
1137                error("corrupt loose object '%s'", sha1_to_hex(sha1));
1138        else if (stream->avail_in)
1139                error("garbage at end of loose object '%s'",
1140                      sha1_to_hex(sha1));
1141        free(buf);
1142        return NULL;
1143}
1144
1145/*
1146 * We used to just use "sscanf()", but that's actually way
1147 * too permissive for what we want to check. So do an anal
1148 * object header parse by hand.
1149 */
1150static int parse_sha1_header(const char *hdr, unsigned long *sizep)
1151{
1152        char type[10];
1153        int i;
1154        unsigned long size;
1155
1156        /*
1157         * The type can be at most ten bytes (including the
1158         * terminating '\0' that we add), and is followed by
1159         * a space.
1160         */
1161        i = 0;
1162        for (;;) {
1163                char c = *hdr++;
1164                if (c == ' ')
1165                        break;
1166                type[i++] = c;
1167                if (i >= sizeof(type))
1168                        return -1;
1169        }
1170        type[i] = 0;
1171
1172        /*
1173         * The length must follow immediately, and be in canonical
1174         * decimal format (ie "010" is not valid).
1175         */
1176        size = *hdr++ - '0';
1177        if (size > 9)
1178                return -1;
1179        if (size) {
1180                for (;;) {
1181                        unsigned long c = *hdr - '0';
1182                        if (c > 9)
1183                                break;
1184                        hdr++;
1185                        size = size * 10 + c;
1186                }
1187        }
1188        *sizep = size;
1189
1190        /*
1191         * The length must be followed by a zero byte
1192         */
1193        return *hdr ? -1 : type_from_string(type);
1194}
1195
1196static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
1197{
1198        int ret;
1199        z_stream stream;
1200        char hdr[8192];
1201
1202        ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr));
1203        if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0)
1204                return NULL;
1205
1206        return unpack_sha1_rest(&stream, hdr, *size, sha1);
1207}
1208
1209unsigned long get_size_from_delta(struct packed_git *p,
1210                                  struct pack_window **w_curs,
1211                                  off_t curpos)
1212{
1213        const unsigned char *data;
1214        unsigned char delta_head[20], *in;
1215        z_stream stream;
1216        int st;
1217
1218        memset(&stream, 0, sizeof(stream));
1219        stream.next_out = delta_head;
1220        stream.avail_out = sizeof(delta_head);
1221
1222        inflateInit(&stream);
1223        do {
1224                in = use_pack(p, w_curs, curpos, &stream.avail_in);
1225                stream.next_in = in;
1226                st = inflate(&stream, Z_FINISH);
1227                curpos += stream.next_in - in;
1228        } while ((st == Z_OK || st == Z_BUF_ERROR) &&
1229                 stream.total_out < sizeof(delta_head));
1230        inflateEnd(&stream);
1231        if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head))
1232                die("delta data unpack-initial failed");
1233
1234        /* Examine the initial part of the delta to figure out
1235         * the result size.
1236         */
1237        data = delta_head;
1238
1239        /* ignore base size */
1240        get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
1241
1242        /* Read the result size */
1243        return get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
1244}
1245
1246static off_t get_delta_base(struct packed_git *p,
1247                                    struct pack_window **w_curs,
1248                                    off_t *curpos,
1249                                    enum object_type type,
1250                                    off_t delta_obj_offset)
1251{
1252        unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL);
1253        off_t base_offset;
1254
1255        /* use_pack() assured us we have [base_info, base_info + 20)
1256         * as a range that we can look at without walking off the
1257         * end of the mapped window.  Its actually the hash size
1258         * that is assured.  An OFS_DELTA longer than the hash size
1259         * is stupid, as then a REF_DELTA would be smaller to store.
1260         */
1261        if (type == OBJ_OFS_DELTA) {
1262                unsigned used = 0;
1263                unsigned char c = base_info[used++];
1264                base_offset = c & 127;
1265                while (c & 128) {
1266                        base_offset += 1;
1267                        if (!base_offset || MSB(base_offset, 7))
1268                                die("offset value overflow for delta base object");
1269                        c = base_info[used++];
1270                        base_offset = (base_offset << 7) + (c & 127);
1271                }
1272                base_offset = delta_obj_offset - base_offset;
1273                if (base_offset >= delta_obj_offset)
1274                        die("delta base offset out of bound");
1275                *curpos += used;
1276        } else if (type == OBJ_REF_DELTA) {
1277                /* The base entry _must_ be in the same pack */
1278                base_offset = find_pack_entry_one(base_info, p);
1279                if (!base_offset)
1280                        die("failed to find delta-pack base object %s",
1281                                sha1_to_hex(base_info));
1282                *curpos += 20;
1283        } else
1284                die("I am totally screwed");
1285        return base_offset;
1286}
1287
1288/* forward declaration for a mutually recursive function */
1289static int packed_object_info(struct packed_git *p, off_t offset,
1290                              unsigned long *sizep);
1291
1292static int packed_delta_info(struct packed_git *p,
1293                             struct pack_window **w_curs,
1294                             off_t curpos,
1295                             enum object_type type,
1296                             off_t obj_offset,
1297                             unsigned long *sizep)
1298{
1299        off_t base_offset;
1300
1301        base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
1302        type = packed_object_info(p, base_offset, NULL);
1303
1304        /* We choose to only get the type of the base object and
1305         * ignore potentially corrupt pack file that expects the delta
1306         * based on a base with a wrong size.  This saves tons of
1307         * inflate() calls.
1308         */
1309        if (sizep)
1310                *sizep = get_size_from_delta(p, w_curs, curpos);
1311
1312        return type;
1313}
1314
1315static int unpack_object_header(struct packed_git *p,
1316                                struct pack_window **w_curs,
1317                                off_t *curpos,
1318                                unsigned long *sizep)
1319{
1320        unsigned char *base;
1321        unsigned int left;
1322        unsigned long used;
1323        enum object_type type;
1324
1325        /* use_pack() assures us we have [base, base + 20) available
1326         * as a range that we can look at at.  (Its actually the hash
1327         * size that is assured.)  With our object header encoding
1328         * the maximum deflated object size is 2^137, which is just
1329         * insane, so we know won't exceed what we have been given.
1330         */
1331        base = use_pack(p, w_curs, *curpos, &left);
1332        used = unpack_object_header_gently(base, left, &type, sizep);
1333        if (!used)
1334                die("object offset outside of pack file");
1335        *curpos += used;
1336
1337        return type;
1338}
1339
1340const char *packed_object_info_detail(struct packed_git *p,
1341                                      off_t obj_offset,
1342                                      unsigned long *size,
1343                                      unsigned long *store_size,
1344                                      unsigned int *delta_chain_length,
1345                                      unsigned char *base_sha1)
1346{
1347        struct pack_window *w_curs = NULL;
1348        off_t curpos;
1349        unsigned long dummy;
1350        unsigned char *next_sha1;
1351        enum object_type type;
1352
1353        *delta_chain_length = 0;
1354        curpos = obj_offset;
1355        type = unpack_object_header(p, &w_curs, &curpos, size);
1356
1357        for (;;) {
1358                switch (type) {
1359                default:
1360                        die("pack %s contains unknown object type %d",
1361                            p->pack_name, type);
1362                case OBJ_COMMIT:
1363                case OBJ_TREE:
1364                case OBJ_BLOB:
1365                case OBJ_TAG:
1366                        *store_size = 0; /* notyet */
1367                        unuse_pack(&w_curs);
1368                        return typename(type);
1369                case OBJ_OFS_DELTA:
1370                        obj_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
1371                        if (*delta_chain_length == 0) {
1372                                /* TODO: find base_sha1 as pointed by curpos */
1373                                hashclr(base_sha1);
1374                        }
1375                        break;
1376                case OBJ_REF_DELTA:
1377                        next_sha1 = use_pack(p, &w_curs, curpos, NULL);
1378                        if (*delta_chain_length == 0)
1379                                hashcpy(base_sha1, next_sha1);
1380                        obj_offset = find_pack_entry_one(next_sha1, p);
1381                        break;
1382                }
1383                (*delta_chain_length)++;
1384                curpos = obj_offset;
1385                type = unpack_object_header(p, &w_curs, &curpos, &dummy);
1386        }
1387}
1388
1389static int packed_object_info(struct packed_git *p, off_t obj_offset,
1390                              unsigned long *sizep)
1391{
1392        struct pack_window *w_curs = NULL;
1393        unsigned long size;
1394        off_t curpos = obj_offset;
1395        enum object_type type;
1396
1397        type = unpack_object_header(p, &w_curs, &curpos, &size);
1398
1399        switch (type) {
1400        case OBJ_OFS_DELTA:
1401        case OBJ_REF_DELTA:
1402                type = packed_delta_info(p, &w_curs, curpos,
1403                                         type, obj_offset, sizep);
1404                break;
1405        case OBJ_COMMIT:
1406        case OBJ_TREE:
1407        case OBJ_BLOB:
1408        case OBJ_TAG:
1409                if (sizep)
1410                        *sizep = size;
1411                break;
1412        default:
1413                die("pack %s contains unknown object type %d",
1414                    p->pack_name, type);
1415        }
1416        unuse_pack(&w_curs);
1417        return type;
1418}
1419
1420static void *unpack_compressed_entry(struct packed_git *p,
1421                                    struct pack_window **w_curs,
1422                                    off_t curpos,
1423                                    unsigned long size)
1424{
1425        int st;
1426        z_stream stream;
1427        unsigned char *buffer, *in;
1428
1429        buffer = xmalloc(size + 1);
1430        buffer[size] = 0;
1431        memset(&stream, 0, sizeof(stream));
1432        stream.next_out = buffer;
1433        stream.avail_out = size;
1434
1435        inflateInit(&stream);
1436        do {
1437                in = use_pack(p, w_curs, curpos, &stream.avail_in);
1438                stream.next_in = in;
1439                st = inflate(&stream, Z_FINISH);
1440                curpos += stream.next_in - in;
1441        } while (st == Z_OK || st == Z_BUF_ERROR);
1442        inflateEnd(&stream);
1443        if ((st != Z_STREAM_END) || stream.total_out != size) {
1444                free(buffer);
1445                return NULL;
1446        }
1447
1448        return buffer;
1449}
1450
1451#define MAX_DELTA_CACHE (256)
1452
1453static size_t delta_base_cached;
1454
1455static struct delta_base_cache_lru_list {
1456        struct delta_base_cache_lru_list *prev;
1457        struct delta_base_cache_lru_list *next;
1458} delta_base_cache_lru = { &delta_base_cache_lru, &delta_base_cache_lru };
1459
1460static struct delta_base_cache_entry {
1461        struct delta_base_cache_lru_list lru;
1462        void *data;
1463        struct packed_git *p;
1464        off_t base_offset;
1465        unsigned long size;
1466        enum object_type type;
1467} delta_base_cache[MAX_DELTA_CACHE];
1468
1469static unsigned long pack_entry_hash(struct packed_git *p, off_t base_offset)
1470{
1471        unsigned long hash;
1472
1473        hash = (unsigned long)p + (unsigned long)base_offset;
1474        hash += (hash >> 8) + (hash >> 16);
1475        return hash % MAX_DELTA_CACHE;
1476}
1477
1478static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
1479        unsigned long *base_size, enum object_type *type, int keep_cache)
1480{
1481        void *ret;
1482        unsigned long hash = pack_entry_hash(p, base_offset);
1483        struct delta_base_cache_entry *ent = delta_base_cache + hash;
1484
1485        ret = ent->data;
1486        if (ret && ent->p == p && ent->base_offset == base_offset)
1487                goto found_cache_entry;
1488        return unpack_entry(p, base_offset, type, base_size);
1489
1490found_cache_entry:
1491        if (!keep_cache) {
1492                ent->data = NULL;
1493                ent->lru.next->prev = ent->lru.prev;
1494                ent->lru.prev->next = ent->lru.next;
1495                delta_base_cached -= ent->size;
1496        }
1497        else {
1498                ret = xmalloc(ent->size + 1);
1499                memcpy(ret, ent->data, ent->size);
1500                ((char *)ret)[ent->size] = 0;
1501        }
1502        *type = ent->type;
1503        *base_size = ent->size;
1504        return ret;
1505}
1506
1507static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
1508{
1509        if (ent->data) {
1510                free(ent->data);
1511                ent->data = NULL;
1512                ent->lru.next->prev = ent->lru.prev;
1513                ent->lru.prev->next = ent->lru.next;
1514                delta_base_cached -= ent->size;
1515        }
1516}
1517
1518static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
1519        void *base, unsigned long base_size, enum object_type type)
1520{
1521        unsigned long hash = pack_entry_hash(p, base_offset);
1522        struct delta_base_cache_entry *ent = delta_base_cache + hash;
1523        struct delta_base_cache_lru_list *lru;
1524
1525        release_delta_base_cache(ent);
1526        delta_base_cached += base_size;
1527
1528        for (lru = delta_base_cache_lru.next;
1529             delta_base_cached > delta_base_cache_limit
1530             && lru != &delta_base_cache_lru;
1531             lru = lru->next) {
1532                struct delta_base_cache_entry *f = (void *)lru;
1533                if (f->type == OBJ_BLOB)
1534                        release_delta_base_cache(f);
1535        }
1536        for (lru = delta_base_cache_lru.next;
1537             delta_base_cached > delta_base_cache_limit
1538             && lru != &delta_base_cache_lru;
1539             lru = lru->next) {
1540                struct delta_base_cache_entry *f = (void *)lru;
1541                release_delta_base_cache(f);
1542        }
1543
1544        ent->p = p;
1545        ent->base_offset = base_offset;
1546        ent->type = type;
1547        ent->data = base;
1548        ent->size = base_size;
1549        ent->lru.next = &delta_base_cache_lru;
1550        ent->lru.prev = delta_base_cache_lru.prev;
1551        delta_base_cache_lru.prev->next = &ent->lru;
1552        delta_base_cache_lru.prev = &ent->lru;
1553}
1554
1555static void *unpack_delta_entry(struct packed_git *p,
1556                                struct pack_window **w_curs,
1557                                off_t curpos,
1558                                unsigned long delta_size,
1559                                off_t obj_offset,
1560                                enum object_type *type,
1561                                unsigned long *sizep)
1562{
1563        void *delta_data, *result, *base;
1564        unsigned long base_size;
1565        off_t base_offset;
1566
1567        base_offset = get_delta_base(p, w_curs, &curpos, *type, obj_offset);
1568        base = cache_or_unpack_entry(p, base_offset, &base_size, type, 0);
1569        if (!base)
1570                die("failed to read delta base object"
1571                    " at %"PRIuMAX" from %s",
1572                    (uintmax_t)base_offset, p->pack_name);
1573
1574        delta_data = unpack_compressed_entry(p, w_curs, curpos, delta_size);
1575        if (!delta_data)
1576                die("failed to unpack compressed delta"
1577                    " at %"PRIuMAX" from %s",
1578                    (uintmax_t)curpos, p->pack_name);
1579        result = patch_delta(base, base_size,
1580                             delta_data, delta_size,
1581                             sizep);
1582        if (!result)
1583                die("failed to apply delta");
1584        free(delta_data);
1585        add_delta_base_cache(p, base_offset, base, base_size, *type);
1586        return result;
1587}
1588
1589void *unpack_entry(struct packed_git *p, off_t obj_offset,
1590                   enum object_type *type, unsigned long *sizep)
1591{
1592        struct pack_window *w_curs = NULL;
1593        off_t curpos = obj_offset;
1594        void *data;
1595
1596        *type = unpack_object_header(p, &w_curs, &curpos, sizep);
1597        switch (*type) {
1598        case OBJ_OFS_DELTA:
1599        case OBJ_REF_DELTA:
1600                data = unpack_delta_entry(p, &w_curs, curpos, *sizep,
1601                                          obj_offset, type, sizep);
1602                break;
1603        case OBJ_COMMIT:
1604        case OBJ_TREE:
1605        case OBJ_BLOB:
1606        case OBJ_TAG:
1607                data = unpack_compressed_entry(p, &w_curs, curpos, *sizep);
1608                break;
1609        default:
1610                die("unknown object type %i in %s", *type, p->pack_name);
1611        }
1612        unuse_pack(&w_curs);
1613        return data;
1614}
1615
1616const unsigned char *nth_packed_object_sha1(struct packed_git *p,
1617                                            uint32_t n)
1618{
1619        const unsigned char *index = p->index_data;
1620        if (!index) {
1621                if (open_pack_index(p))
1622                        return NULL;
1623                index = p->index_data;
1624        }
1625        if (n >= p->num_objects)
1626                return NULL;
1627        index += 4 * 256;
1628        if (p->index_version == 1) {
1629                return index + 24 * n + 4;
1630        } else {
1631                index += 8;
1632                return index + 20 * n;
1633        }
1634}
1635
1636static off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
1637{
1638        const unsigned char *index = p->index_data;
1639        index += 4 * 256;
1640        if (p->index_version == 1) {
1641                return ntohl(*((uint32_t *)(index + 24 * n)));
1642        } else {
1643                uint32_t off;
1644                index += 8 + p->num_objects * (20 + 4);
1645                off = ntohl(*((uint32_t *)(index + 4 * n)));
1646                if (!(off & 0x80000000))
1647                        return off;
1648                index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
1649                return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
1650                                   ntohl(*((uint32_t *)(index + 4)));
1651        }
1652}
1653
1654off_t find_pack_entry_one(const unsigned char *sha1,
1655                                  struct packed_git *p)
1656{
1657        const uint32_t *level1_ofs = p->index_data;
1658        const unsigned char *index = p->index_data;
1659        unsigned hi, lo;
1660
1661        if (!index) {
1662                if (open_pack_index(p))
1663                        return 0;
1664                level1_ofs = p->index_data;
1665                index = p->index_data;
1666        }
1667        if (p->index_version > 1) {
1668                level1_ofs += 2;
1669                index += 8;
1670        }
1671        index += 4 * 256;
1672        hi = ntohl(level1_ofs[*sha1]);
1673        lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
1674
1675        do {
1676                unsigned mi = (lo + hi) / 2;
1677                unsigned x = (p->index_version > 1) ? (mi * 20) : (mi * 24 + 4);
1678                int cmp = hashcmp(index + x, sha1);
1679                if (!cmp)
1680                        return nth_packed_object_offset(p, mi);
1681                if (cmp > 0)
1682                        hi = mi;
1683                else
1684                        lo = mi+1;
1685        } while (lo < hi);
1686        return 0;
1687}
1688
1689static int matches_pack_name(struct packed_git *p, const char *ig)
1690{
1691        const char *last_c, *c;
1692
1693        if (!strcmp(p->pack_name, ig))
1694                return 0;
1695
1696        for (c = p->pack_name, last_c = c; *c;)
1697                if (*c == '/')
1698                        last_c = ++c;
1699                else
1700                        ++c;
1701        if (!strcmp(last_c, ig))
1702                return 0;
1703
1704        return 1;
1705}
1706
1707static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e, const char **ignore_packed)
1708{
1709        static struct packed_git *last_found = (void *)1;
1710        struct packed_git *p;
1711        off_t offset;
1712
1713        prepare_packed_git();
1714        if (!packed_git)
1715                return 0;
1716        p = (last_found == (void *)1) ? packed_git : last_found;
1717
1718        do {
1719                if (ignore_packed) {
1720                        const char **ig;
1721                        for (ig = ignore_packed; *ig; ig++)
1722                                if (!matches_pack_name(p, *ig))
1723                                        break;
1724                        if (*ig)
1725                                goto next;
1726                }
1727
1728                offset = find_pack_entry_one(sha1, p);
1729                if (offset) {
1730                        /*
1731                         * We are about to tell the caller where they can
1732                         * locate the requested object.  We better make
1733                         * sure the packfile is still here and can be
1734                         * accessed before supplying that answer, as
1735                         * it may have been deleted since the index
1736                         * was loaded!
1737                         */
1738                        if (p->pack_fd == -1 && open_packed_git(p)) {
1739                                error("packfile %s cannot be accessed", p->pack_name);
1740                                goto next;
1741                        }
1742                        e->offset = offset;
1743                        e->p = p;
1744                        hashcpy(e->sha1, sha1);
1745                        last_found = p;
1746                        return 1;
1747                }
1748
1749                next:
1750                if (p == last_found)
1751                        p = packed_git;
1752                else
1753                        p = p->next;
1754                if (p == last_found)
1755                        p = p->next;
1756        } while (p);
1757        return 0;
1758}
1759
1760struct packed_git *find_sha1_pack(const unsigned char *sha1,
1761                                  struct packed_git *packs)
1762{
1763        struct packed_git *p;
1764
1765        for (p = packs; p; p = p->next) {
1766                if (find_pack_entry_one(sha1, p))
1767                        return p;
1768        }
1769        return NULL;
1770
1771}
1772
1773static int sha1_loose_object_info(const unsigned char *sha1, unsigned long *sizep)
1774{
1775        int status;
1776        unsigned long mapsize, size;
1777        void *map;
1778        z_stream stream;
1779        char hdr[32];
1780
1781        map = map_sha1_file(sha1, &mapsize);
1782        if (!map)
1783                return error("unable to find %s", sha1_to_hex(sha1));
1784        if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
1785                status = error("unable to unpack %s header",
1786                               sha1_to_hex(sha1));
1787        else if ((status = parse_sha1_header(hdr, &size)) < 0)
1788                status = error("unable to parse %s header", sha1_to_hex(sha1));
1789        else if (sizep)
1790                *sizep = size;
1791        inflateEnd(&stream);
1792        munmap(map, mapsize);
1793        return status;
1794}
1795
1796int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
1797{
1798        struct pack_entry e;
1799
1800        if (!find_pack_entry(sha1, &e, NULL)) {
1801                reprepare_packed_git();
1802                if (!find_pack_entry(sha1, &e, NULL))
1803                        return sha1_loose_object_info(sha1, sizep);
1804        }
1805        return packed_object_info(e.p, e.offset, sizep);
1806}
1807
1808static void *read_packed_sha1(const unsigned char *sha1,
1809                              enum object_type *type, unsigned long *size)
1810{
1811        struct pack_entry e;
1812
1813        if (!find_pack_entry(sha1, &e, NULL))
1814                return NULL;
1815        else
1816                return cache_or_unpack_entry(e.p, e.offset, size, type, 1);
1817}
1818
1819/*
1820 * This is meant to hold a *small* number of objects that you would
1821 * want read_sha1_file() to be able to return, but yet you do not want
1822 * to write them into the object store (e.g. a browse-only
1823 * application).
1824 */
1825static struct cached_object {
1826        unsigned char sha1[20];
1827        enum object_type type;
1828        void *buf;
1829        unsigned long size;
1830} *cached_objects;
1831static int cached_object_nr, cached_object_alloc;
1832
1833static struct cached_object *find_cached_object(const unsigned char *sha1)
1834{
1835        int i;
1836        struct cached_object *co = cached_objects;
1837
1838        for (i = 0; i < cached_object_nr; i++, co++) {
1839                if (!hashcmp(co->sha1, sha1))
1840                        return co;
1841        }
1842        return NULL;
1843}
1844
1845int pretend_sha1_file(void *buf, unsigned long len, enum object_type type,
1846                      unsigned char *sha1)
1847{
1848        struct cached_object *co;
1849
1850        hash_sha1_file(buf, len, typename(type), sha1);
1851        if (has_sha1_file(sha1) || find_cached_object(sha1))
1852                return 0;
1853        if (cached_object_alloc <= cached_object_nr) {
1854                cached_object_alloc = alloc_nr(cached_object_alloc);
1855                cached_objects = xrealloc(cached_objects,
1856                                          sizeof(*cached_objects) *
1857                                          cached_object_alloc);
1858        }
1859        co = &cached_objects[cached_object_nr++];
1860        co->size = len;
1861        co->type = type;
1862        co->buf = xmalloc(len);
1863        memcpy(co->buf, buf, len);
1864        hashcpy(co->sha1, sha1);
1865        return 0;
1866}
1867
1868void *read_sha1_file(const unsigned char *sha1, enum object_type *type,
1869                     unsigned long *size)
1870{
1871        unsigned long mapsize;
1872        void *map, *buf;
1873        struct cached_object *co;
1874
1875        co = find_cached_object(sha1);
1876        if (co) {
1877                buf = xmalloc(co->size + 1);
1878                memcpy(buf, co->buf, co->size);
1879                ((char*)buf)[co->size] = 0;
1880                *type = co->type;
1881                *size = co->size;
1882                return buf;
1883        }
1884
1885        buf = read_packed_sha1(sha1, type, size);
1886        if (buf)
1887                return buf;
1888        map = map_sha1_file(sha1, &mapsize);
1889        if (map) {
1890                buf = unpack_sha1_file(map, mapsize, type, size, sha1);
1891                munmap(map, mapsize);
1892                return buf;
1893        }
1894        reprepare_packed_git();
1895        return read_packed_sha1(sha1, type, size);
1896}
1897
1898void *read_object_with_reference(const unsigned char *sha1,
1899                                 const char *required_type_name,
1900                                 unsigned long *size,
1901                                 unsigned char *actual_sha1_return)
1902{
1903        enum object_type type, required_type;
1904        void *buffer;
1905        unsigned long isize;
1906        unsigned char actual_sha1[20];
1907
1908        required_type = type_from_string(required_type_name);
1909        hashcpy(actual_sha1, sha1);
1910        while (1) {
1911                int ref_length = -1;
1912                const char *ref_type = NULL;
1913
1914                buffer = read_sha1_file(actual_sha1, &type, &isize);
1915                if (!buffer)
1916                        return NULL;
1917                if (type == required_type) {
1918                        *size = isize;
1919                        if (actual_sha1_return)
1920                                hashcpy(actual_sha1_return, actual_sha1);
1921                        return buffer;
1922                }
1923                /* Handle references */
1924                else if (type == OBJ_COMMIT)
1925                        ref_type = "tree ";
1926                else if (type == OBJ_TAG)
1927                        ref_type = "object ";
1928                else {
1929                        free(buffer);
1930                        return NULL;
1931                }
1932                ref_length = strlen(ref_type);
1933
1934                if (memcmp(buffer, ref_type, ref_length) ||
1935                    get_sha1_hex((char *) buffer + ref_length, actual_sha1)) {
1936                        free(buffer);
1937                        return NULL;
1938                }
1939                free(buffer);
1940                /* Now we have the ID of the referred-to object in
1941                 * actual_sha1.  Check again. */
1942        }
1943}
1944
1945static void write_sha1_file_prepare(const void *buf, unsigned long len,
1946                                    const char *type, unsigned char *sha1,
1947                                    char *hdr, int *hdrlen)
1948{
1949        SHA_CTX c;
1950
1951        /* Generate the header */
1952        *hdrlen = sprintf(hdr, "%s %lu", type, len)+1;
1953
1954        /* Sha1.. */
1955        SHA1_Init(&c);
1956        SHA1_Update(&c, hdr, *hdrlen);
1957        SHA1_Update(&c, buf, len);
1958        SHA1_Final(sha1, &c);
1959}
1960
1961/*
1962 * Link the tempfile to the final place, possibly creating the
1963 * last directory level as you do so.
1964 *
1965 * Returns the errno on failure, 0 on success.
1966 */
1967static int link_temp_to_file(const char *tmpfile, const char *filename)
1968{
1969        int ret;
1970        char *dir;
1971
1972        if (!link(tmpfile, filename))
1973                return 0;
1974
1975        /*
1976         * Try to mkdir the last path component if that failed.
1977         *
1978         * Re-try the "link()" regardless of whether the mkdir
1979         * succeeds, since a race might mean that somebody
1980         * else succeeded.
1981         */
1982        ret = errno;
1983        dir = strrchr(filename, '/');
1984        if (dir) {
1985                *dir = 0;
1986                if (!mkdir(filename, 0777) && adjust_shared_perm(filename)) {
1987                        *dir = '/';
1988                        return -2;
1989                }
1990                *dir = '/';
1991                if (!link(tmpfile, filename))
1992                        return 0;
1993                ret = errno;
1994        }
1995        return ret;
1996}
1997
1998/*
1999 * Move the just written object into its final resting place
2000 */
2001int move_temp_to_file(const char *tmpfile, const char *filename)
2002{
2003        int ret = link_temp_to_file(tmpfile, filename);
2004
2005        /*
2006         * Coda hack - coda doesn't like cross-directory links,
2007         * so we fall back to a rename, which will mean that it
2008         * won't be able to check collisions, but that's not a
2009         * big deal.
2010         *
2011         * The same holds for FAT formatted media.
2012         *
2013         * When this succeeds, we just return 0. We have nothing
2014         * left to unlink.
2015         */
2016        if (ret && ret != EEXIST) {
2017                if (!rename(tmpfile, filename))
2018                        return 0;
2019                ret = errno;
2020        }
2021        unlink(tmpfile);
2022        if (ret) {
2023                if (ret != EEXIST) {
2024                        return error("unable to write sha1 filename %s: %s\n", filename, strerror(ret));
2025                }
2026                /* FIXME!!! Collision check here ? */
2027        }
2028
2029        return 0;
2030}
2031
2032static int write_buffer(int fd, const void *buf, size_t len)
2033{
2034        if (write_in_full(fd, buf, len) < 0)
2035                return error("file write error (%s)", strerror(errno));
2036        return 0;
2037}
2038
2039int hash_sha1_file(const void *buf, unsigned long len, const char *type,
2040                   unsigned char *sha1)
2041{
2042        char hdr[32];
2043        int hdrlen;
2044        write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
2045        return 0;
2046}
2047
2048int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
2049{
2050        int size, ret;
2051        unsigned char *compressed;
2052        z_stream stream;
2053        unsigned char sha1[20];
2054        char *filename;
2055        static char tmpfile[PATH_MAX];
2056        char hdr[32];
2057        int fd, hdrlen;
2058
2059        /* Normally if we have it in the pack then we do not bother writing
2060         * it out into .git/objects/??/?{38} file.
2061         */
2062        write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
2063        filename = sha1_file_name(sha1);
2064        if (returnsha1)
2065                hashcpy(returnsha1, sha1);
2066        if (has_sha1_file(sha1))
2067                return 0;
2068        fd = open(filename, O_RDONLY);
2069        if (fd >= 0) {
2070                /*
2071                 * FIXME!!! We might do collision checking here, but we'd
2072                 * need to uncompress the old file and check it. Later.
2073                 */
2074                close(fd);
2075                return 0;
2076        }
2077
2078        if (errno != ENOENT) {
2079                return error("sha1 file %s: %s\n", filename, strerror(errno));
2080        }
2081
2082        snprintf(tmpfile, sizeof(tmpfile), "%s/tmp_obj_XXXXXX", get_object_directory());
2083
2084        fd = mkstemp(tmpfile);
2085        if (fd < 0) {
2086                if (errno == EPERM)
2087                        return error("insufficient permission for adding an object to repository database %s\n", get_object_directory());
2088                else
2089                        return error("unable to create temporary sha1 filename %s: %s\n", tmpfile, strerror(errno));
2090        }
2091
2092        /* Set it up */
2093        memset(&stream, 0, sizeof(stream));
2094        deflateInit(&stream, zlib_compression_level);
2095        size = 8 + deflateBound(&stream, len+hdrlen);
2096        compressed = xmalloc(size);
2097
2098        /* Compress it */
2099        stream.next_out = compressed;
2100        stream.avail_out = size;
2101
2102        /* First header.. */
2103        stream.next_in = (unsigned char *)hdr;
2104        stream.avail_in = hdrlen;
2105        while (deflate(&stream, 0) == Z_OK)
2106                /* nothing */;
2107
2108        /* Then the data itself.. */
2109        stream.next_in = buf;
2110        stream.avail_in = len;
2111        ret = deflate(&stream, Z_FINISH);
2112        if (ret != Z_STREAM_END)
2113                die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret);
2114
2115        ret = deflateEnd(&stream);
2116        if (ret != Z_OK)
2117                die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret);
2118
2119        size = stream.total_out;
2120
2121        if (write_buffer(fd, compressed, size) < 0)
2122                die("unable to write sha1 file");
2123        fchmod(fd, 0444);
2124        if (close(fd))
2125                die("unable to write sha1 file");
2126        free(compressed);
2127
2128        return move_temp_to_file(tmpfile, filename);
2129}
2130
2131/*
2132 * We need to unpack and recompress the object for writing
2133 * it out to a different file.
2134 */
2135static void *repack_object(const unsigned char *sha1, unsigned long *objsize)
2136{
2137        size_t size;
2138        z_stream stream;
2139        unsigned char *unpacked;
2140        unsigned long len;
2141        enum object_type type;
2142        char hdr[32];
2143        int hdrlen;
2144        void *buf;
2145
2146        /* need to unpack and recompress it by itself */
2147        unpacked = read_packed_sha1(sha1, &type, &len);
2148        if (!unpacked)
2149                error("cannot read sha1_file for %s", sha1_to_hex(sha1));
2150
2151        hdrlen = sprintf(hdr, "%s %lu", typename(type), len) + 1;
2152
2153        /* Set it up */
2154        memset(&stream, 0, sizeof(stream));
2155        deflateInit(&stream, zlib_compression_level);
2156        size = deflateBound(&stream, len + hdrlen);
2157        buf = xmalloc(size);
2158
2159        /* Compress it */
2160        stream.next_out = buf;
2161        stream.avail_out = size;
2162
2163        /* First header.. */
2164        stream.next_in = (void *)hdr;
2165        stream.avail_in = hdrlen;
2166        while (deflate(&stream, 0) == Z_OK)
2167                /* nothing */;
2168
2169        /* Then the data itself.. */
2170        stream.next_in = unpacked;
2171        stream.avail_in = len;
2172        while (deflate(&stream, Z_FINISH) == Z_OK)
2173                /* nothing */;
2174        deflateEnd(&stream);
2175        free(unpacked);
2176
2177        *objsize = stream.total_out;
2178        return buf;
2179}
2180
2181int write_sha1_to_fd(int fd, const unsigned char *sha1)
2182{
2183        int retval;
2184        unsigned long objsize;
2185        void *buf = map_sha1_file(sha1, &objsize);
2186
2187        if (buf) {
2188                retval = write_buffer(fd, buf, objsize);
2189                munmap(buf, objsize);
2190                return retval;
2191        }
2192
2193        buf = repack_object(sha1, &objsize);
2194        retval = write_buffer(fd, buf, objsize);
2195        free(buf);
2196        return retval;
2197}
2198
2199int write_sha1_from_fd(const unsigned char *sha1, int fd, char *buffer,
2200                       size_t bufsize, size_t *bufposn)
2201{
2202        char tmpfile[PATH_MAX];
2203        int local;
2204        z_stream stream;
2205        unsigned char real_sha1[20];
2206        unsigned char discard[4096];
2207        int ret;
2208        SHA_CTX c;
2209
2210        snprintf(tmpfile, sizeof(tmpfile), "%s/tmp_obj_XXXXXX", get_object_directory());
2211
2212        local = mkstemp(tmpfile);
2213        if (local < 0) {
2214                if (errno == EPERM)
2215                        return error("insufficient permission for adding an object to repository database %s\n", get_object_directory());
2216                else
2217                        return error("unable to create temporary sha1 filename %s: %s\n", tmpfile, strerror(errno));
2218        }
2219
2220        memset(&stream, 0, sizeof(stream));
2221
2222        inflateInit(&stream);
2223
2224        SHA1_Init(&c);
2225
2226        do {
2227                ssize_t size;
2228                if (*bufposn) {
2229                        stream.avail_in = *bufposn;
2230                        stream.next_in = (unsigned char *) buffer;
2231                        do {
2232                                stream.next_out = discard;
2233                                stream.avail_out = sizeof(discard);
2234                                ret = inflate(&stream, Z_SYNC_FLUSH);
2235                                SHA1_Update(&c, discard, sizeof(discard) -
2236                                            stream.avail_out);
2237                        } while (stream.avail_in && ret == Z_OK);
2238                        if (write_buffer(local, buffer, *bufposn - stream.avail_in) < 0)
2239                                die("unable to write sha1 file");
2240                        memmove(buffer, buffer + *bufposn - stream.avail_in,
2241                                stream.avail_in);
2242                        *bufposn = stream.avail_in;
2243                        if (ret != Z_OK)
2244                                break;
2245                }
2246                size = xread(fd, buffer + *bufposn, bufsize - *bufposn);
2247                if (size <= 0) {
2248                        close(local);
2249                        unlink(tmpfile);
2250                        if (!size)
2251                                return error("Connection closed?");
2252                        perror("Reading from connection");
2253                        return -1;
2254                }
2255                *bufposn += size;
2256        } while (1);
2257        inflateEnd(&stream);
2258
2259        fchmod(local, 0444);
2260        if (close(local) != 0)
2261                die("unable to write sha1 file");
2262        SHA1_Final(real_sha1, &c);
2263        if (ret != Z_STREAM_END) {
2264                unlink(tmpfile);
2265                return error("File %s corrupted", sha1_to_hex(sha1));
2266        }
2267        if (hashcmp(sha1, real_sha1)) {
2268                unlink(tmpfile);
2269                return error("File %s has bad hash", sha1_to_hex(sha1));
2270        }
2271
2272        return move_temp_to_file(tmpfile, sha1_file_name(sha1));
2273}
2274
2275int has_pack_index(const unsigned char *sha1)
2276{
2277        struct stat st;
2278        if (stat(sha1_pack_index_name(sha1), &st))
2279                return 0;
2280        return 1;
2281}
2282
2283int has_pack_file(const unsigned char *sha1)
2284{
2285        struct stat st;
2286        if (stat(sha1_pack_name(sha1), &st))
2287                return 0;
2288        return 1;
2289}
2290
2291int has_sha1_pack(const unsigned char *sha1, const char **ignore_packed)
2292{
2293        struct pack_entry e;
2294        return find_pack_entry(sha1, &e, ignore_packed);
2295}
2296
2297int has_sha1_file(const unsigned char *sha1)
2298{
2299        struct stat st;
2300        struct pack_entry e;
2301
2302        if (find_pack_entry(sha1, &e, NULL))
2303                return 1;
2304        return find_sha1_file(sha1, &st) ? 1 : 0;
2305}
2306
2307/*
2308 * reads from fd as long as possible into a supplied buffer of size bytes.
2309 * If necessary the buffer's size is increased using realloc()
2310 *
2311 * returns 0 if anything went fine and -1 otherwise
2312 *
2313 * The buffer is always NUL-terminated, not including it in returned size.
2314 *
2315 * NOTE: both buf and size may change, but even when -1 is returned
2316 * you still have to free() it yourself.
2317 */
2318int read_fd(int fd, char **return_buf, unsigned long *return_size)
2319{
2320        char *buf = *return_buf;
2321        unsigned long size = *return_size;
2322        ssize_t iret;
2323        unsigned long off = 0;
2324
2325        if (!buf || size <= 1) {
2326                size = 1024;
2327                buf = xrealloc(buf, size);
2328        }
2329
2330        do {
2331                iret = xread(fd, buf + off, (size - 1) - off);
2332                if (iret > 0) {
2333                        off += iret;
2334                        if (off == size - 1) {
2335                                size = alloc_nr(size);
2336                                buf = xrealloc(buf, size);
2337                        }
2338                }
2339        } while (iret > 0);
2340
2341        buf[off] = '\0';
2342
2343        *return_buf = buf;
2344        *return_size = off;
2345
2346        if (iret < 0)
2347                return -1;
2348        return 0;
2349}
2350
2351int index_pipe(unsigned char *sha1, int fd, const char *type, int write_object)
2352{
2353        unsigned long size = 4096;
2354        char *buf = xmalloc(size);
2355        int ret;
2356
2357        if (read_fd(fd, &buf, &size)) {
2358                free(buf);
2359                return -1;
2360        }
2361
2362        if (!type)
2363                type = blob_type;
2364        if (write_object)
2365                ret = write_sha1_file(buf, size, type, sha1);
2366        else
2367                ret = hash_sha1_file(buf, size, type, sha1);
2368        free(buf);
2369        return ret;
2370}
2371
2372int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object,
2373             enum object_type type, const char *path)
2374{
2375        size_t size = xsize_t(st->st_size);
2376        void *buf = NULL;
2377        int ret, re_allocated = 0;
2378
2379        if (size)
2380                buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
2381        close(fd);
2382
2383        if (!type)
2384                type = OBJ_BLOB;
2385
2386        /*
2387         * Convert blobs to git internal format
2388         */
2389        if ((type == OBJ_BLOB) && S_ISREG(st->st_mode)) {
2390                unsigned long nsize = size;
2391                char *nbuf = convert_to_git(path, buf, &nsize);
2392                if (nbuf) {
2393                        munmap(buf, size);
2394                        size = nsize;
2395                        buf = nbuf;
2396                        re_allocated = 1;
2397                }
2398        }
2399
2400        if (write_object)
2401                ret = write_sha1_file(buf, size, typename(type), sha1);
2402        else
2403                ret = hash_sha1_file(buf, size, typename(type), sha1);
2404        if (re_allocated) {
2405                free(buf);
2406                return ret;
2407        }
2408        if (size)
2409                munmap(buf, size);
2410        return ret;
2411}
2412
2413int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object)
2414{
2415        int fd;
2416        char *target;
2417        size_t len;
2418
2419        switch (st->st_mode & S_IFMT) {
2420        case S_IFREG:
2421                fd = open(path, O_RDONLY);
2422                if (fd < 0)
2423                        return error("open(\"%s\"): %s", path,
2424                                     strerror(errno));
2425                if (index_fd(sha1, fd, st, write_object, OBJ_BLOB, path) < 0)
2426                        return error("%s: failed to insert into database",
2427                                     path);
2428                break;
2429        case S_IFLNK:
2430                len = xsize_t(st->st_size);
2431                target = xmalloc(len + 1);
2432                if (readlink(path, target, len + 1) != st->st_size) {
2433                        char *errstr = strerror(errno);
2434                        free(target);
2435                        return error("readlink(\"%s\"): %s", path,
2436                                     errstr);
2437                }
2438                if (!write_object)
2439                        hash_sha1_file(target, len, blob_type, sha1);
2440                else if (write_sha1_file(target, len, blob_type, sha1))
2441                        return error("%s: failed to insert into database",
2442                                     path);
2443                free(target);
2444                break;
2445        case S_IFDIR:
2446                return resolve_gitlink_ref(path, "HEAD", sha1);
2447        default:
2448                return error("%s: unsupported file type", path);
2449        }
2450        return 0;
2451}
2452
2453int read_pack_header(int fd, struct pack_header *header)
2454{
2455        char *c = (char*)header;
2456        ssize_t remaining = sizeof(struct pack_header);
2457        do {
2458                ssize_t r = xread(fd, c, remaining);
2459                if (r <= 0)
2460                        /* "eof before pack header was fully read" */
2461                        return PH_ERROR_EOF;
2462                remaining -= r;
2463                c += r;
2464        } while (remaining > 0);
2465        if (header->hdr_signature != htonl(PACK_SIGNATURE))
2466                /* "protocol error (pack signature mismatch detected)" */
2467                return PH_ERROR_PACK_SIGNATURE;
2468        if (!pack_version_ok(header->hdr_version))
2469                /* "protocol error (pack version unsupported)" */
2470                return PH_ERROR_PROTOCOL;
2471        return 0;
2472}