sha1_file.con commit update-ref --stdin: make error messages more consistent (9255f05)
   1/*
   2 * GIT - The information manager from hell
   3 *
   4 * Copyright (C) Linus Torvalds, 2005
   5 *
   6 * This handles basic git sha1 object files - packing, unpacking,
   7 * creation etc.
   8 */
   9#include "cache.h"
  10#include "string-list.h"
  11#include "delta.h"
  12#include "pack.h"
  13#include "blob.h"
  14#include "commit.h"
  15#include "run-command.h"
  16#include "tag.h"
  17#include "tree.h"
  18#include "tree-walk.h"
  19#include "refs.h"
  20#include "pack-revindex.h"
  21#include "sha1-lookup.h"
  22#include "bulk-checkin.h"
  23#include "streaming.h"
  24#include "dir.h"
  25
  26#ifndef O_NOATIME
  27#if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
  28#define O_NOATIME 01000000
  29#else
  30#define O_NOATIME 0
  31#endif
  32#endif
  33
  34#define SZ_FMT PRIuMAX
  35static inline uintmax_t sz_fmt(size_t s) { return s; }
  36
  37const unsigned char null_sha1[20];
  38
  39static const char *no_log_pack_access = "no_log_pack_access";
  40static const char *log_pack_access;
  41
  42/*
  43 * This is meant to hold a *small* number of objects that you would
  44 * want read_sha1_file() to be able to return, but yet you do not want
  45 * to write them into the object store (e.g. a browse-only
  46 * application).
  47 */
  48static struct cached_object {
  49        unsigned char sha1[20];
  50        enum object_type type;
  51        void *buf;
  52        unsigned long size;
  53} *cached_objects;
  54static int cached_object_nr, cached_object_alloc;
  55
  56static struct cached_object empty_tree = {
  57        EMPTY_TREE_SHA1_BIN_LITERAL,
  58        OBJ_TREE,
  59        "",
  60        0
  61};
  62
  63/*
  64 * A pointer to the last packed_git in which an object was found.
  65 * When an object is sought, we look in this packfile first, because
  66 * objects that are looked up at similar times are often in the same
  67 * packfile as one another.
  68 */
  69static struct packed_git *last_found_pack;
  70
  71static struct cached_object *find_cached_object(const unsigned char *sha1)
  72{
  73        int i;
  74        struct cached_object *co = cached_objects;
  75
  76        for (i = 0; i < cached_object_nr; i++, co++) {
  77                if (!hashcmp(co->sha1, sha1))
  78                        return co;
  79        }
  80        if (!hashcmp(sha1, empty_tree.sha1))
  81                return &empty_tree;
  82        return NULL;
  83}
  84
  85int mkdir_in_gitdir(const char *path)
  86{
  87        if (mkdir(path, 0777)) {
  88                int saved_errno = errno;
  89                struct stat st;
  90                struct strbuf sb = STRBUF_INIT;
  91
  92                if (errno != EEXIST)
  93                        return -1;
  94                /*
  95                 * Are we looking at a path in a symlinked worktree
  96                 * whose original repository does not yet have it?
  97                 * e.g. .git/rr-cache pointing at its original
  98                 * repository in which the user hasn't performed any
  99                 * conflict resolution yet?
 100                 */
 101                if (lstat(path, &st) || !S_ISLNK(st.st_mode) ||
 102                    strbuf_readlink(&sb, path, st.st_size) ||
 103                    !is_absolute_path(sb.buf) ||
 104                    mkdir(sb.buf, 0777)) {
 105                        strbuf_release(&sb);
 106                        errno = saved_errno;
 107                        return -1;
 108                }
 109                strbuf_release(&sb);
 110        }
 111        return adjust_shared_perm(path);
 112}
 113
 114enum scld_error safe_create_leading_directories(char *path)
 115{
 116        char *next_component = path + offset_1st_component(path);
 117        enum scld_error ret = SCLD_OK;
 118
 119        while (ret == SCLD_OK && next_component) {
 120                struct stat st;
 121                char *slash = next_component, slash_character;
 122
 123                while (*slash && !is_dir_sep(*slash))
 124                        slash++;
 125
 126                if (!*slash)
 127                        break;
 128
 129                next_component = slash + 1;
 130                while (is_dir_sep(*next_component))
 131                        next_component++;
 132                if (!*next_component)
 133                        break;
 134
 135                slash_character = *slash;
 136                *slash = '\0';
 137                if (!stat(path, &st)) {
 138                        /* path exists */
 139                        if (!S_ISDIR(st.st_mode))
 140                                ret = SCLD_EXISTS;
 141                } else if (mkdir(path, 0777)) {
 142                        if (errno == EEXIST &&
 143                            !stat(path, &st) && S_ISDIR(st.st_mode))
 144                                ; /* somebody created it since we checked */
 145                        else if (errno == ENOENT)
 146                                /*
 147                                 * Either mkdir() failed because
 148                                 * somebody just pruned the containing
 149                                 * directory, or stat() failed because
 150                                 * the file that was in our way was
 151                                 * just removed.  Either way, inform
 152                                 * the caller that it might be worth
 153                                 * trying again:
 154                                 */
 155                                ret = SCLD_VANISHED;
 156                        else
 157                                ret = SCLD_FAILED;
 158                } else if (adjust_shared_perm(path)) {
 159                        ret = SCLD_PERMS;
 160                }
 161                *slash = slash_character;
 162        }
 163        return ret;
 164}
 165
 166enum scld_error safe_create_leading_directories_const(const char *path)
 167{
 168        /* path points to cache entries, so xstrdup before messing with it */
 169        char *buf = xstrdup(path);
 170        enum scld_error result = safe_create_leading_directories(buf);
 171        free(buf);
 172        return result;
 173}
 174
 175static void fill_sha1_path(char *pathbuf, const unsigned char *sha1)
 176{
 177        int i;
 178        for (i = 0; i < 20; i++) {
 179                static char hex[] = "0123456789abcdef";
 180                unsigned int val = sha1[i];
 181                char *pos = pathbuf + i*2 + (i > 0);
 182                *pos++ = hex[val >> 4];
 183                *pos = hex[val & 0xf];
 184        }
 185}
 186
 187const char *sha1_file_name(const unsigned char *sha1)
 188{
 189        static char buf[PATH_MAX];
 190        const char *objdir;
 191        int len;
 192
 193        objdir = get_object_directory();
 194        len = strlen(objdir);
 195
 196        /* '/' + sha1(2) + '/' + sha1(38) + '\0' */
 197        if (len + 43 > PATH_MAX)
 198                die("insanely long object directory %s", objdir);
 199        memcpy(buf, objdir, len);
 200        buf[len] = '/';
 201        buf[len+3] = '/';
 202        buf[len+42] = '\0';
 203        fill_sha1_path(buf + len + 1, sha1);
 204        return buf;
 205}
 206
 207/*
 208 * Return the name of the pack or index file with the specified sha1
 209 * in its filename.  *base and *name are scratch space that must be
 210 * provided by the caller.  which should be "pack" or "idx".
 211 */
 212static char *sha1_get_pack_name(const unsigned char *sha1,
 213                                char **name, char **base, const char *which)
 214{
 215        static const char hex[] = "0123456789abcdef";
 216        char *buf;
 217        int i;
 218
 219        if (!*base) {
 220                const char *sha1_file_directory = get_object_directory();
 221                int len = strlen(sha1_file_directory);
 222                *base = xmalloc(len + 60);
 223                sprintf(*base, "%s/pack/pack-1234567890123456789012345678901234567890.%s",
 224                        sha1_file_directory, which);
 225                *name = *base + len + 11;
 226        }
 227
 228        buf = *name;
 229
 230        for (i = 0; i < 20; i++) {
 231                unsigned int val = *sha1++;
 232                *buf++ = hex[val >> 4];
 233                *buf++ = hex[val & 0xf];
 234        }
 235
 236        return *base;
 237}
 238
 239char *sha1_pack_name(const unsigned char *sha1)
 240{
 241        static char *name, *base;
 242
 243        return sha1_get_pack_name(sha1, &name, &base, "pack");
 244}
 245
 246char *sha1_pack_index_name(const unsigned char *sha1)
 247{
 248        static char *name, *base;
 249
 250        return sha1_get_pack_name(sha1, &name, &base, "idx");
 251}
 252
 253struct alternate_object_database *alt_odb_list;
 254static struct alternate_object_database **alt_odb_tail;
 255
 256/*
 257 * Prepare alternate object database registry.
 258 *
 259 * The variable alt_odb_list points at the list of struct
 260 * alternate_object_database.  The elements on this list come from
 261 * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
 262 * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
 263 * whose contents is similar to that environment variable but can be
 264 * LF separated.  Its base points at a statically allocated buffer that
 265 * contains "/the/directory/corresponding/to/.git/objects/...", while
 266 * its name points just after the slash at the end of ".git/objects/"
 267 * in the example above, and has enough space to hold 40-byte hex
 268 * SHA1, an extra slash for the first level indirection, and the
 269 * terminating NUL.
 270 */
 271static int link_alt_odb_entry(const char *entry, const char *relative_base, int depth)
 272{
 273        const char *objdir = get_object_directory();
 274        struct alternate_object_database *ent;
 275        struct alternate_object_database *alt;
 276        int pfxlen, entlen;
 277        struct strbuf pathbuf = STRBUF_INIT;
 278
 279        if (!is_absolute_path(entry) && relative_base) {
 280                strbuf_addstr(&pathbuf, real_path(relative_base));
 281                strbuf_addch(&pathbuf, '/');
 282        }
 283        strbuf_addstr(&pathbuf, entry);
 284
 285        normalize_path_copy(pathbuf.buf, pathbuf.buf);
 286
 287        pfxlen = strlen(pathbuf.buf);
 288
 289        /*
 290         * The trailing slash after the directory name is given by
 291         * this function at the end. Remove duplicates.
 292         */
 293        while (pfxlen && pathbuf.buf[pfxlen-1] == '/')
 294                pfxlen -= 1;
 295
 296        entlen = pfxlen + 43; /* '/' + 2 hex + '/' + 38 hex + NUL */
 297        ent = xmalloc(sizeof(*ent) + entlen);
 298        memcpy(ent->base, pathbuf.buf, pfxlen);
 299        strbuf_release(&pathbuf);
 300
 301        ent->name = ent->base + pfxlen + 1;
 302        ent->base[pfxlen + 3] = '/';
 303        ent->base[pfxlen] = ent->base[entlen-1] = 0;
 304
 305        /* Detect cases where alternate disappeared */
 306        if (!is_directory(ent->base)) {
 307                error("object directory %s does not exist; "
 308                      "check .git/objects/info/alternates.",
 309                      ent->base);
 310                free(ent);
 311                return -1;
 312        }
 313
 314        /* Prevent the common mistake of listing the same
 315         * thing twice, or object directory itself.
 316         */
 317        for (alt = alt_odb_list; alt; alt = alt->next) {
 318                if (!memcmp(ent->base, alt->base, pfxlen)) {
 319                        free(ent);
 320                        return -1;
 321                }
 322        }
 323        if (!strcmp(ent->base, objdir)) {
 324                free(ent);
 325                return -1;
 326        }
 327
 328        /* add the alternate entry */
 329        *alt_odb_tail = ent;
 330        alt_odb_tail = &(ent->next);
 331        ent->next = NULL;
 332
 333        /* recursively add alternates */
 334        read_info_alternates(ent->base, depth + 1);
 335
 336        ent->base[pfxlen] = '/';
 337
 338        return 0;
 339}
 340
 341static void link_alt_odb_entries(const char *alt, int len, int sep,
 342                                 const char *relative_base, int depth)
 343{
 344        struct string_list entries = STRING_LIST_INIT_NODUP;
 345        char *alt_copy;
 346        int i;
 347
 348        if (depth > 5) {
 349                error("%s: ignoring alternate object stores, nesting too deep.",
 350                                relative_base);
 351                return;
 352        }
 353
 354        alt_copy = xmemdupz(alt, len);
 355        string_list_split_in_place(&entries, alt_copy, sep, -1);
 356        for (i = 0; i < entries.nr; i++) {
 357                const char *entry = entries.items[i].string;
 358                if (entry[0] == '\0' || entry[0] == '#')
 359                        continue;
 360                if (!is_absolute_path(entry) && depth) {
 361                        error("%s: ignoring relative alternate object store %s",
 362                                        relative_base, entry);
 363                } else {
 364                        link_alt_odb_entry(entry, relative_base, depth);
 365                }
 366        }
 367        string_list_clear(&entries, 0);
 368        free(alt_copy);
 369}
 370
 371void read_info_alternates(const char * relative_base, int depth)
 372{
 373        char *map;
 374        size_t mapsz;
 375        struct stat st;
 376        const char alt_file_name[] = "info/alternates";
 377        /* Given that relative_base is no longer than PATH_MAX,
 378           ensure that "path" has enough space to append "/", the
 379           file name, "info/alternates", and a trailing NUL.  */
 380        char path[PATH_MAX + 1 + sizeof alt_file_name];
 381        int fd;
 382
 383        sprintf(path, "%s/%s", relative_base, alt_file_name);
 384        fd = git_open_noatime(path);
 385        if (fd < 0)
 386                return;
 387        if (fstat(fd, &st) || (st.st_size == 0)) {
 388                close(fd);
 389                return;
 390        }
 391        mapsz = xsize_t(st.st_size);
 392        map = xmmap(NULL, mapsz, PROT_READ, MAP_PRIVATE, fd, 0);
 393        close(fd);
 394
 395        link_alt_odb_entries(map, mapsz, '\n', relative_base, depth);
 396
 397        munmap(map, mapsz);
 398}
 399
 400void add_to_alternates_file(const char *reference)
 401{
 402        struct lock_file *lock = xcalloc(1, sizeof(struct lock_file));
 403        int fd = hold_lock_file_for_append(lock, git_path("objects/info/alternates"), LOCK_DIE_ON_ERROR);
 404        char *alt = mkpath("%s\n", reference);
 405        write_or_die(fd, alt, strlen(alt));
 406        if (commit_lock_file(lock))
 407                die("could not close alternates file");
 408        if (alt_odb_tail)
 409                link_alt_odb_entries(alt, strlen(alt), '\n', NULL, 0);
 410}
 411
 412void foreach_alt_odb(alt_odb_fn fn, void *cb)
 413{
 414        struct alternate_object_database *ent;
 415
 416        prepare_alt_odb();
 417        for (ent = alt_odb_list; ent; ent = ent->next)
 418                if (fn(ent, cb))
 419                        return;
 420}
 421
 422void prepare_alt_odb(void)
 423{
 424        const char *alt;
 425
 426        if (alt_odb_tail)
 427                return;
 428
 429        alt = getenv(ALTERNATE_DB_ENVIRONMENT);
 430        if (!alt) alt = "";
 431
 432        alt_odb_tail = &alt_odb_list;
 433        link_alt_odb_entries(alt, strlen(alt), PATH_SEP, NULL, 0);
 434
 435        read_info_alternates(get_object_directory(), 0);
 436}
 437
 438static int has_loose_object_local(const unsigned char *sha1)
 439{
 440        return !access(sha1_file_name(sha1), F_OK);
 441}
 442
 443int has_loose_object_nonlocal(const unsigned char *sha1)
 444{
 445        struct alternate_object_database *alt;
 446        prepare_alt_odb();
 447        for (alt = alt_odb_list; alt; alt = alt->next) {
 448                fill_sha1_path(alt->name, sha1);
 449                if (!access(alt->base, F_OK))
 450                        return 1;
 451        }
 452        return 0;
 453}
 454
 455static int has_loose_object(const unsigned char *sha1)
 456{
 457        return has_loose_object_local(sha1) ||
 458               has_loose_object_nonlocal(sha1);
 459}
 460
 461static unsigned int pack_used_ctr;
 462static unsigned int pack_mmap_calls;
 463static unsigned int peak_pack_open_windows;
 464static unsigned int pack_open_windows;
 465static unsigned int pack_open_fds;
 466static unsigned int pack_max_fds;
 467static size_t peak_pack_mapped;
 468static size_t pack_mapped;
 469struct packed_git *packed_git;
 470
 471void pack_report(void)
 472{
 473        fprintf(stderr,
 474                "pack_report: getpagesize()            = %10" SZ_FMT "\n"
 475                "pack_report: core.packedGitWindowSize = %10" SZ_FMT "\n"
 476                "pack_report: core.packedGitLimit      = %10" SZ_FMT "\n",
 477                sz_fmt(getpagesize()),
 478                sz_fmt(packed_git_window_size),
 479                sz_fmt(packed_git_limit));
 480        fprintf(stderr,
 481                "pack_report: pack_used_ctr            = %10u\n"
 482                "pack_report: pack_mmap_calls          = %10u\n"
 483                "pack_report: pack_open_windows        = %10u / %10u\n"
 484                "pack_report: pack_mapped              = "
 485                        "%10" SZ_FMT " / %10" SZ_FMT "\n",
 486                pack_used_ctr,
 487                pack_mmap_calls,
 488                pack_open_windows, peak_pack_open_windows,
 489                sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped));
 490}
 491
 492/*
 493 * Open and mmap the index file at path, perform a couple of
 494 * consistency checks, then record its information to p.  Return 0 on
 495 * success.
 496 */
 497static int check_packed_git_idx(const char *path, struct packed_git *p)
 498{
 499        void *idx_map;
 500        struct pack_idx_header *hdr;
 501        size_t idx_size;
 502        uint32_t version, nr, i, *index;
 503        int fd = git_open_noatime(path);
 504        struct stat st;
 505
 506        if (fd < 0)
 507                return -1;
 508        if (fstat(fd, &st)) {
 509                close(fd);
 510                return -1;
 511        }
 512        idx_size = xsize_t(st.st_size);
 513        if (idx_size < 4 * 256 + 20 + 20) {
 514                close(fd);
 515                return error("index file %s is too small", path);
 516        }
 517        idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
 518        close(fd);
 519
 520        hdr = idx_map;
 521        if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
 522                version = ntohl(hdr->idx_version);
 523                if (version < 2 || version > 2) {
 524                        munmap(idx_map, idx_size);
 525                        return error("index file %s is version %"PRIu32
 526                                     " and is not supported by this binary"
 527                                     " (try upgrading GIT to a newer version)",
 528                                     path, version);
 529                }
 530        } else
 531                version = 1;
 532
 533        nr = 0;
 534        index = idx_map;
 535        if (version > 1)
 536                index += 2;  /* skip index header */
 537        for (i = 0; i < 256; i++) {
 538                uint32_t n = ntohl(index[i]);
 539                if (n < nr) {
 540                        munmap(idx_map, idx_size);
 541                        return error("non-monotonic index %s", path);
 542                }
 543                nr = n;
 544        }
 545
 546        if (version == 1) {
 547                /*
 548                 * Total size:
 549                 *  - 256 index entries 4 bytes each
 550                 *  - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
 551                 *  - 20-byte SHA1 of the packfile
 552                 *  - 20-byte SHA1 file checksum
 553                 */
 554                if (idx_size != 4*256 + nr * 24 + 20 + 20) {
 555                        munmap(idx_map, idx_size);
 556                        return error("wrong index v1 file size in %s", path);
 557                }
 558        } else if (version == 2) {
 559                /*
 560                 * Minimum size:
 561                 *  - 8 bytes of header
 562                 *  - 256 index entries 4 bytes each
 563                 *  - 20-byte sha1 entry * nr
 564                 *  - 4-byte crc entry * nr
 565                 *  - 4-byte offset entry * nr
 566                 *  - 20-byte SHA1 of the packfile
 567                 *  - 20-byte SHA1 file checksum
 568                 * And after the 4-byte offset table might be a
 569                 * variable sized table containing 8-byte entries
 570                 * for offsets larger than 2^31.
 571                 */
 572                unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
 573                unsigned long max_size = min_size;
 574                if (nr)
 575                        max_size += (nr - 1)*8;
 576                if (idx_size < min_size || idx_size > max_size) {
 577                        munmap(idx_map, idx_size);
 578                        return error("wrong index v2 file size in %s", path);
 579                }
 580                if (idx_size != min_size &&
 581                    /*
 582                     * make sure we can deal with large pack offsets.
 583                     * 31-bit signed offset won't be enough, neither
 584                     * 32-bit unsigned one will be.
 585                     */
 586                    (sizeof(off_t) <= 4)) {
 587                        munmap(idx_map, idx_size);
 588                        return error("pack too large for current definition of off_t in %s", path);
 589                }
 590        }
 591
 592        p->index_version = version;
 593        p->index_data = idx_map;
 594        p->index_size = idx_size;
 595        p->num_objects = nr;
 596        return 0;
 597}
 598
 599int open_pack_index(struct packed_git *p)
 600{
 601        char *idx_name;
 602        int ret;
 603
 604        if (p->index_data)
 605                return 0;
 606
 607        idx_name = xstrdup(p->pack_name);
 608        strcpy(idx_name + strlen(idx_name) - strlen(".pack"), ".idx");
 609        ret = check_packed_git_idx(idx_name, p);
 610        free(idx_name);
 611        return ret;
 612}
 613
 614static void scan_windows(struct packed_git *p,
 615        struct packed_git **lru_p,
 616        struct pack_window **lru_w,
 617        struct pack_window **lru_l)
 618{
 619        struct pack_window *w, *w_l;
 620
 621        for (w_l = NULL, w = p->windows; w; w = w->next) {
 622                if (!w->inuse_cnt) {
 623                        if (!*lru_w || w->last_used < (*lru_w)->last_used) {
 624                                *lru_p = p;
 625                                *lru_w = w;
 626                                *lru_l = w_l;
 627                        }
 628                }
 629                w_l = w;
 630        }
 631}
 632
 633static int unuse_one_window(struct packed_git *current)
 634{
 635        struct packed_git *p, *lru_p = NULL;
 636        struct pack_window *lru_w = NULL, *lru_l = NULL;
 637
 638        if (current)
 639                scan_windows(current, &lru_p, &lru_w, &lru_l);
 640        for (p = packed_git; p; p = p->next)
 641                scan_windows(p, &lru_p, &lru_w, &lru_l);
 642        if (lru_p) {
 643                munmap(lru_w->base, lru_w->len);
 644                pack_mapped -= lru_w->len;
 645                if (lru_l)
 646                        lru_l->next = lru_w->next;
 647                else
 648                        lru_p->windows = lru_w->next;
 649                free(lru_w);
 650                pack_open_windows--;
 651                return 1;
 652        }
 653        return 0;
 654}
 655
 656void release_pack_memory(size_t need)
 657{
 658        size_t cur = pack_mapped;
 659        while (need >= (cur - pack_mapped) && unuse_one_window(NULL))
 660                ; /* nothing */
 661}
 662
 663void *xmmap(void *start, size_t length,
 664        int prot, int flags, int fd, off_t offset)
 665{
 666        void *ret = mmap(start, length, prot, flags, fd, offset);
 667        if (ret == MAP_FAILED) {
 668                if (!length)
 669                        return NULL;
 670                release_pack_memory(length);
 671                ret = mmap(start, length, prot, flags, fd, offset);
 672                if (ret == MAP_FAILED)
 673                        die_errno("Out of memory? mmap failed");
 674        }
 675        return ret;
 676}
 677
 678void close_pack_windows(struct packed_git *p)
 679{
 680        while (p->windows) {
 681                struct pack_window *w = p->windows;
 682
 683                if (w->inuse_cnt)
 684                        die("pack '%s' still has open windows to it",
 685                            p->pack_name);
 686                munmap(w->base, w->len);
 687                pack_mapped -= w->len;
 688                pack_open_windows--;
 689                p->windows = w->next;
 690                free(w);
 691        }
 692}
 693
 694/*
 695 * The LRU pack is the one with the oldest MRU window, preferring packs
 696 * with no used windows, or the oldest mtime if it has no windows allocated.
 697 */
 698static void find_lru_pack(struct packed_git *p, struct packed_git **lru_p, struct pack_window **mru_w, int *accept_windows_inuse)
 699{
 700        struct pack_window *w, *this_mru_w;
 701        int has_windows_inuse = 0;
 702
 703        /*
 704         * Reject this pack if it has windows and the previously selected
 705         * one does not.  If this pack does not have windows, reject
 706         * it if the pack file is newer than the previously selected one.
 707         */
 708        if (*lru_p && !*mru_w && (p->windows || p->mtime > (*lru_p)->mtime))
 709                return;
 710
 711        for (w = this_mru_w = p->windows; w; w = w->next) {
 712                /*
 713                 * Reject this pack if any of its windows are in use,
 714                 * but the previously selected pack did not have any
 715                 * inuse windows.  Otherwise, record that this pack
 716                 * has windows in use.
 717                 */
 718                if (w->inuse_cnt) {
 719                        if (*accept_windows_inuse)
 720                                has_windows_inuse = 1;
 721                        else
 722                                return;
 723                }
 724
 725                if (w->last_used > this_mru_w->last_used)
 726                        this_mru_w = w;
 727
 728                /*
 729                 * Reject this pack if it has windows that have been
 730                 * used more recently than the previously selected pack.
 731                 * If the previously selected pack had windows inuse and
 732                 * we have not encountered a window in this pack that is
 733                 * inuse, skip this check since we prefer a pack with no
 734                 * inuse windows to one that has inuse windows.
 735                 */
 736                if (*mru_w && *accept_windows_inuse == has_windows_inuse &&
 737                    this_mru_w->last_used > (*mru_w)->last_used)
 738                        return;
 739        }
 740
 741        /*
 742         * Select this pack.
 743         */
 744        *mru_w = this_mru_w;
 745        *lru_p = p;
 746        *accept_windows_inuse = has_windows_inuse;
 747}
 748
 749static int close_one_pack(void)
 750{
 751        struct packed_git *p, *lru_p = NULL;
 752        struct pack_window *mru_w = NULL;
 753        int accept_windows_inuse = 1;
 754
 755        for (p = packed_git; p; p = p->next) {
 756                if (p->pack_fd == -1)
 757                        continue;
 758                find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse);
 759        }
 760
 761        if (lru_p) {
 762                close(lru_p->pack_fd);
 763                pack_open_fds--;
 764                lru_p->pack_fd = -1;
 765                return 1;
 766        }
 767
 768        return 0;
 769}
 770
 771void unuse_pack(struct pack_window **w_cursor)
 772{
 773        struct pack_window *w = *w_cursor;
 774        if (w) {
 775                w->inuse_cnt--;
 776                *w_cursor = NULL;
 777        }
 778}
 779
 780void close_pack_index(struct packed_git *p)
 781{
 782        if (p->index_data) {
 783                munmap((void *)p->index_data, p->index_size);
 784                p->index_data = NULL;
 785        }
 786}
 787
 788/*
 789 * This is used by git-repack in case a newly created pack happens to
 790 * contain the same set of objects as an existing one.  In that case
 791 * the resulting file might be different even if its name would be the
 792 * same.  It is best to close any reference to the old pack before it is
 793 * replaced on disk.  Of course no index pointers nor windows for given pack
 794 * must subsist at this point.  If ever objects from this pack are requested
 795 * again, the new version of the pack will be reinitialized through
 796 * reprepare_packed_git().
 797 */
 798void free_pack_by_name(const char *pack_name)
 799{
 800        struct packed_git *p, **pp = &packed_git;
 801
 802        while (*pp) {
 803                p = *pp;
 804                if (strcmp(pack_name, p->pack_name) == 0) {
 805                        clear_delta_base_cache();
 806                        close_pack_windows(p);
 807                        if (p->pack_fd != -1) {
 808                                close(p->pack_fd);
 809                                pack_open_fds--;
 810                        }
 811                        close_pack_index(p);
 812                        free(p->bad_object_sha1);
 813                        *pp = p->next;
 814                        if (last_found_pack == p)
 815                                last_found_pack = NULL;
 816                        free(p);
 817                        return;
 818                }
 819                pp = &p->next;
 820        }
 821}
 822
 823static unsigned int get_max_fd_limit(void)
 824{
 825#ifdef RLIMIT_NOFILE
 826        {
 827                struct rlimit lim;
 828
 829                if (!getrlimit(RLIMIT_NOFILE, &lim))
 830                        return lim.rlim_cur;
 831        }
 832#endif
 833
 834#ifdef _SC_OPEN_MAX
 835        {
 836                long open_max = sysconf(_SC_OPEN_MAX);
 837                if (0 < open_max)
 838                        return open_max;
 839                /*
 840                 * Otherwise, we got -1 for one of the two
 841                 * reasons:
 842                 *
 843                 * (1) sysconf() did not understand _SC_OPEN_MAX
 844                 *     and signaled an error with -1; or
 845                 * (2) sysconf() said there is no limit.
 846                 *
 847                 * We _could_ clear errno before calling sysconf() to
 848                 * tell these two cases apart and return a huge number
 849                 * in the latter case to let the caller cap it to a
 850                 * value that is not so selfish, but letting the
 851                 * fallback OPEN_MAX codepath take care of these cases
 852                 * is a lot simpler.
 853                 */
 854        }
 855#endif
 856
 857#ifdef OPEN_MAX
 858        return OPEN_MAX;
 859#else
 860        return 1; /* see the caller ;-) */
 861#endif
 862}
 863
 864/*
 865 * Do not call this directly as this leaks p->pack_fd on error return;
 866 * call open_packed_git() instead.
 867 */
 868static int open_packed_git_1(struct packed_git *p)
 869{
 870        struct stat st;
 871        struct pack_header hdr;
 872        unsigned char sha1[20];
 873        unsigned char *idx_sha1;
 874        long fd_flag;
 875
 876        if (!p->index_data && open_pack_index(p))
 877                return error("packfile %s index unavailable", p->pack_name);
 878
 879        if (!pack_max_fds) {
 880                unsigned int max_fds = get_max_fd_limit();
 881
 882                /* Save 3 for stdin/stdout/stderr, 22 for work */
 883                if (25 < max_fds)
 884                        pack_max_fds = max_fds - 25;
 885                else
 886                        pack_max_fds = 1;
 887        }
 888
 889        while (pack_max_fds <= pack_open_fds && close_one_pack())
 890                ; /* nothing */
 891
 892        p->pack_fd = git_open_noatime(p->pack_name);
 893        if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
 894                return -1;
 895        pack_open_fds++;
 896
 897        /* If we created the struct before we had the pack we lack size. */
 898        if (!p->pack_size) {
 899                if (!S_ISREG(st.st_mode))
 900                        return error("packfile %s not a regular file", p->pack_name);
 901                p->pack_size = st.st_size;
 902        } else if (p->pack_size != st.st_size)
 903                return error("packfile %s size changed", p->pack_name);
 904
 905        /* We leave these file descriptors open with sliding mmap;
 906         * there is no point keeping them open across exec(), though.
 907         */
 908        fd_flag = fcntl(p->pack_fd, F_GETFD, 0);
 909        if (fd_flag < 0)
 910                return error("cannot determine file descriptor flags");
 911        fd_flag |= FD_CLOEXEC;
 912        if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
 913                return error("cannot set FD_CLOEXEC");
 914
 915        /* Verify we recognize this pack file format. */
 916        if (read_in_full(p->pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
 917                return error("file %s is far too short to be a packfile", p->pack_name);
 918        if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
 919                return error("file %s is not a GIT packfile", p->pack_name);
 920        if (!pack_version_ok(hdr.hdr_version))
 921                return error("packfile %s is version %"PRIu32" and not"
 922                        " supported (try upgrading GIT to a newer version)",
 923                        p->pack_name, ntohl(hdr.hdr_version));
 924
 925        /* Verify the pack matches its index. */
 926        if (p->num_objects != ntohl(hdr.hdr_entries))
 927                return error("packfile %s claims to have %"PRIu32" objects"
 928                             " while index indicates %"PRIu32" objects",
 929                             p->pack_name, ntohl(hdr.hdr_entries),
 930                             p->num_objects);
 931        if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
 932                return error("end of packfile %s is unavailable", p->pack_name);
 933        if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1))
 934                return error("packfile %s signature is unavailable", p->pack_name);
 935        idx_sha1 = ((unsigned char *)p->index_data) + p->index_size - 40;
 936        if (hashcmp(sha1, idx_sha1))
 937                return error("packfile %s does not match index", p->pack_name);
 938        return 0;
 939}
 940
 941static int open_packed_git(struct packed_git *p)
 942{
 943        if (!open_packed_git_1(p))
 944                return 0;
 945        if (p->pack_fd != -1) {
 946                close(p->pack_fd);
 947                pack_open_fds--;
 948                p->pack_fd = -1;
 949        }
 950        return -1;
 951}
 952
 953static int in_window(struct pack_window *win, off_t offset)
 954{
 955        /* We must promise at least 20 bytes (one hash) after the
 956         * offset is available from this window, otherwise the offset
 957         * is not actually in this window and a different window (which
 958         * has that one hash excess) must be used.  This is to support
 959         * the object header and delta base parsing routines below.
 960         */
 961        off_t win_off = win->offset;
 962        return win_off <= offset
 963                && (offset + 20) <= (win_off + win->len);
 964}
 965
 966unsigned char *use_pack(struct packed_git *p,
 967                struct pack_window **w_cursor,
 968                off_t offset,
 969                unsigned long *left)
 970{
 971        struct pack_window *win = *w_cursor;
 972
 973        /* Since packfiles end in a hash of their content and it's
 974         * pointless to ask for an offset into the middle of that
 975         * hash, and the in_window function above wouldn't match
 976         * don't allow an offset too close to the end of the file.
 977         */
 978        if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p))
 979                die("packfile %s cannot be accessed", p->pack_name);
 980        if (offset > (p->pack_size - 20))
 981                die("offset beyond end of packfile (truncated pack?)");
 982
 983        if (!win || !in_window(win, offset)) {
 984                if (win)
 985                        win->inuse_cnt--;
 986                for (win = p->windows; win; win = win->next) {
 987                        if (in_window(win, offset))
 988                                break;
 989                }
 990                if (!win) {
 991                        size_t window_align = packed_git_window_size / 2;
 992                        off_t len;
 993
 994                        if (p->pack_fd == -1 && open_packed_git(p))
 995                                die("packfile %s cannot be accessed", p->pack_name);
 996
 997                        win = xcalloc(1, sizeof(*win));
 998                        win->offset = (offset / window_align) * window_align;
 999                        len = p->pack_size - win->offset;
1000                        if (len > packed_git_window_size)
1001                                len = packed_git_window_size;
1002                        win->len = (size_t)len;
1003                        pack_mapped += win->len;
1004                        while (packed_git_limit < pack_mapped
1005                                && unuse_one_window(p))
1006                                ; /* nothing */
1007                        win->base = xmmap(NULL, win->len,
1008                                PROT_READ, MAP_PRIVATE,
1009                                p->pack_fd, win->offset);
1010                        if (win->base == MAP_FAILED)
1011                                die("packfile %s cannot be mapped: %s",
1012                                        p->pack_name,
1013                                        strerror(errno));
1014                        if (!win->offset && win->len == p->pack_size
1015                                && !p->do_not_close) {
1016                                close(p->pack_fd);
1017                                pack_open_fds--;
1018                                p->pack_fd = -1;
1019                        }
1020                        pack_mmap_calls++;
1021                        pack_open_windows++;
1022                        if (pack_mapped > peak_pack_mapped)
1023                                peak_pack_mapped = pack_mapped;
1024                        if (pack_open_windows > peak_pack_open_windows)
1025                                peak_pack_open_windows = pack_open_windows;
1026                        win->next = p->windows;
1027                        p->windows = win;
1028                }
1029        }
1030        if (win != *w_cursor) {
1031                win->last_used = pack_used_ctr++;
1032                win->inuse_cnt++;
1033                *w_cursor = win;
1034        }
1035        offset -= win->offset;
1036        if (left)
1037                *left = win->len - xsize_t(offset);
1038        return win->base + offset;
1039}
1040
1041static struct packed_git *alloc_packed_git(int extra)
1042{
1043        struct packed_git *p = xmalloc(sizeof(*p) + extra);
1044        memset(p, 0, sizeof(*p));
1045        p->pack_fd = -1;
1046        return p;
1047}
1048
1049static void try_to_free_pack_memory(size_t size)
1050{
1051        release_pack_memory(size);
1052}
1053
1054struct packed_git *add_packed_git(const char *path, int path_len, int local)
1055{
1056        static int have_set_try_to_free_routine;
1057        struct stat st;
1058        struct packed_git *p = alloc_packed_git(path_len + 2);
1059
1060        if (!have_set_try_to_free_routine) {
1061                have_set_try_to_free_routine = 1;
1062                set_try_to_free_routine(try_to_free_pack_memory);
1063        }
1064
1065        /*
1066         * Make sure a corresponding .pack file exists and that
1067         * the index looks sane.
1068         */
1069        path_len -= strlen(".idx");
1070        if (path_len < 1) {
1071                free(p);
1072                return NULL;
1073        }
1074        memcpy(p->pack_name, path, path_len);
1075
1076        strcpy(p->pack_name + path_len, ".keep");
1077        if (!access(p->pack_name, F_OK))
1078                p->pack_keep = 1;
1079
1080        strcpy(p->pack_name + path_len, ".pack");
1081        if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
1082                free(p);
1083                return NULL;
1084        }
1085
1086        /* ok, it looks sane as far as we can check without
1087         * actually mapping the pack file.
1088         */
1089        p->pack_size = st.st_size;
1090        p->pack_local = local;
1091        p->mtime = st.st_mtime;
1092        if (path_len < 40 || get_sha1_hex(path + path_len - 40, p->sha1))
1093                hashclr(p->sha1);
1094        return p;
1095}
1096
1097struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
1098{
1099        const char *path = sha1_pack_name(sha1);
1100        struct packed_git *p = alloc_packed_git(strlen(path) + 1);
1101
1102        strcpy(p->pack_name, path);
1103        hashcpy(p->sha1, sha1);
1104        if (check_packed_git_idx(idx_path, p)) {
1105                free(p);
1106                return NULL;
1107        }
1108
1109        return p;
1110}
1111
1112void install_packed_git(struct packed_git *pack)
1113{
1114        if (pack->pack_fd != -1)
1115                pack_open_fds++;
1116
1117        pack->next = packed_git;
1118        packed_git = pack;
1119}
1120
1121void (*report_garbage)(const char *desc, const char *path);
1122
1123static void report_helper(const struct string_list *list,
1124                          int seen_bits, int first, int last)
1125{
1126        const char *msg;
1127        switch (seen_bits) {
1128        case 0:
1129                msg = "no corresponding .idx nor .pack";
1130                break;
1131        case 1:
1132                msg = "no corresponding .idx";
1133                break;
1134        case 2:
1135                msg = "no corresponding .pack";
1136                break;
1137        default:
1138                return;
1139        }
1140        for (; first < last; first++)
1141                report_garbage(msg, list->items[first].string);
1142}
1143
1144static void report_pack_garbage(struct string_list *list)
1145{
1146        int i, baselen = -1, first = 0, seen_bits = 0;
1147
1148        if (!report_garbage)
1149                return;
1150
1151        sort_string_list(list);
1152
1153        for (i = 0; i < list->nr; i++) {
1154                const char *path = list->items[i].string;
1155                if (baselen != -1 &&
1156                    strncmp(path, list->items[first].string, baselen)) {
1157                        report_helper(list, seen_bits, first, i);
1158                        baselen = -1;
1159                        seen_bits = 0;
1160                }
1161                if (baselen == -1) {
1162                        const char *dot = strrchr(path, '.');
1163                        if (!dot) {
1164                                report_garbage("garbage found", path);
1165                                continue;
1166                        }
1167                        baselen = dot - path + 1;
1168                        first = i;
1169                }
1170                if (!strcmp(path + baselen, "pack"))
1171                        seen_bits |= 1;
1172                else if (!strcmp(path + baselen, "idx"))
1173                        seen_bits |= 2;
1174        }
1175        report_helper(list, seen_bits, first, list->nr);
1176}
1177
1178static void prepare_packed_git_one(char *objdir, int local)
1179{
1180        /* Ensure that this buffer is large enough so that we can
1181           append "/pack/" without clobbering the stack even if
1182           strlen(objdir) were PATH_MAX.  */
1183        char path[PATH_MAX + 1 + 4 + 1 + 1];
1184        int len;
1185        DIR *dir;
1186        struct dirent *de;
1187        struct string_list garbage = STRING_LIST_INIT_DUP;
1188
1189        sprintf(path, "%s/pack", objdir);
1190        len = strlen(path);
1191        dir = opendir(path);
1192        if (!dir) {
1193                if (errno != ENOENT)
1194                        error("unable to open object pack directory: %s: %s",
1195                              path, strerror(errno));
1196                return;
1197        }
1198        path[len++] = '/';
1199        while ((de = readdir(dir)) != NULL) {
1200                int namelen = strlen(de->d_name);
1201                struct packed_git *p;
1202
1203                if (len + namelen + 1 > sizeof(path)) {
1204                        if (report_garbage) {
1205                                struct strbuf sb = STRBUF_INIT;
1206                                strbuf_addf(&sb, "%.*s/%s", len - 1, path, de->d_name);
1207                                report_garbage("path too long", sb.buf);
1208                                strbuf_release(&sb);
1209                        }
1210                        continue;
1211                }
1212
1213                if (is_dot_or_dotdot(de->d_name))
1214                        continue;
1215
1216                strcpy(path + len, de->d_name);
1217
1218                if (has_extension(de->d_name, ".idx")) {
1219                        /* Don't reopen a pack we already have. */
1220                        for (p = packed_git; p; p = p->next) {
1221                                if (!memcmp(path, p->pack_name, len + namelen - 4))
1222                                        break;
1223                        }
1224                        if (p == NULL &&
1225                            /*
1226                             * See if it really is a valid .idx file with
1227                             * corresponding .pack file that we can map.
1228                             */
1229                            (p = add_packed_git(path, len + namelen, local)) != NULL)
1230                                install_packed_git(p);
1231                }
1232
1233                if (!report_garbage)
1234                        continue;
1235
1236                if (has_extension(de->d_name, ".idx") ||
1237                    has_extension(de->d_name, ".pack") ||
1238                    has_extension(de->d_name, ".bitmap") ||
1239                    has_extension(de->d_name, ".keep"))
1240                        string_list_append(&garbage, path);
1241                else
1242                        report_garbage("garbage found", path);
1243        }
1244        closedir(dir);
1245        report_pack_garbage(&garbage);
1246        string_list_clear(&garbage, 0);
1247}
1248
1249static int sort_pack(const void *a_, const void *b_)
1250{
1251        struct packed_git *a = *((struct packed_git **)a_);
1252        struct packed_git *b = *((struct packed_git **)b_);
1253        int st;
1254
1255        /*
1256         * Local packs tend to contain objects specific to our
1257         * variant of the project than remote ones.  In addition,
1258         * remote ones could be on a network mounted filesystem.
1259         * Favor local ones for these reasons.
1260         */
1261        st = a->pack_local - b->pack_local;
1262        if (st)
1263                return -st;
1264
1265        /*
1266         * Younger packs tend to contain more recent objects,
1267         * and more recent objects tend to get accessed more
1268         * often.
1269         */
1270        if (a->mtime < b->mtime)
1271                return 1;
1272        else if (a->mtime == b->mtime)
1273                return 0;
1274        return -1;
1275}
1276
1277static void rearrange_packed_git(void)
1278{
1279        struct packed_git **ary, *p;
1280        int i, n;
1281
1282        for (n = 0, p = packed_git; p; p = p->next)
1283                n++;
1284        if (n < 2)
1285                return;
1286
1287        /* prepare an array of packed_git for easier sorting */
1288        ary = xcalloc(n, sizeof(struct packed_git *));
1289        for (n = 0, p = packed_git; p; p = p->next)
1290                ary[n++] = p;
1291
1292        qsort(ary, n, sizeof(struct packed_git *), sort_pack);
1293
1294        /* link them back again */
1295        for (i = 0; i < n - 1; i++)
1296                ary[i]->next = ary[i + 1];
1297        ary[n - 1]->next = NULL;
1298        packed_git = ary[0];
1299
1300        free(ary);
1301}
1302
1303static int prepare_packed_git_run_once = 0;
1304void prepare_packed_git(void)
1305{
1306        struct alternate_object_database *alt;
1307
1308        if (prepare_packed_git_run_once)
1309                return;
1310        prepare_packed_git_one(get_object_directory(), 1);
1311        prepare_alt_odb();
1312        for (alt = alt_odb_list; alt; alt = alt->next) {
1313                alt->name[-1] = 0;
1314                prepare_packed_git_one(alt->base, 0);
1315                alt->name[-1] = '/';
1316        }
1317        rearrange_packed_git();
1318        prepare_packed_git_run_once = 1;
1319}
1320
1321void reprepare_packed_git(void)
1322{
1323        prepare_packed_git_run_once = 0;
1324        prepare_packed_git();
1325}
1326
1327static void mark_bad_packed_object(struct packed_git *p,
1328                                   const unsigned char *sha1)
1329{
1330        unsigned i;
1331        for (i = 0; i < p->num_bad_objects; i++)
1332                if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
1333                        return;
1334        p->bad_object_sha1 = xrealloc(p->bad_object_sha1, 20 * (p->num_bad_objects + 1));
1335        hashcpy(p->bad_object_sha1 + 20 * p->num_bad_objects, sha1);
1336        p->num_bad_objects++;
1337}
1338
1339static const struct packed_git *has_packed_and_bad(const unsigned char *sha1)
1340{
1341        struct packed_git *p;
1342        unsigned i;
1343
1344        for (p = packed_git; p; p = p->next)
1345                for (i = 0; i < p->num_bad_objects; i++)
1346                        if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
1347                                return p;
1348        return NULL;
1349}
1350
1351/*
1352 * With an in-core object data in "map", rehash it to make sure the
1353 * object name actually matches "sha1" to detect object corruption.
1354 * With "map" == NULL, try reading the object named with "sha1" using
1355 * the streaming interface and rehash it to do the same.
1356 */
1357int check_sha1_signature(const unsigned char *sha1, void *map,
1358                         unsigned long size, const char *type)
1359{
1360        unsigned char real_sha1[20];
1361        enum object_type obj_type;
1362        struct git_istream *st;
1363        git_SHA_CTX c;
1364        char hdr[32];
1365        int hdrlen;
1366
1367        if (map) {
1368                hash_sha1_file(map, size, type, real_sha1);
1369                return hashcmp(sha1, real_sha1) ? -1 : 0;
1370        }
1371
1372        st = open_istream(sha1, &obj_type, &size, NULL);
1373        if (!st)
1374                return -1;
1375
1376        /* Generate the header */
1377        hdrlen = sprintf(hdr, "%s %lu", typename(obj_type), size) + 1;
1378
1379        /* Sha1.. */
1380        git_SHA1_Init(&c);
1381        git_SHA1_Update(&c, hdr, hdrlen);
1382        for (;;) {
1383                char buf[1024 * 16];
1384                ssize_t readlen = read_istream(st, buf, sizeof(buf));
1385
1386                if (readlen < 0) {
1387                        close_istream(st);
1388                        return -1;
1389                }
1390                if (!readlen)
1391                        break;
1392                git_SHA1_Update(&c, buf, readlen);
1393        }
1394        git_SHA1_Final(real_sha1, &c);
1395        close_istream(st);
1396        return hashcmp(sha1, real_sha1) ? -1 : 0;
1397}
1398
1399int git_open_noatime(const char *name)
1400{
1401        static int sha1_file_open_flag = O_NOATIME;
1402
1403        for (;;) {
1404                int fd = open(name, O_RDONLY | sha1_file_open_flag);
1405                if (fd >= 0)
1406                        return fd;
1407
1408                /* Might the failure be due to O_NOATIME? */
1409                if (errno != ENOENT && sha1_file_open_flag) {
1410                        sha1_file_open_flag = 0;
1411                        continue;
1412                }
1413
1414                return -1;
1415        }
1416}
1417
1418static int stat_sha1_file(const unsigned char *sha1, struct stat *st)
1419{
1420        struct alternate_object_database *alt;
1421
1422        if (!lstat(sha1_file_name(sha1), st))
1423                return 0;
1424
1425        prepare_alt_odb();
1426        errno = ENOENT;
1427        for (alt = alt_odb_list; alt; alt = alt->next) {
1428                fill_sha1_path(alt->name, sha1);
1429                if (!lstat(alt->base, st))
1430                        return 0;
1431        }
1432
1433        return -1;
1434}
1435
1436static int open_sha1_file(const unsigned char *sha1)
1437{
1438        int fd;
1439        struct alternate_object_database *alt;
1440
1441        fd = git_open_noatime(sha1_file_name(sha1));
1442        if (fd >= 0)
1443                return fd;
1444
1445        prepare_alt_odb();
1446        errno = ENOENT;
1447        for (alt = alt_odb_list; alt; alt = alt->next) {
1448                fill_sha1_path(alt->name, sha1);
1449                fd = git_open_noatime(alt->base);
1450                if (fd >= 0)
1451                        return fd;
1452        }
1453        return -1;
1454}
1455
1456void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
1457{
1458        void *map;
1459        int fd;
1460
1461        fd = open_sha1_file(sha1);
1462        map = NULL;
1463        if (fd >= 0) {
1464                struct stat st;
1465
1466                if (!fstat(fd, &st)) {
1467                        *size = xsize_t(st.st_size);
1468                        if (!*size) {
1469                                /* mmap() is forbidden on empty files */
1470                                error("object file %s is empty", sha1_file_name(sha1));
1471                                return NULL;
1472                        }
1473                        map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
1474                }
1475                close(fd);
1476        }
1477        return map;
1478}
1479
1480unsigned long unpack_object_header_buffer(const unsigned char *buf,
1481                unsigned long len, enum object_type *type, unsigned long *sizep)
1482{
1483        unsigned shift;
1484        unsigned long size, c;
1485        unsigned long used = 0;
1486
1487        c = buf[used++];
1488        *type = (c >> 4) & 7;
1489        size = c & 15;
1490        shift = 4;
1491        while (c & 0x80) {
1492                if (len <= used || bitsizeof(long) <= shift) {
1493                        error("bad object header");
1494                        size = used = 0;
1495                        break;
1496                }
1497                c = buf[used++];
1498                size += (c & 0x7f) << shift;
1499                shift += 7;
1500        }
1501        *sizep = size;
1502        return used;
1503}
1504
1505int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
1506{
1507        /* Get the data stream */
1508        memset(stream, 0, sizeof(*stream));
1509        stream->next_in = map;
1510        stream->avail_in = mapsize;
1511        stream->next_out = buffer;
1512        stream->avail_out = bufsiz;
1513
1514        git_inflate_init(stream);
1515        return git_inflate(stream, 0);
1516}
1517
1518static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
1519{
1520        int bytes = strlen(buffer) + 1;
1521        unsigned char *buf = xmallocz(size);
1522        unsigned long n;
1523        int status = Z_OK;
1524
1525        n = stream->total_out - bytes;
1526        if (n > size)
1527                n = size;
1528        memcpy(buf, (char *) buffer + bytes, n);
1529        bytes = n;
1530        if (bytes <= size) {
1531                /*
1532                 * The above condition must be (bytes <= size), not
1533                 * (bytes < size).  In other words, even though we
1534                 * expect no more output and set avail_out to zero,
1535                 * the input zlib stream may have bytes that express
1536                 * "this concludes the stream", and we *do* want to
1537                 * eat that input.
1538                 *
1539                 * Otherwise we would not be able to test that we
1540                 * consumed all the input to reach the expected size;
1541                 * we also want to check that zlib tells us that all
1542                 * went well with status == Z_STREAM_END at the end.
1543                 */
1544                stream->next_out = buf + bytes;
1545                stream->avail_out = size - bytes;
1546                while (status == Z_OK)
1547                        status = git_inflate(stream, Z_FINISH);
1548        }
1549        if (status == Z_STREAM_END && !stream->avail_in) {
1550                git_inflate_end(stream);
1551                return buf;
1552        }
1553
1554        if (status < 0)
1555                error("corrupt loose object '%s'", sha1_to_hex(sha1));
1556        else if (stream->avail_in)
1557                error("garbage at end of loose object '%s'",
1558                      sha1_to_hex(sha1));
1559        free(buf);
1560        return NULL;
1561}
1562
1563/*
1564 * We used to just use "sscanf()", but that's actually way
1565 * too permissive for what we want to check. So do an anal
1566 * object header parse by hand.
1567 */
1568int parse_sha1_header(const char *hdr, unsigned long *sizep)
1569{
1570        char type[10];
1571        int i;
1572        unsigned long size;
1573
1574        /*
1575         * The type can be at most ten bytes (including the
1576         * terminating '\0' that we add), and is followed by
1577         * a space.
1578         */
1579        i = 0;
1580        for (;;) {
1581                char c = *hdr++;
1582                if (c == ' ')
1583                        break;
1584                type[i++] = c;
1585                if (i >= sizeof(type))
1586                        return -1;
1587        }
1588        type[i] = 0;
1589
1590        /*
1591         * The length must follow immediately, and be in canonical
1592         * decimal format (ie "010" is not valid).
1593         */
1594        size = *hdr++ - '0';
1595        if (size > 9)
1596                return -1;
1597        if (size) {
1598                for (;;) {
1599                        unsigned long c = *hdr - '0';
1600                        if (c > 9)
1601                                break;
1602                        hdr++;
1603                        size = size * 10 + c;
1604                }
1605        }
1606        *sizep = size;
1607
1608        /*
1609         * The length must be followed by a zero byte
1610         */
1611        return *hdr ? -1 : type_from_string(type);
1612}
1613
1614static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
1615{
1616        int ret;
1617        git_zstream stream;
1618        char hdr[8192];
1619
1620        ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr));
1621        if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0)
1622                return NULL;
1623
1624        return unpack_sha1_rest(&stream, hdr, *size, sha1);
1625}
1626
1627unsigned long get_size_from_delta(struct packed_git *p,
1628                                  struct pack_window **w_curs,
1629                                  off_t curpos)
1630{
1631        const unsigned char *data;
1632        unsigned char delta_head[20], *in;
1633        git_zstream stream;
1634        int st;
1635
1636        memset(&stream, 0, sizeof(stream));
1637        stream.next_out = delta_head;
1638        stream.avail_out = sizeof(delta_head);
1639
1640        git_inflate_init(&stream);
1641        do {
1642                in = use_pack(p, w_curs, curpos, &stream.avail_in);
1643                stream.next_in = in;
1644                st = git_inflate(&stream, Z_FINISH);
1645                curpos += stream.next_in - in;
1646        } while ((st == Z_OK || st == Z_BUF_ERROR) &&
1647                 stream.total_out < sizeof(delta_head));
1648        git_inflate_end(&stream);
1649        if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head)) {
1650                error("delta data unpack-initial failed");
1651                return 0;
1652        }
1653
1654        /* Examine the initial part of the delta to figure out
1655         * the result size.
1656         */
1657        data = delta_head;
1658
1659        /* ignore base size */
1660        get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
1661
1662        /* Read the result size */
1663        return get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
1664}
1665
1666static off_t get_delta_base(struct packed_git *p,
1667                                    struct pack_window **w_curs,
1668                                    off_t *curpos,
1669                                    enum object_type type,
1670                                    off_t delta_obj_offset)
1671{
1672        unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL);
1673        off_t base_offset;
1674
1675        /* use_pack() assured us we have [base_info, base_info + 20)
1676         * as a range that we can look at without walking off the
1677         * end of the mapped window.  Its actually the hash size
1678         * that is assured.  An OFS_DELTA longer than the hash size
1679         * is stupid, as then a REF_DELTA would be smaller to store.
1680         */
1681        if (type == OBJ_OFS_DELTA) {
1682                unsigned used = 0;
1683                unsigned char c = base_info[used++];
1684                base_offset = c & 127;
1685                while (c & 128) {
1686                        base_offset += 1;
1687                        if (!base_offset || MSB(base_offset, 7))
1688                                return 0;  /* overflow */
1689                        c = base_info[used++];
1690                        base_offset = (base_offset << 7) + (c & 127);
1691                }
1692                base_offset = delta_obj_offset - base_offset;
1693                if (base_offset <= 0 || base_offset >= delta_obj_offset)
1694                        return 0;  /* out of bound */
1695                *curpos += used;
1696        } else if (type == OBJ_REF_DELTA) {
1697                /* The base entry _must_ be in the same pack */
1698                base_offset = find_pack_entry_one(base_info, p);
1699                *curpos += 20;
1700        } else
1701                die("I am totally screwed");
1702        return base_offset;
1703}
1704
1705/*
1706 * Like get_delta_base above, but we return the sha1 instead of the pack
1707 * offset. This means it is cheaper for REF deltas (we do not have to do
1708 * the final object lookup), but more expensive for OFS deltas (we
1709 * have to load the revidx to convert the offset back into a sha1).
1710 */
1711static const unsigned char *get_delta_base_sha1(struct packed_git *p,
1712                                                struct pack_window **w_curs,
1713                                                off_t curpos,
1714                                                enum object_type type,
1715                                                off_t delta_obj_offset)
1716{
1717        if (type == OBJ_REF_DELTA) {
1718                unsigned char *base = use_pack(p, w_curs, curpos, NULL);
1719                return base;
1720        } else if (type == OBJ_OFS_DELTA) {
1721                struct revindex_entry *revidx;
1722                off_t base_offset = get_delta_base(p, w_curs, &curpos,
1723                                                   type, delta_obj_offset);
1724
1725                if (!base_offset)
1726                        return NULL;
1727
1728                revidx = find_pack_revindex(p, base_offset);
1729                if (!revidx)
1730                        return NULL;
1731
1732                return nth_packed_object_sha1(p, revidx->nr);
1733        } else
1734                return NULL;
1735}
1736
1737int unpack_object_header(struct packed_git *p,
1738                         struct pack_window **w_curs,
1739                         off_t *curpos,
1740                         unsigned long *sizep)
1741{
1742        unsigned char *base;
1743        unsigned long left;
1744        unsigned long used;
1745        enum object_type type;
1746
1747        /* use_pack() assures us we have [base, base + 20) available
1748         * as a range that we can look at.  (Its actually the hash
1749         * size that is assured.)  With our object header encoding
1750         * the maximum deflated object size is 2^137, which is just
1751         * insane, so we know won't exceed what we have been given.
1752         */
1753        base = use_pack(p, w_curs, *curpos, &left);
1754        used = unpack_object_header_buffer(base, left, &type, sizep);
1755        if (!used) {
1756                type = OBJ_BAD;
1757        } else
1758                *curpos += used;
1759
1760        return type;
1761}
1762
1763static int retry_bad_packed_offset(struct packed_git *p, off_t obj_offset)
1764{
1765        int type;
1766        struct revindex_entry *revidx;
1767        const unsigned char *sha1;
1768        revidx = find_pack_revindex(p, obj_offset);
1769        if (!revidx)
1770                return OBJ_BAD;
1771        sha1 = nth_packed_object_sha1(p, revidx->nr);
1772        mark_bad_packed_object(p, sha1);
1773        type = sha1_object_info(sha1, NULL);
1774        if (type <= OBJ_NONE)
1775                return OBJ_BAD;
1776        return type;
1777}
1778
1779#define POI_STACK_PREALLOC 64
1780
1781static enum object_type packed_to_object_type(struct packed_git *p,
1782                                              off_t obj_offset,
1783                                              enum object_type type,
1784                                              struct pack_window **w_curs,
1785                                              off_t curpos)
1786{
1787        off_t small_poi_stack[POI_STACK_PREALLOC];
1788        off_t *poi_stack = small_poi_stack;
1789        int poi_stack_nr = 0, poi_stack_alloc = POI_STACK_PREALLOC;
1790
1791        while (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
1792                off_t base_offset;
1793                unsigned long size;
1794                /* Push the object we're going to leave behind */
1795                if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) {
1796                        poi_stack_alloc = alloc_nr(poi_stack_nr);
1797                        poi_stack = xmalloc(sizeof(off_t)*poi_stack_alloc);
1798                        memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr);
1799                } else {
1800                        ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc);
1801                }
1802                poi_stack[poi_stack_nr++] = obj_offset;
1803                /* If parsing the base offset fails, just unwind */
1804                base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
1805                if (!base_offset)
1806                        goto unwind;
1807                curpos = obj_offset = base_offset;
1808                type = unpack_object_header(p, w_curs, &curpos, &size);
1809                if (type <= OBJ_NONE) {
1810                        /* If getting the base itself fails, we first
1811                         * retry the base, otherwise unwind */
1812                        type = retry_bad_packed_offset(p, base_offset);
1813                        if (type > OBJ_NONE)
1814                                goto out;
1815                        goto unwind;
1816                }
1817        }
1818
1819        switch (type) {
1820        case OBJ_BAD:
1821        case OBJ_COMMIT:
1822        case OBJ_TREE:
1823        case OBJ_BLOB:
1824        case OBJ_TAG:
1825                break;
1826        default:
1827                error("unknown object type %i at offset %"PRIuMAX" in %s",
1828                      type, (uintmax_t)obj_offset, p->pack_name);
1829                type = OBJ_BAD;
1830        }
1831
1832out:
1833        if (poi_stack != small_poi_stack)
1834                free(poi_stack);
1835        return type;
1836
1837unwind:
1838        while (poi_stack_nr) {
1839                obj_offset = poi_stack[--poi_stack_nr];
1840                type = retry_bad_packed_offset(p, obj_offset);
1841                if (type > OBJ_NONE)
1842                        goto out;
1843        }
1844        type = OBJ_BAD;
1845        goto out;
1846}
1847
1848static int packed_object_info(struct packed_git *p, off_t obj_offset,
1849                              struct object_info *oi)
1850{
1851        struct pack_window *w_curs = NULL;
1852        unsigned long size;
1853        off_t curpos = obj_offset;
1854        enum object_type type;
1855
1856        /*
1857         * We always get the representation type, but only convert it to
1858         * a "real" type later if the caller is interested.
1859         */
1860        type = unpack_object_header(p, &w_curs, &curpos, &size);
1861
1862        if (oi->sizep) {
1863                if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
1864                        off_t tmp_pos = curpos;
1865                        off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos,
1866                                                           type, obj_offset);
1867                        if (!base_offset) {
1868                                type = OBJ_BAD;
1869                                goto out;
1870                        }
1871                        *oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos);
1872                        if (*oi->sizep == 0) {
1873                                type = OBJ_BAD;
1874                                goto out;
1875                        }
1876                } else {
1877                        *oi->sizep = size;
1878                }
1879        }
1880
1881        if (oi->disk_sizep) {
1882                struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
1883                *oi->disk_sizep = revidx[1].offset - obj_offset;
1884        }
1885
1886        if (oi->typep) {
1887                *oi->typep = packed_to_object_type(p, obj_offset, type, &w_curs, curpos);
1888                if (*oi->typep < 0) {
1889                        type = OBJ_BAD;
1890                        goto out;
1891                }
1892        }
1893
1894        if (oi->delta_base_sha1) {
1895                if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
1896                        const unsigned char *base;
1897
1898                        base = get_delta_base_sha1(p, &w_curs, curpos,
1899                                                   type, obj_offset);
1900                        if (!base) {
1901                                type = OBJ_BAD;
1902                                goto out;
1903                        }
1904
1905                        hashcpy(oi->delta_base_sha1, base);
1906                } else
1907                        hashclr(oi->delta_base_sha1);
1908        }
1909
1910out:
1911        unuse_pack(&w_curs);
1912        return type;
1913}
1914
1915static void *unpack_compressed_entry(struct packed_git *p,
1916                                    struct pack_window **w_curs,
1917                                    off_t curpos,
1918                                    unsigned long size)
1919{
1920        int st;
1921        git_zstream stream;
1922        unsigned char *buffer, *in;
1923
1924        buffer = xmallocz(size);
1925        memset(&stream, 0, sizeof(stream));
1926        stream.next_out = buffer;
1927        stream.avail_out = size + 1;
1928
1929        git_inflate_init(&stream);
1930        do {
1931                in = use_pack(p, w_curs, curpos, &stream.avail_in);
1932                stream.next_in = in;
1933                st = git_inflate(&stream, Z_FINISH);
1934                if (!stream.avail_out)
1935                        break; /* the payload is larger than it should be */
1936                curpos += stream.next_in - in;
1937        } while (st == Z_OK || st == Z_BUF_ERROR);
1938        git_inflate_end(&stream);
1939        if ((st != Z_STREAM_END) || stream.total_out != size) {
1940                free(buffer);
1941                return NULL;
1942        }
1943
1944        return buffer;
1945}
1946
1947#define MAX_DELTA_CACHE (256)
1948
1949static size_t delta_base_cached;
1950
1951static struct delta_base_cache_lru_list {
1952        struct delta_base_cache_lru_list *prev;
1953        struct delta_base_cache_lru_list *next;
1954} delta_base_cache_lru = { &delta_base_cache_lru, &delta_base_cache_lru };
1955
1956static struct delta_base_cache_entry {
1957        struct delta_base_cache_lru_list lru;
1958        void *data;
1959        struct packed_git *p;
1960        off_t base_offset;
1961        unsigned long size;
1962        enum object_type type;
1963} delta_base_cache[MAX_DELTA_CACHE];
1964
1965static unsigned long pack_entry_hash(struct packed_git *p, off_t base_offset)
1966{
1967        unsigned long hash;
1968
1969        hash = (unsigned long)p + (unsigned long)base_offset;
1970        hash += (hash >> 8) + (hash >> 16);
1971        return hash % MAX_DELTA_CACHE;
1972}
1973
1974static struct delta_base_cache_entry *
1975get_delta_base_cache_entry(struct packed_git *p, off_t base_offset)
1976{
1977        unsigned long hash = pack_entry_hash(p, base_offset);
1978        return delta_base_cache + hash;
1979}
1980
1981static int eq_delta_base_cache_entry(struct delta_base_cache_entry *ent,
1982                                     struct packed_git *p, off_t base_offset)
1983{
1984        return (ent->data && ent->p == p && ent->base_offset == base_offset);
1985}
1986
1987static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
1988{
1989        struct delta_base_cache_entry *ent;
1990        ent = get_delta_base_cache_entry(p, base_offset);
1991        return eq_delta_base_cache_entry(ent, p, base_offset);
1992}
1993
1994static void clear_delta_base_cache_entry(struct delta_base_cache_entry *ent)
1995{
1996        ent->data = NULL;
1997        ent->lru.next->prev = ent->lru.prev;
1998        ent->lru.prev->next = ent->lru.next;
1999        delta_base_cached -= ent->size;
2000}
2001
2002static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
2003        unsigned long *base_size, enum object_type *type, int keep_cache)
2004{
2005        struct delta_base_cache_entry *ent;
2006        void *ret;
2007
2008        ent = get_delta_base_cache_entry(p, base_offset);
2009
2010        if (!eq_delta_base_cache_entry(ent, p, base_offset))
2011                return unpack_entry(p, base_offset, type, base_size);
2012
2013        ret = ent->data;
2014
2015        if (!keep_cache)
2016                clear_delta_base_cache_entry(ent);
2017        else
2018                ret = xmemdupz(ent->data, ent->size);
2019        *type = ent->type;
2020        *base_size = ent->size;
2021        return ret;
2022}
2023
2024static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
2025{
2026        if (ent->data) {
2027                free(ent->data);
2028                ent->data = NULL;
2029                ent->lru.next->prev = ent->lru.prev;
2030                ent->lru.prev->next = ent->lru.next;
2031                delta_base_cached -= ent->size;
2032        }
2033}
2034
2035void clear_delta_base_cache(void)
2036{
2037        unsigned long p;
2038        for (p = 0; p < MAX_DELTA_CACHE; p++)
2039                release_delta_base_cache(&delta_base_cache[p]);
2040}
2041
2042static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
2043        void *base, unsigned long base_size, enum object_type type)
2044{
2045        unsigned long hash = pack_entry_hash(p, base_offset);
2046        struct delta_base_cache_entry *ent = delta_base_cache + hash;
2047        struct delta_base_cache_lru_list *lru;
2048
2049        release_delta_base_cache(ent);
2050        delta_base_cached += base_size;
2051
2052        for (lru = delta_base_cache_lru.next;
2053             delta_base_cached > delta_base_cache_limit
2054             && lru != &delta_base_cache_lru;
2055             lru = lru->next) {
2056                struct delta_base_cache_entry *f = (void *)lru;
2057                if (f->type == OBJ_BLOB)
2058                        release_delta_base_cache(f);
2059        }
2060        for (lru = delta_base_cache_lru.next;
2061             delta_base_cached > delta_base_cache_limit
2062             && lru != &delta_base_cache_lru;
2063             lru = lru->next) {
2064                struct delta_base_cache_entry *f = (void *)lru;
2065                release_delta_base_cache(f);
2066        }
2067
2068        ent->p = p;
2069        ent->base_offset = base_offset;
2070        ent->type = type;
2071        ent->data = base;
2072        ent->size = base_size;
2073        ent->lru.next = &delta_base_cache_lru;
2074        ent->lru.prev = delta_base_cache_lru.prev;
2075        delta_base_cache_lru.prev->next = &ent->lru;
2076        delta_base_cache_lru.prev = &ent->lru;
2077}
2078
2079static void *read_object(const unsigned char *sha1, enum object_type *type,
2080                         unsigned long *size);
2081
2082static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
2083{
2084        static FILE *log_file;
2085
2086        if (!log_pack_access)
2087                log_pack_access = getenv("GIT_TRACE_PACK_ACCESS");
2088        if (!log_pack_access)
2089                log_pack_access = no_log_pack_access;
2090        if (log_pack_access == no_log_pack_access)
2091                return;
2092
2093        if (!log_file) {
2094                log_file = fopen(log_pack_access, "w");
2095                if (!log_file) {
2096                        error("cannot open pack access log '%s' for writing: %s",
2097                              log_pack_access, strerror(errno));
2098                        log_pack_access = no_log_pack_access;
2099                        return;
2100                }
2101        }
2102        fprintf(log_file, "%s %"PRIuMAX"\n",
2103                p->pack_name, (uintmax_t)obj_offset);
2104        fflush(log_file);
2105}
2106
2107int do_check_packed_object_crc;
2108
2109#define UNPACK_ENTRY_STACK_PREALLOC 64
2110struct unpack_entry_stack_ent {
2111        off_t obj_offset;
2112        off_t curpos;
2113        unsigned long size;
2114};
2115
2116void *unpack_entry(struct packed_git *p, off_t obj_offset,
2117                   enum object_type *final_type, unsigned long *final_size)
2118{
2119        struct pack_window *w_curs = NULL;
2120        off_t curpos = obj_offset;
2121        void *data = NULL;
2122        unsigned long size;
2123        enum object_type type;
2124        struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC];
2125        struct unpack_entry_stack_ent *delta_stack = small_delta_stack;
2126        int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC;
2127        int base_from_cache = 0;
2128
2129        if (log_pack_access != no_log_pack_access)
2130                write_pack_access_log(p, obj_offset);
2131
2132        /* PHASE 1: drill down to the innermost base object */
2133        for (;;) {
2134                off_t base_offset;
2135                int i;
2136                struct delta_base_cache_entry *ent;
2137
2138                ent = get_delta_base_cache_entry(p, curpos);
2139                if (eq_delta_base_cache_entry(ent, p, curpos)) {
2140                        type = ent->type;
2141                        data = ent->data;
2142                        size = ent->size;
2143                        clear_delta_base_cache_entry(ent);
2144                        base_from_cache = 1;
2145                        break;
2146                }
2147
2148                if (do_check_packed_object_crc && p->index_version > 1) {
2149                        struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
2150                        unsigned long len = revidx[1].offset - obj_offset;
2151                        if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
2152                                const unsigned char *sha1 =
2153                                        nth_packed_object_sha1(p, revidx->nr);
2154                                error("bad packed object CRC for %s",
2155                                      sha1_to_hex(sha1));
2156                                mark_bad_packed_object(p, sha1);
2157                                unuse_pack(&w_curs);
2158                                return NULL;
2159                        }
2160                }
2161
2162                type = unpack_object_header(p, &w_curs, &curpos, &size);
2163                if (type != OBJ_OFS_DELTA && type != OBJ_REF_DELTA)
2164                        break;
2165
2166                base_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
2167                if (!base_offset) {
2168                        error("failed to validate delta base reference "
2169                              "at offset %"PRIuMAX" from %s",
2170                              (uintmax_t)curpos, p->pack_name);
2171                        /* bail to phase 2, in hopes of recovery */
2172                        data = NULL;
2173                        break;
2174                }
2175
2176                /* push object, proceed to base */
2177                if (delta_stack_nr >= delta_stack_alloc
2178                    && delta_stack == small_delta_stack) {
2179                        delta_stack_alloc = alloc_nr(delta_stack_nr);
2180                        delta_stack = xmalloc(sizeof(*delta_stack)*delta_stack_alloc);
2181                        memcpy(delta_stack, small_delta_stack,
2182                               sizeof(*delta_stack)*delta_stack_nr);
2183                } else {
2184                        ALLOC_GROW(delta_stack, delta_stack_nr+1, delta_stack_alloc);
2185                }
2186                i = delta_stack_nr++;
2187                delta_stack[i].obj_offset = obj_offset;
2188                delta_stack[i].curpos = curpos;
2189                delta_stack[i].size = size;
2190
2191                curpos = obj_offset = base_offset;
2192        }
2193
2194        /* PHASE 2: handle the base */
2195        switch (type) {
2196        case OBJ_OFS_DELTA:
2197        case OBJ_REF_DELTA:
2198                if (data)
2199                        die("BUG in unpack_entry: left loop at a valid delta");
2200                break;
2201        case OBJ_COMMIT:
2202        case OBJ_TREE:
2203        case OBJ_BLOB:
2204        case OBJ_TAG:
2205                if (!base_from_cache)
2206                        data = unpack_compressed_entry(p, &w_curs, curpos, size);
2207                break;
2208        default:
2209                data = NULL;
2210                error("unknown object type %i at offset %"PRIuMAX" in %s",
2211                      type, (uintmax_t)obj_offset, p->pack_name);
2212        }
2213
2214        /* PHASE 3: apply deltas in order */
2215
2216        /* invariants:
2217         *   'data' holds the base data, or NULL if there was corruption
2218         */
2219        while (delta_stack_nr) {
2220                void *delta_data;
2221                void *base = data;
2222                unsigned long delta_size, base_size = size;
2223                int i;
2224
2225                data = NULL;
2226
2227                if (base)
2228                        add_delta_base_cache(p, obj_offset, base, base_size, type);
2229
2230                if (!base) {
2231                        /*
2232                         * We're probably in deep shit, but let's try to fetch
2233                         * the required base anyway from another pack or loose.
2234                         * This is costly but should happen only in the presence
2235                         * of a corrupted pack, and is better than failing outright.
2236                         */
2237                        struct revindex_entry *revidx;
2238                        const unsigned char *base_sha1;
2239                        revidx = find_pack_revindex(p, obj_offset);
2240                        if (revidx) {
2241                                base_sha1 = nth_packed_object_sha1(p, revidx->nr);
2242                                error("failed to read delta base object %s"
2243                                      " at offset %"PRIuMAX" from %s",
2244                                      sha1_to_hex(base_sha1), (uintmax_t)obj_offset,
2245                                      p->pack_name);
2246                                mark_bad_packed_object(p, base_sha1);
2247                                base = read_object(base_sha1, &type, &base_size);
2248                        }
2249                }
2250
2251                i = --delta_stack_nr;
2252                obj_offset = delta_stack[i].obj_offset;
2253                curpos = delta_stack[i].curpos;
2254                delta_size = delta_stack[i].size;
2255
2256                if (!base)
2257                        continue;
2258
2259                delta_data = unpack_compressed_entry(p, &w_curs, curpos, delta_size);
2260
2261                if (!delta_data) {
2262                        error("failed to unpack compressed delta "
2263                              "at offset %"PRIuMAX" from %s",
2264                              (uintmax_t)curpos, p->pack_name);
2265                        data = NULL;
2266                        continue;
2267                }
2268
2269                data = patch_delta(base, base_size,
2270                                   delta_data, delta_size,
2271                                   &size);
2272
2273                /*
2274                 * We could not apply the delta; warn the user, but keep going.
2275                 * Our failure will be noticed either in the next iteration of
2276                 * the loop, or if this is the final delta, in the caller when
2277                 * we return NULL. Those code paths will take care of making
2278                 * a more explicit warning and retrying with another copy of
2279                 * the object.
2280                 */
2281                if (!data)
2282                        error("failed to apply delta");
2283
2284                free(delta_data);
2285        }
2286
2287        *final_type = type;
2288        *final_size = size;
2289
2290        unuse_pack(&w_curs);
2291
2292        if (delta_stack != small_delta_stack)
2293                free(delta_stack);
2294
2295        return data;
2296}
2297
2298const unsigned char *nth_packed_object_sha1(struct packed_git *p,
2299                                            uint32_t n)
2300{
2301        const unsigned char *index = p->index_data;
2302        if (!index) {
2303                if (open_pack_index(p))
2304                        return NULL;
2305                index = p->index_data;
2306        }
2307        if (n >= p->num_objects)
2308                return NULL;
2309        index += 4 * 256;
2310        if (p->index_version == 1) {
2311                return index + 24 * n + 4;
2312        } else {
2313                index += 8;
2314                return index + 20 * n;
2315        }
2316}
2317
2318off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
2319{
2320        const unsigned char *index = p->index_data;
2321        index += 4 * 256;
2322        if (p->index_version == 1) {
2323                return ntohl(*((uint32_t *)(index + 24 * n)));
2324        } else {
2325                uint32_t off;
2326                index += 8 + p->num_objects * (20 + 4);
2327                off = ntohl(*((uint32_t *)(index + 4 * n)));
2328                if (!(off & 0x80000000))
2329                        return off;
2330                index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
2331                return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
2332                                   ntohl(*((uint32_t *)(index + 4)));
2333        }
2334}
2335
2336off_t find_pack_entry_one(const unsigned char *sha1,
2337                                  struct packed_git *p)
2338{
2339        const uint32_t *level1_ofs = p->index_data;
2340        const unsigned char *index = p->index_data;
2341        unsigned hi, lo, stride;
2342        static int use_lookup = -1;
2343        static int debug_lookup = -1;
2344
2345        if (debug_lookup < 0)
2346                debug_lookup = !!getenv("GIT_DEBUG_LOOKUP");
2347
2348        if (!index) {
2349                if (open_pack_index(p))
2350                        return 0;
2351                level1_ofs = p->index_data;
2352                index = p->index_data;
2353        }
2354        if (p->index_version > 1) {
2355                level1_ofs += 2;
2356                index += 8;
2357        }
2358        index += 4 * 256;
2359        hi = ntohl(level1_ofs[*sha1]);
2360        lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
2361        if (p->index_version > 1) {
2362                stride = 20;
2363        } else {
2364                stride = 24;
2365                index += 4;
2366        }
2367
2368        if (debug_lookup)
2369                printf("%02x%02x%02x... lo %u hi %u nr %"PRIu32"\n",
2370                       sha1[0], sha1[1], sha1[2], lo, hi, p->num_objects);
2371
2372        if (use_lookup < 0)
2373                use_lookup = !!getenv("GIT_USE_LOOKUP");
2374        if (use_lookup) {
2375                int pos = sha1_entry_pos(index, stride, 0,
2376                                         lo, hi, p->num_objects, sha1);
2377                if (pos < 0)
2378                        return 0;
2379                return nth_packed_object_offset(p, pos);
2380        }
2381
2382        do {
2383                unsigned mi = (lo + hi) / 2;
2384                int cmp = hashcmp(index + mi * stride, sha1);
2385
2386                if (debug_lookup)
2387                        printf("lo %u hi %u rg %u mi %u\n",
2388                               lo, hi, hi - lo, mi);
2389                if (!cmp)
2390                        return nth_packed_object_offset(p, mi);
2391                if (cmp > 0)
2392                        hi = mi;
2393                else
2394                        lo = mi+1;
2395        } while (lo < hi);
2396        return 0;
2397}
2398
2399int is_pack_valid(struct packed_git *p)
2400{
2401        /* An already open pack is known to be valid. */
2402        if (p->pack_fd != -1)
2403                return 1;
2404
2405        /* If the pack has one window completely covering the
2406         * file size, the pack is known to be valid even if
2407         * the descriptor is not currently open.
2408         */
2409        if (p->windows) {
2410                struct pack_window *w = p->windows;
2411
2412                if (!w->offset && w->len == p->pack_size)
2413                        return 1;
2414        }
2415
2416        /* Force the pack to open to prove its valid. */
2417        return !open_packed_git(p);
2418}
2419
2420static int fill_pack_entry(const unsigned char *sha1,
2421                           struct pack_entry *e,
2422                           struct packed_git *p)
2423{
2424        off_t offset;
2425
2426        if (p->num_bad_objects) {
2427                unsigned i;
2428                for (i = 0; i < p->num_bad_objects; i++)
2429                        if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
2430                                return 0;
2431        }
2432
2433        offset = find_pack_entry_one(sha1, p);
2434        if (!offset)
2435                return 0;
2436
2437        /*
2438         * We are about to tell the caller where they can locate the
2439         * requested object.  We better make sure the packfile is
2440         * still here and can be accessed before supplying that
2441         * answer, as it may have been deleted since the index was
2442         * loaded!
2443         */
2444        if (!is_pack_valid(p)) {
2445                warning("packfile %s cannot be accessed", p->pack_name);
2446                return 0;
2447        }
2448        e->offset = offset;
2449        e->p = p;
2450        hashcpy(e->sha1, sha1);
2451        return 1;
2452}
2453
2454/*
2455 * Iff a pack file contains the object named by sha1, return true and
2456 * store its location to e.
2457 */
2458static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
2459{
2460        struct packed_git *p;
2461
2462        prepare_packed_git();
2463        if (!packed_git)
2464                return 0;
2465
2466        if (last_found_pack && fill_pack_entry(sha1, e, last_found_pack))
2467                return 1;
2468
2469        for (p = packed_git; p; p = p->next) {
2470                if (p == last_found_pack)
2471                        continue; /* we already checked this one */
2472
2473                if (fill_pack_entry(sha1, e, p)) {
2474                        last_found_pack = p;
2475                        return 1;
2476                }
2477        }
2478        return 0;
2479}
2480
2481struct packed_git *find_sha1_pack(const unsigned char *sha1,
2482                                  struct packed_git *packs)
2483{
2484        struct packed_git *p;
2485
2486        for (p = packs; p; p = p->next) {
2487                if (find_pack_entry_one(sha1, p))
2488                        return p;
2489        }
2490        return NULL;
2491
2492}
2493
2494static int sha1_loose_object_info(const unsigned char *sha1,
2495                                  struct object_info *oi)
2496{
2497        int status;
2498        unsigned long mapsize, size;
2499        void *map;
2500        git_zstream stream;
2501        char hdr[32];
2502
2503        if (oi->delta_base_sha1)
2504                hashclr(oi->delta_base_sha1);
2505
2506        /*
2507         * If we don't care about type or size, then we don't
2508         * need to look inside the object at all. Note that we
2509         * do not optimize out the stat call, even if the
2510         * caller doesn't care about the disk-size, since our
2511         * return value implicitly indicates whether the
2512         * object even exists.
2513         */
2514        if (!oi->typep && !oi->sizep) {
2515                struct stat st;
2516                if (stat_sha1_file(sha1, &st) < 0)
2517                        return -1;
2518                if (oi->disk_sizep)
2519                        *oi->disk_sizep = st.st_size;
2520                return 0;
2521        }
2522
2523        map = map_sha1_file(sha1, &mapsize);
2524        if (!map)
2525                return -1;
2526        if (oi->disk_sizep)
2527                *oi->disk_sizep = mapsize;
2528        if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
2529                status = error("unable to unpack %s header",
2530                               sha1_to_hex(sha1));
2531        else if ((status = parse_sha1_header(hdr, &size)) < 0)
2532                status = error("unable to parse %s header", sha1_to_hex(sha1));
2533        else if (oi->sizep)
2534                *oi->sizep = size;
2535        git_inflate_end(&stream);
2536        munmap(map, mapsize);
2537        if (oi->typep)
2538                *oi->typep = status;
2539        return 0;
2540}
2541
2542int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags)
2543{
2544        struct cached_object *co;
2545        struct pack_entry e;
2546        int rtype;
2547        const unsigned char *real = lookup_replace_object_extended(sha1, flags);
2548
2549        co = find_cached_object(real);
2550        if (co) {
2551                if (oi->typep)
2552                        *(oi->typep) = co->type;
2553                if (oi->sizep)
2554                        *(oi->sizep) = co->size;
2555                if (oi->disk_sizep)
2556                        *(oi->disk_sizep) = 0;
2557                if (oi->delta_base_sha1)
2558                        hashclr(oi->delta_base_sha1);
2559                oi->whence = OI_CACHED;
2560                return 0;
2561        }
2562
2563        if (!find_pack_entry(real, &e)) {
2564                /* Most likely it's a loose object. */
2565                if (!sha1_loose_object_info(real, oi)) {
2566                        oi->whence = OI_LOOSE;
2567                        return 0;
2568                }
2569
2570                /* Not a loose object; someone else may have just packed it. */
2571                reprepare_packed_git();
2572                if (!find_pack_entry(real, &e))
2573                        return -1;
2574        }
2575
2576        rtype = packed_object_info(e.p, e.offset, oi);
2577        if (rtype < 0) {
2578                mark_bad_packed_object(e.p, real);
2579                return sha1_object_info_extended(real, oi, 0);
2580        } else if (in_delta_base_cache(e.p, e.offset)) {
2581                oi->whence = OI_DBCACHED;
2582        } else {
2583                oi->whence = OI_PACKED;
2584                oi->u.packed.offset = e.offset;
2585                oi->u.packed.pack = e.p;
2586                oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
2587                                         rtype == OBJ_OFS_DELTA);
2588        }
2589
2590        return 0;
2591}
2592
2593/* returns enum object_type or negative */
2594int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
2595{
2596        enum object_type type;
2597        struct object_info oi = {NULL};
2598
2599        oi.typep = &type;
2600        oi.sizep = sizep;
2601        if (sha1_object_info_extended(sha1, &oi, LOOKUP_REPLACE_OBJECT) < 0)
2602                return -1;
2603        return type;
2604}
2605
2606static void *read_packed_sha1(const unsigned char *sha1,
2607                              enum object_type *type, unsigned long *size)
2608{
2609        struct pack_entry e;
2610        void *data;
2611
2612        if (!find_pack_entry(sha1, &e))
2613                return NULL;
2614        data = cache_or_unpack_entry(e.p, e.offset, size, type, 1);
2615        if (!data) {
2616                /*
2617                 * We're probably in deep shit, but let's try to fetch
2618                 * the required object anyway from another pack or loose.
2619                 * This should happen only in the presence of a corrupted
2620                 * pack, and is better than failing outright.
2621                 */
2622                error("failed to read object %s at offset %"PRIuMAX" from %s",
2623                      sha1_to_hex(sha1), (uintmax_t)e.offset, e.p->pack_name);
2624                mark_bad_packed_object(e.p, sha1);
2625                data = read_object(sha1, type, size);
2626        }
2627        return data;
2628}
2629
2630int pretend_sha1_file(void *buf, unsigned long len, enum object_type type,
2631                      unsigned char *sha1)
2632{
2633        struct cached_object *co;
2634
2635        hash_sha1_file(buf, len, typename(type), sha1);
2636        if (has_sha1_file(sha1) || find_cached_object(sha1))
2637                return 0;
2638        ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc);
2639        co = &cached_objects[cached_object_nr++];
2640        co->size = len;
2641        co->type = type;
2642        co->buf = xmalloc(len);
2643        memcpy(co->buf, buf, len);
2644        hashcpy(co->sha1, sha1);
2645        return 0;
2646}
2647
2648static void *read_object(const unsigned char *sha1, enum object_type *type,
2649                         unsigned long *size)
2650{
2651        unsigned long mapsize;
2652        void *map, *buf;
2653        struct cached_object *co;
2654
2655        co = find_cached_object(sha1);
2656        if (co) {
2657                *type = co->type;
2658                *size = co->size;
2659                return xmemdupz(co->buf, co->size);
2660        }
2661
2662        buf = read_packed_sha1(sha1, type, size);
2663        if (buf)
2664                return buf;
2665        map = map_sha1_file(sha1, &mapsize);
2666        if (map) {
2667                buf = unpack_sha1_file(map, mapsize, type, size, sha1);
2668                munmap(map, mapsize);
2669                return buf;
2670        }
2671        reprepare_packed_git();
2672        return read_packed_sha1(sha1, type, size);
2673}
2674
2675/*
2676 * This function dies on corrupt objects; the callers who want to
2677 * deal with them should arrange to call read_object() and give error
2678 * messages themselves.
2679 */
2680void *read_sha1_file_extended(const unsigned char *sha1,
2681                              enum object_type *type,
2682                              unsigned long *size,
2683                              unsigned flag)
2684{
2685        void *data;
2686        const struct packed_git *p;
2687        const unsigned char *repl = lookup_replace_object_extended(sha1, flag);
2688
2689        errno = 0;
2690        data = read_object(repl, type, size);
2691        if (data)
2692                return data;
2693
2694        if (errno && errno != ENOENT)
2695                die_errno("failed to read object %s", sha1_to_hex(sha1));
2696
2697        /* die if we replaced an object with one that does not exist */
2698        if (repl != sha1)
2699                die("replacement %s not found for %s",
2700                    sha1_to_hex(repl), sha1_to_hex(sha1));
2701
2702        if (has_loose_object(repl)) {
2703                const char *path = sha1_file_name(sha1);
2704
2705                die("loose object %s (stored in %s) is corrupt",
2706                    sha1_to_hex(repl), path);
2707        }
2708
2709        if ((p = has_packed_and_bad(repl)) != NULL)
2710                die("packed object %s (stored in %s) is corrupt",
2711                    sha1_to_hex(repl), p->pack_name);
2712
2713        return NULL;
2714}
2715
2716void *read_object_with_reference(const unsigned char *sha1,
2717                                 const char *required_type_name,
2718                                 unsigned long *size,
2719                                 unsigned char *actual_sha1_return)
2720{
2721        enum object_type type, required_type;
2722        void *buffer;
2723        unsigned long isize;
2724        unsigned char actual_sha1[20];
2725
2726        required_type = type_from_string(required_type_name);
2727        hashcpy(actual_sha1, sha1);
2728        while (1) {
2729                int ref_length = -1;
2730                const char *ref_type = NULL;
2731
2732                buffer = read_sha1_file(actual_sha1, &type, &isize);
2733                if (!buffer)
2734                        return NULL;
2735                if (type == required_type) {
2736                        *size = isize;
2737                        if (actual_sha1_return)
2738                                hashcpy(actual_sha1_return, actual_sha1);
2739                        return buffer;
2740                }
2741                /* Handle references */
2742                else if (type == OBJ_COMMIT)
2743                        ref_type = "tree ";
2744                else if (type == OBJ_TAG)
2745                        ref_type = "object ";
2746                else {
2747                        free(buffer);
2748                        return NULL;
2749                }
2750                ref_length = strlen(ref_type);
2751
2752                if (ref_length + 40 > isize ||
2753                    memcmp(buffer, ref_type, ref_length) ||
2754                    get_sha1_hex((char *) buffer + ref_length, actual_sha1)) {
2755                        free(buffer);
2756                        return NULL;
2757                }
2758                free(buffer);
2759                /* Now we have the ID of the referred-to object in
2760                 * actual_sha1.  Check again. */
2761        }
2762}
2763
2764static void write_sha1_file_prepare(const void *buf, unsigned long len,
2765                                    const char *type, unsigned char *sha1,
2766                                    char *hdr, int *hdrlen)
2767{
2768        git_SHA_CTX c;
2769
2770        /* Generate the header */
2771        *hdrlen = sprintf(hdr, "%s %lu", type, len)+1;
2772
2773        /* Sha1.. */
2774        git_SHA1_Init(&c);
2775        git_SHA1_Update(&c, hdr, *hdrlen);
2776        git_SHA1_Update(&c, buf, len);
2777        git_SHA1_Final(sha1, &c);
2778}
2779
2780/*
2781 * Move the just written object into its final resting place.
2782 * NEEDSWORK: this should be renamed to finalize_temp_file() as
2783 * "moving" is only a part of what it does, when no patch between
2784 * master to pu changes the call sites of this function.
2785 */
2786int move_temp_to_file(const char *tmpfile, const char *filename)
2787{
2788        int ret = 0;
2789
2790        if (object_creation_mode == OBJECT_CREATION_USES_RENAMES)
2791                goto try_rename;
2792        else if (link(tmpfile, filename))
2793                ret = errno;
2794
2795        /*
2796         * Coda hack - coda doesn't like cross-directory links,
2797         * so we fall back to a rename, which will mean that it
2798         * won't be able to check collisions, but that's not a
2799         * big deal.
2800         *
2801         * The same holds for FAT formatted media.
2802         *
2803         * When this succeeds, we just return.  We have nothing
2804         * left to unlink.
2805         */
2806        if (ret && ret != EEXIST) {
2807        try_rename:
2808                if (!rename(tmpfile, filename))
2809                        goto out;
2810                ret = errno;
2811        }
2812        unlink_or_warn(tmpfile);
2813        if (ret) {
2814                if (ret != EEXIST) {
2815                        return error("unable to write sha1 filename %s: %s", filename, strerror(ret));
2816                }
2817                /* FIXME!!! Collision check here ? */
2818        }
2819
2820out:
2821        if (adjust_shared_perm(filename))
2822                return error("unable to set permission to '%s'", filename);
2823        return 0;
2824}
2825
2826static int write_buffer(int fd, const void *buf, size_t len)
2827{
2828        if (write_in_full(fd, buf, len) < 0)
2829                return error("file write error (%s)", strerror(errno));
2830        return 0;
2831}
2832
2833int hash_sha1_file(const void *buf, unsigned long len, const char *type,
2834                   unsigned char *sha1)
2835{
2836        char hdr[32];
2837        int hdrlen;
2838        write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
2839        return 0;
2840}
2841
2842/* Finalize a file on disk, and close it. */
2843static void close_sha1_file(int fd)
2844{
2845        if (fsync_object_files)
2846                fsync_or_die(fd, "sha1 file");
2847        if (close(fd) != 0)
2848                die_errno("error when closing sha1 file");
2849}
2850
2851/* Size of directory component, including the ending '/' */
2852static inline int directory_size(const char *filename)
2853{
2854        const char *s = strrchr(filename, '/');
2855        if (!s)
2856                return 0;
2857        return s - filename + 1;
2858}
2859
2860/*
2861 * This creates a temporary file in the same directory as the final
2862 * 'filename'
2863 *
2864 * We want to avoid cross-directory filename renames, because those
2865 * can have problems on various filesystems (FAT, NFS, Coda).
2866 */
2867static int create_tmpfile(char *buffer, size_t bufsiz, const char *filename)
2868{
2869        int fd, dirlen = directory_size(filename);
2870
2871        if (dirlen + 20 > bufsiz) {
2872                errno = ENAMETOOLONG;
2873                return -1;
2874        }
2875        memcpy(buffer, filename, dirlen);
2876        strcpy(buffer + dirlen, "tmp_obj_XXXXXX");
2877        fd = git_mkstemp_mode(buffer, 0444);
2878        if (fd < 0 && dirlen && errno == ENOENT) {
2879                /* Make sure the directory exists */
2880                memcpy(buffer, filename, dirlen);
2881                buffer[dirlen-1] = 0;
2882                if (mkdir(buffer, 0777) && errno != EEXIST)
2883                        return -1;
2884                if (adjust_shared_perm(buffer))
2885                        return -1;
2886
2887                /* Try again */
2888                strcpy(buffer + dirlen - 1, "/tmp_obj_XXXXXX");
2889                fd = git_mkstemp_mode(buffer, 0444);
2890        }
2891        return fd;
2892}
2893
2894static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
2895                              const void *buf, unsigned long len, time_t mtime)
2896{
2897        int fd, ret;
2898        unsigned char compressed[4096];
2899        git_zstream stream;
2900        git_SHA_CTX c;
2901        unsigned char parano_sha1[20];
2902        static char tmp_file[PATH_MAX];
2903        const char *filename = sha1_file_name(sha1);
2904
2905        fd = create_tmpfile(tmp_file, sizeof(tmp_file), filename);
2906        if (fd < 0) {
2907                if (errno == EACCES)
2908                        return error("insufficient permission for adding an object to repository database %s", get_object_directory());
2909                else
2910                        return error("unable to create temporary file: %s", strerror(errno));
2911        }
2912
2913        /* Set it up */
2914        memset(&stream, 0, sizeof(stream));
2915        git_deflate_init(&stream, zlib_compression_level);
2916        stream.next_out = compressed;
2917        stream.avail_out = sizeof(compressed);
2918        git_SHA1_Init(&c);
2919
2920        /* First header.. */
2921        stream.next_in = (unsigned char *)hdr;
2922        stream.avail_in = hdrlen;
2923        while (git_deflate(&stream, 0) == Z_OK)
2924                ; /* nothing */
2925        git_SHA1_Update(&c, hdr, hdrlen);
2926
2927        /* Then the data itself.. */
2928        stream.next_in = (void *)buf;
2929        stream.avail_in = len;
2930        do {
2931                unsigned char *in0 = stream.next_in;
2932                ret = git_deflate(&stream, Z_FINISH);
2933                git_SHA1_Update(&c, in0, stream.next_in - in0);
2934                if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
2935                        die("unable to write sha1 file");
2936                stream.next_out = compressed;
2937                stream.avail_out = sizeof(compressed);
2938        } while (ret == Z_OK);
2939
2940        if (ret != Z_STREAM_END)
2941                die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret);
2942        ret = git_deflate_end_gently(&stream);
2943        if (ret != Z_OK)
2944                die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret);
2945        git_SHA1_Final(parano_sha1, &c);
2946        if (hashcmp(sha1, parano_sha1) != 0)
2947                die("confused by unstable object source data for %s", sha1_to_hex(sha1));
2948
2949        close_sha1_file(fd);
2950
2951        if (mtime) {
2952                struct utimbuf utb;
2953                utb.actime = mtime;
2954                utb.modtime = mtime;
2955                if (utime(tmp_file, &utb) < 0)
2956                        warning("failed utime() on %s: %s",
2957                                tmp_file, strerror(errno));
2958        }
2959
2960        return move_temp_to_file(tmp_file, filename);
2961}
2962
2963int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
2964{
2965        unsigned char sha1[20];
2966        char hdr[32];
2967        int hdrlen;
2968
2969        /* Normally if we have it in the pack then we do not bother writing
2970         * it out into .git/objects/??/?{38} file.
2971         */
2972        write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
2973        if (returnsha1)
2974                hashcpy(returnsha1, sha1);
2975        if (has_sha1_file(sha1))
2976                return 0;
2977        return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
2978}
2979
2980int force_object_loose(const unsigned char *sha1, time_t mtime)
2981{
2982        void *buf;
2983        unsigned long len;
2984        enum object_type type;
2985        char hdr[32];
2986        int hdrlen;
2987        int ret;
2988
2989        if (has_loose_object(sha1))
2990                return 0;
2991        buf = read_packed_sha1(sha1, &type, &len);
2992        if (!buf)
2993                return error("cannot read sha1_file for %s", sha1_to_hex(sha1));
2994        hdrlen = sprintf(hdr, "%s %lu", typename(type), len) + 1;
2995        ret = write_loose_object(sha1, hdr, hdrlen, buf, len, mtime);
2996        free(buf);
2997
2998        return ret;
2999}
3000
3001int has_pack_index(const unsigned char *sha1)
3002{
3003        struct stat st;
3004        if (stat(sha1_pack_index_name(sha1), &st))
3005                return 0;
3006        return 1;
3007}
3008
3009int has_sha1_pack(const unsigned char *sha1)
3010{
3011        struct pack_entry e;
3012        return find_pack_entry(sha1, &e);
3013}
3014
3015int has_sha1_file(const unsigned char *sha1)
3016{
3017        struct pack_entry e;
3018
3019        if (find_pack_entry(sha1, &e))
3020                return 1;
3021        if (has_loose_object(sha1))
3022                return 1;
3023        reprepare_packed_git();
3024        return find_pack_entry(sha1, &e);
3025}
3026
3027static void check_tree(const void *buf, size_t size)
3028{
3029        struct tree_desc desc;
3030        struct name_entry entry;
3031
3032        init_tree_desc(&desc, buf, size);
3033        while (tree_entry(&desc, &entry))
3034                /* do nothing
3035                 * tree_entry() will die() on malformed entries */
3036                ;
3037}
3038
3039static void check_commit(const void *buf, size_t size)
3040{
3041        struct commit c;
3042        memset(&c, 0, sizeof(c));
3043        if (parse_commit_buffer(&c, buf, size))
3044                die("corrupt commit");
3045}
3046
3047static void check_tag(const void *buf, size_t size)
3048{
3049        struct tag t;
3050        memset(&t, 0, sizeof(t));
3051        if (parse_tag_buffer(&t, buf, size))
3052                die("corrupt tag");
3053}
3054
3055static int index_mem(unsigned char *sha1, void *buf, size_t size,
3056                     enum object_type type,
3057                     const char *path, unsigned flags)
3058{
3059        int ret, re_allocated = 0;
3060        int write_object = flags & HASH_WRITE_OBJECT;
3061
3062        if (!type)
3063                type = OBJ_BLOB;
3064
3065        /*
3066         * Convert blobs to git internal format
3067         */
3068        if ((type == OBJ_BLOB) && path) {
3069                struct strbuf nbuf = STRBUF_INIT;
3070                if (convert_to_git(path, buf, size, &nbuf,
3071                                   write_object ? safe_crlf : SAFE_CRLF_FALSE)) {
3072                        buf = strbuf_detach(&nbuf, &size);
3073                        re_allocated = 1;
3074                }
3075        }
3076        if (flags & HASH_FORMAT_CHECK) {
3077                if (type == OBJ_TREE)
3078                        check_tree(buf, size);
3079                if (type == OBJ_COMMIT)
3080                        check_commit(buf, size);
3081                if (type == OBJ_TAG)
3082                        check_tag(buf, size);
3083        }
3084
3085        if (write_object)
3086                ret = write_sha1_file(buf, size, typename(type), sha1);
3087        else
3088                ret = hash_sha1_file(buf, size, typename(type), sha1);
3089        if (re_allocated)
3090                free(buf);
3091        return ret;
3092}
3093
3094static int index_pipe(unsigned char *sha1, int fd, enum object_type type,
3095                      const char *path, unsigned flags)
3096{
3097        struct strbuf sbuf = STRBUF_INIT;
3098        int ret;
3099
3100        if (strbuf_read(&sbuf, fd, 4096) >= 0)
3101                ret = index_mem(sha1, sbuf.buf, sbuf.len, type, path, flags);
3102        else
3103                ret = -1;
3104        strbuf_release(&sbuf);
3105        return ret;
3106}
3107
3108#define SMALL_FILE_SIZE (32*1024)
3109
3110static int index_core(unsigned char *sha1, int fd, size_t size,
3111                      enum object_type type, const char *path,
3112                      unsigned flags)
3113{
3114        int ret;
3115
3116        if (!size) {
3117                ret = index_mem(sha1, NULL, size, type, path, flags);
3118        } else if (size <= SMALL_FILE_SIZE) {
3119                char *buf = xmalloc(size);
3120                if (size == read_in_full(fd, buf, size))
3121                        ret = index_mem(sha1, buf, size, type, path, flags);
3122                else
3123                        ret = error("short read %s", strerror(errno));
3124                free(buf);
3125        } else {
3126                void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
3127                ret = index_mem(sha1, buf, size, type, path, flags);
3128                munmap(buf, size);
3129        }
3130        return ret;
3131}
3132
3133/*
3134 * This creates one packfile per large blob unless bulk-checkin
3135 * machinery is "plugged".
3136 *
3137 * This also bypasses the usual "convert-to-git" dance, and that is on
3138 * purpose. We could write a streaming version of the converting
3139 * functions and insert that before feeding the data to fast-import
3140 * (or equivalent in-core API described above). However, that is
3141 * somewhat complicated, as we do not know the size of the filter
3142 * result, which we need to know beforehand when writing a git object.
3143 * Since the primary motivation for trying to stream from the working
3144 * tree file and to avoid mmaping it in core is to deal with large
3145 * binary blobs, they generally do not want to get any conversion, and
3146 * callers should avoid this code path when filters are requested.
3147 */
3148static int index_stream(unsigned char *sha1, int fd, size_t size,
3149                        enum object_type type, const char *path,
3150                        unsigned flags)
3151{
3152        return index_bulk_checkin(sha1, fd, size, type, path, flags);
3153}
3154
3155int index_fd(unsigned char *sha1, int fd, struct stat *st,
3156             enum object_type type, const char *path, unsigned flags)
3157{
3158        int ret;
3159        size_t size = xsize_t(st->st_size);
3160
3161        if (!S_ISREG(st->st_mode))
3162                ret = index_pipe(sha1, fd, type, path, flags);
3163        else if (size <= big_file_threshold || type != OBJ_BLOB ||
3164                 (path && would_convert_to_git(path, NULL, 0, 0)))
3165                ret = index_core(sha1, fd, size, type, path, flags);
3166        else
3167                ret = index_stream(sha1, fd, size, type, path, flags);
3168        close(fd);
3169        return ret;
3170}
3171
3172int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags)
3173{
3174        int fd;
3175        struct strbuf sb = STRBUF_INIT;
3176
3177        switch (st->st_mode & S_IFMT) {
3178        case S_IFREG:
3179                fd = open(path, O_RDONLY);
3180                if (fd < 0)
3181                        return error("open(\"%s\"): %s", path,
3182                                     strerror(errno));
3183                if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0)
3184                        return error("%s: failed to insert into database",
3185                                     path);
3186                break;
3187        case S_IFLNK:
3188                if (strbuf_readlink(&sb, path, st->st_size)) {
3189                        char *errstr = strerror(errno);
3190                        return error("readlink(\"%s\"): %s", path,
3191                                     errstr);
3192                }
3193                if (!(flags & HASH_WRITE_OBJECT))
3194                        hash_sha1_file(sb.buf, sb.len, blob_type, sha1);
3195                else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1))
3196                        return error("%s: failed to insert into database",
3197                                     path);
3198                strbuf_release(&sb);
3199                break;
3200        case S_IFDIR:
3201                return resolve_gitlink_ref(path, "HEAD", sha1);
3202        default:
3203                return error("%s: unsupported file type", path);
3204        }
3205        return 0;
3206}
3207
3208int read_pack_header(int fd, struct pack_header *header)
3209{
3210        if (read_in_full(fd, header, sizeof(*header)) < sizeof(*header))
3211                /* "eof before pack header was fully read" */
3212                return PH_ERROR_EOF;
3213
3214        if (header->hdr_signature != htonl(PACK_SIGNATURE))
3215                /* "protocol error (pack signature mismatch detected)" */
3216                return PH_ERROR_PACK_SIGNATURE;
3217        if (!pack_version_ok(header->hdr_version))
3218                /* "protocol error (pack version unsupported)" */
3219                return PH_ERROR_PROTOCOL;
3220        return 0;
3221}
3222
3223void assert_sha1_type(const unsigned char *sha1, enum object_type expect)
3224{
3225        enum object_type type = sha1_object_info(sha1, NULL);
3226        if (type < 0)
3227                die("%s is not a valid object", sha1_to_hex(sha1));
3228        if (type != expect)
3229                die("%s is not a valid '%s' object", sha1_to_hex(sha1),
3230                    typename(expect));
3231}