95186c019a1572cba6a428392c6256b3fd9bf088
   1/*
   2 * GIT - The information manager from hell
   3 *
   4 * Copyright (C) Linus Torvalds, 2005
   5 *
   6 * This handles basic git sha1 object files - packing, unpacking,
   7 * creation etc.
   8 */
   9#include "cache.h"
  10#include "config.h"
  11#include "string-list.h"
  12#include "lockfile.h"
  13#include "delta.h"
  14#include "pack.h"
  15#include "blob.h"
  16#include "commit.h"
  17#include "run-command.h"
  18#include "tag.h"
  19#include "tree.h"
  20#include "tree-walk.h"
  21#include "refs.h"
  22#include "pack-revindex.h"
  23#include "sha1-lookup.h"
  24#include "bulk-checkin.h"
  25#include "repository.h"
  26#include "replace-object.h"
  27#include "streaming.h"
  28#include "dir.h"
  29#include "list.h"
  30#include "mergesort.h"
  31#include "quote.h"
  32#include "packfile.h"
  33#include "fetch-object.h"
  34#include "object-store.h"
  35
  36/* The maximum size for an object header. */
  37#define MAX_HEADER_LEN 32
  38
  39
  40#define EMPTY_TREE_SHA1_BIN_LITERAL \
  41         "\x4b\x82\x5d\xc6\x42\xcb\x6e\xb9\xa0\x60" \
  42         "\xe5\x4b\xf8\xd6\x92\x88\xfb\xee\x49\x04"
  43
  44#define EMPTY_BLOB_SHA1_BIN_LITERAL \
  45        "\xe6\x9d\xe2\x9b\xb2\xd1\xd6\x43\x4b\x8b" \
  46        "\x29\xae\x77\x5a\xd8\xc2\xe4\x8c\x53\x91"
  47
  48const unsigned char null_sha1[GIT_MAX_RAWSZ];
  49const struct object_id null_oid;
  50static const struct object_id empty_tree_oid = {
  51        EMPTY_TREE_SHA1_BIN_LITERAL
  52};
  53static const struct object_id empty_blob_oid = {
  54        EMPTY_BLOB_SHA1_BIN_LITERAL
  55};
  56
  57static void git_hash_sha1_init(git_hash_ctx *ctx)
  58{
  59        git_SHA1_Init(&ctx->sha1);
  60}
  61
  62static void git_hash_sha1_update(git_hash_ctx *ctx, const void *data, size_t len)
  63{
  64        git_SHA1_Update(&ctx->sha1, data, len);
  65}
  66
  67static void git_hash_sha1_final(unsigned char *hash, git_hash_ctx *ctx)
  68{
  69        git_SHA1_Final(hash, &ctx->sha1);
  70}
  71
  72static void git_hash_unknown_init(git_hash_ctx *ctx)
  73{
  74        BUG("trying to init unknown hash");
  75}
  76
  77static void git_hash_unknown_update(git_hash_ctx *ctx, const void *data, size_t len)
  78{
  79        BUG("trying to update unknown hash");
  80}
  81
  82static void git_hash_unknown_final(unsigned char *hash, git_hash_ctx *ctx)
  83{
  84        BUG("trying to finalize unknown hash");
  85}
  86
  87const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = {
  88        {
  89                NULL,
  90                0x00000000,
  91                0,
  92                0,
  93                git_hash_unknown_init,
  94                git_hash_unknown_update,
  95                git_hash_unknown_final,
  96                NULL,
  97                NULL,
  98        },
  99        {
 100                "sha-1",
 101                /* "sha1", big-endian */
 102                0x73686131,
 103                GIT_SHA1_RAWSZ,
 104                GIT_SHA1_HEXSZ,
 105                git_hash_sha1_init,
 106                git_hash_sha1_update,
 107                git_hash_sha1_final,
 108                &empty_tree_oid,
 109                &empty_blob_oid,
 110        },
 111};
 112
 113const char *empty_tree_oid_hex(void)
 114{
 115        static char buf[GIT_MAX_HEXSZ + 1];
 116        return oid_to_hex_r(buf, the_hash_algo->empty_tree);
 117}
 118
 119const char *empty_blob_oid_hex(void)
 120{
 121        static char buf[GIT_MAX_HEXSZ + 1];
 122        return oid_to_hex_r(buf, the_hash_algo->empty_blob);
 123}
 124
 125/*
 126 * This is meant to hold a *small* number of objects that you would
 127 * want read_object_file() to be able to return, but yet you do not want
 128 * to write them into the object store (e.g. a browse-only
 129 * application).
 130 */
 131static struct cached_object {
 132        struct object_id oid;
 133        enum object_type type;
 134        void *buf;
 135        unsigned long size;
 136} *cached_objects;
 137static int cached_object_nr, cached_object_alloc;
 138
 139static struct cached_object empty_tree = {
 140        { EMPTY_TREE_SHA1_BIN_LITERAL },
 141        OBJ_TREE,
 142        "",
 143        0
 144};
 145
 146static struct cached_object *find_cached_object(const struct object_id *oid)
 147{
 148        int i;
 149        struct cached_object *co = cached_objects;
 150
 151        for (i = 0; i < cached_object_nr; i++, co++) {
 152                if (oideq(&co->oid, oid))
 153                        return co;
 154        }
 155        if (oideq(oid, the_hash_algo->empty_tree))
 156                return &empty_tree;
 157        return NULL;
 158}
 159
 160
 161static int get_conv_flags(unsigned flags)
 162{
 163        if (flags & HASH_RENORMALIZE)
 164                return CONV_EOL_RENORMALIZE;
 165        else if (flags & HASH_WRITE_OBJECT)
 166                return global_conv_flags_eol | CONV_WRITE_OBJECT;
 167        else
 168                return 0;
 169}
 170
 171
 172int mkdir_in_gitdir(const char *path)
 173{
 174        if (mkdir(path, 0777)) {
 175                int saved_errno = errno;
 176                struct stat st;
 177                struct strbuf sb = STRBUF_INIT;
 178
 179                if (errno != EEXIST)
 180                        return -1;
 181                /*
 182                 * Are we looking at a path in a symlinked worktree
 183                 * whose original repository does not yet have it?
 184                 * e.g. .git/rr-cache pointing at its original
 185                 * repository in which the user hasn't performed any
 186                 * conflict resolution yet?
 187                 */
 188                if (lstat(path, &st) || !S_ISLNK(st.st_mode) ||
 189                    strbuf_readlink(&sb, path, st.st_size) ||
 190                    !is_absolute_path(sb.buf) ||
 191                    mkdir(sb.buf, 0777)) {
 192                        strbuf_release(&sb);
 193                        errno = saved_errno;
 194                        return -1;
 195                }
 196                strbuf_release(&sb);
 197        }
 198        return adjust_shared_perm(path);
 199}
 200
 201enum scld_error safe_create_leading_directories(char *path)
 202{
 203        char *next_component = path + offset_1st_component(path);
 204        enum scld_error ret = SCLD_OK;
 205
 206        while (ret == SCLD_OK && next_component) {
 207                struct stat st;
 208                char *slash = next_component, slash_character;
 209
 210                while (*slash && !is_dir_sep(*slash))
 211                        slash++;
 212
 213                if (!*slash)
 214                        break;
 215
 216                next_component = slash + 1;
 217                while (is_dir_sep(*next_component))
 218                        next_component++;
 219                if (!*next_component)
 220                        break;
 221
 222                slash_character = *slash;
 223                *slash = '\0';
 224                if (!stat(path, &st)) {
 225                        /* path exists */
 226                        if (!S_ISDIR(st.st_mode)) {
 227                                errno = ENOTDIR;
 228                                ret = SCLD_EXISTS;
 229                        }
 230                } else if (mkdir(path, 0777)) {
 231                        if (errno == EEXIST &&
 232                            !stat(path, &st) && S_ISDIR(st.st_mode))
 233                                ; /* somebody created it since we checked */
 234                        else if (errno == ENOENT)
 235                                /*
 236                                 * Either mkdir() failed because
 237                                 * somebody just pruned the containing
 238                                 * directory, or stat() failed because
 239                                 * the file that was in our way was
 240                                 * just removed.  Either way, inform
 241                                 * the caller that it might be worth
 242                                 * trying again:
 243                                 */
 244                                ret = SCLD_VANISHED;
 245                        else
 246                                ret = SCLD_FAILED;
 247                } else if (adjust_shared_perm(path)) {
 248                        ret = SCLD_PERMS;
 249                }
 250                *slash = slash_character;
 251        }
 252        return ret;
 253}
 254
 255enum scld_error safe_create_leading_directories_const(const char *path)
 256{
 257        int save_errno;
 258        /* path points to cache entries, so xstrdup before messing with it */
 259        char *buf = xstrdup(path);
 260        enum scld_error result = safe_create_leading_directories(buf);
 261
 262        save_errno = errno;
 263        free(buf);
 264        errno = save_errno;
 265        return result;
 266}
 267
 268int raceproof_create_file(const char *path, create_file_fn fn, void *cb)
 269{
 270        /*
 271         * The number of times we will try to remove empty directories
 272         * in the way of path. This is only 1 because if another
 273         * process is racily creating directories that conflict with
 274         * us, we don't want to fight against them.
 275         */
 276        int remove_directories_remaining = 1;
 277
 278        /*
 279         * The number of times that we will try to create the
 280         * directories containing path. We are willing to attempt this
 281         * more than once, because another process could be trying to
 282         * clean up empty directories at the same time as we are
 283         * trying to create them.
 284         */
 285        int create_directories_remaining = 3;
 286
 287        /* A scratch copy of path, filled lazily if we need it: */
 288        struct strbuf path_copy = STRBUF_INIT;
 289
 290        int ret, save_errno;
 291
 292        /* Sanity check: */
 293        assert(*path);
 294
 295retry_fn:
 296        ret = fn(path, cb);
 297        save_errno = errno;
 298        if (!ret)
 299                goto out;
 300
 301        if (errno == EISDIR && remove_directories_remaining-- > 0) {
 302                /*
 303                 * A directory is in the way. Maybe it is empty; try
 304                 * to remove it:
 305                 */
 306                if (!path_copy.len)
 307                        strbuf_addstr(&path_copy, path);
 308
 309                if (!remove_dir_recursively(&path_copy, REMOVE_DIR_EMPTY_ONLY))
 310                        goto retry_fn;
 311        } else if (errno == ENOENT && create_directories_remaining-- > 0) {
 312                /*
 313                 * Maybe the containing directory didn't exist, or
 314                 * maybe it was just deleted by a process that is
 315                 * racing with us to clean up empty directories. Try
 316                 * to create it:
 317                 */
 318                enum scld_error scld_result;
 319
 320                if (!path_copy.len)
 321                        strbuf_addstr(&path_copy, path);
 322
 323                do {
 324                        scld_result = safe_create_leading_directories(path_copy.buf);
 325                        if (scld_result == SCLD_OK)
 326                                goto retry_fn;
 327                } while (scld_result == SCLD_VANISHED && create_directories_remaining-- > 0);
 328        }
 329
 330out:
 331        strbuf_release(&path_copy);
 332        errno = save_errno;
 333        return ret;
 334}
 335
 336static void fill_loose_path(struct strbuf *buf, const struct object_id *oid)
 337{
 338        int i;
 339        for (i = 0; i < the_hash_algo->rawsz; i++) {
 340                static char hex[] = "0123456789abcdef";
 341                unsigned int val = oid->hash[i];
 342                strbuf_addch(buf, hex[val >> 4]);
 343                strbuf_addch(buf, hex[val & 0xf]);
 344                if (!i)
 345                        strbuf_addch(buf, '/');
 346        }
 347}
 348
 349static const char *odb_loose_path(struct object_directory *odb,
 350                                  struct strbuf *buf,
 351                                  const struct object_id *oid)
 352{
 353        strbuf_reset(buf);
 354        strbuf_addstr(buf, odb->path);
 355        strbuf_addch(buf, '/');
 356        fill_loose_path(buf, oid);
 357        return buf->buf;
 358}
 359
 360const char *loose_object_path(struct repository *r, struct strbuf *buf,
 361                              const struct object_id *oid)
 362{
 363        return odb_loose_path(r->objects->odb, buf, oid);
 364}
 365
 366/*
 367 * Return non-zero iff the path is usable as an alternate object database.
 368 */
 369static int alt_odb_usable(struct raw_object_store *o,
 370                          struct strbuf *path,
 371                          const char *normalized_objdir)
 372{
 373        struct object_directory *odb;
 374
 375        /* Detect cases where alternate disappeared */
 376        if (!is_directory(path->buf)) {
 377                error(_("object directory %s does not exist; "
 378                        "check .git/objects/info/alternates"),
 379                      path->buf);
 380                return 0;
 381        }
 382
 383        /*
 384         * Prevent the common mistake of listing the same
 385         * thing twice, or object directory itself.
 386         */
 387        for (odb = o->odb; odb; odb = odb->next) {
 388                if (!fspathcmp(path->buf, odb->path))
 389                        return 0;
 390        }
 391        if (!fspathcmp(path->buf, normalized_objdir))
 392                return 0;
 393
 394        return 1;
 395}
 396
 397/*
 398 * Prepare alternate object database registry.
 399 *
 400 * The variable alt_odb_list points at the list of struct
 401 * object_directory.  The elements on this list come from
 402 * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
 403 * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
 404 * whose contents is similar to that environment variable but can be
 405 * LF separated.  Its base points at a statically allocated buffer that
 406 * contains "/the/directory/corresponding/to/.git/objects/...", while
 407 * its name points just after the slash at the end of ".git/objects/"
 408 * in the example above, and has enough space to hold 40-byte hex
 409 * SHA1, an extra slash for the first level indirection, and the
 410 * terminating NUL.
 411 */
 412static void read_info_alternates(struct repository *r,
 413                                 const char *relative_base,
 414                                 int depth);
 415static int link_alt_odb_entry(struct repository *r, const char *entry,
 416        const char *relative_base, int depth, const char *normalized_objdir)
 417{
 418        struct object_directory *ent;
 419        struct strbuf pathbuf = STRBUF_INIT;
 420
 421        if (!is_absolute_path(entry) && relative_base) {
 422                strbuf_realpath(&pathbuf, relative_base, 1);
 423                strbuf_addch(&pathbuf, '/');
 424        }
 425        strbuf_addstr(&pathbuf, entry);
 426
 427        if (strbuf_normalize_path(&pathbuf) < 0 && relative_base) {
 428                error(_("unable to normalize alternate object path: %s"),
 429                      pathbuf.buf);
 430                strbuf_release(&pathbuf);
 431                return -1;
 432        }
 433
 434        /*
 435         * The trailing slash after the directory name is given by
 436         * this function at the end. Remove duplicates.
 437         */
 438        while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/')
 439                strbuf_setlen(&pathbuf, pathbuf.len - 1);
 440
 441        if (!alt_odb_usable(r->objects, &pathbuf, normalized_objdir)) {
 442                strbuf_release(&pathbuf);
 443                return -1;
 444        }
 445
 446        ent = xcalloc(1, sizeof(*ent));
 447        ent->path = xstrdup(pathbuf.buf);
 448
 449        /* add the alternate entry */
 450        *r->objects->odb_tail = ent;
 451        r->objects->odb_tail = &(ent->next);
 452        ent->next = NULL;
 453
 454        /* recursively add alternates */
 455        read_info_alternates(r, pathbuf.buf, depth + 1);
 456
 457        strbuf_release(&pathbuf);
 458        return 0;
 459}
 460
 461static const char *parse_alt_odb_entry(const char *string,
 462                                       int sep,
 463                                       struct strbuf *out)
 464{
 465        const char *end;
 466
 467        strbuf_reset(out);
 468
 469        if (*string == '#') {
 470                /* comment; consume up to next separator */
 471                end = strchrnul(string, sep);
 472        } else if (*string == '"' && !unquote_c_style(out, string, &end)) {
 473                /*
 474                 * quoted path; unquote_c_style has copied the
 475                 * data for us and set "end". Broken quoting (e.g.,
 476                 * an entry that doesn't end with a quote) falls
 477                 * back to the unquoted case below.
 478                 */
 479        } else {
 480                /* normal, unquoted path */
 481                end = strchrnul(string, sep);
 482                strbuf_add(out, string, end - string);
 483        }
 484
 485        if (*end)
 486                end++;
 487        return end;
 488}
 489
 490static void link_alt_odb_entries(struct repository *r, const char *alt,
 491                                 int sep, const char *relative_base, int depth)
 492{
 493        struct strbuf objdirbuf = STRBUF_INIT;
 494        struct strbuf entry = STRBUF_INIT;
 495
 496        if (!alt || !*alt)
 497                return;
 498
 499        if (depth > 5) {
 500                error(_("%s: ignoring alternate object stores, nesting too deep"),
 501                                relative_base);
 502                return;
 503        }
 504
 505        strbuf_add_absolute_path(&objdirbuf, r->objects->odb->path);
 506        if (strbuf_normalize_path(&objdirbuf) < 0)
 507                die(_("unable to normalize object directory: %s"),
 508                    objdirbuf.buf);
 509
 510        while (*alt) {
 511                alt = parse_alt_odb_entry(alt, sep, &entry);
 512                if (!entry.len)
 513                        continue;
 514                link_alt_odb_entry(r, entry.buf,
 515                                   relative_base, depth, objdirbuf.buf);
 516        }
 517        strbuf_release(&entry);
 518        strbuf_release(&objdirbuf);
 519}
 520
 521static void read_info_alternates(struct repository *r,
 522                                 const char *relative_base,
 523                                 int depth)
 524{
 525        char *path;
 526        struct strbuf buf = STRBUF_INIT;
 527
 528        path = xstrfmt("%s/info/alternates", relative_base);
 529        if (strbuf_read_file(&buf, path, 1024) < 0) {
 530                warn_on_fopen_errors(path);
 531                free(path);
 532                return;
 533        }
 534
 535        link_alt_odb_entries(r, buf.buf, '\n', relative_base, depth);
 536        strbuf_release(&buf);
 537        free(path);
 538}
 539
 540void add_to_alternates_file(const char *reference)
 541{
 542        struct lock_file lock = LOCK_INIT;
 543        char *alts = git_pathdup("objects/info/alternates");
 544        FILE *in, *out;
 545        int found = 0;
 546
 547        hold_lock_file_for_update(&lock, alts, LOCK_DIE_ON_ERROR);
 548        out = fdopen_lock_file(&lock, "w");
 549        if (!out)
 550                die_errno(_("unable to fdopen alternates lockfile"));
 551
 552        in = fopen(alts, "r");
 553        if (in) {
 554                struct strbuf line = STRBUF_INIT;
 555
 556                while (strbuf_getline(&line, in) != EOF) {
 557                        if (!strcmp(reference, line.buf)) {
 558                                found = 1;
 559                                break;
 560                        }
 561                        fprintf_or_die(out, "%s\n", line.buf);
 562                }
 563
 564                strbuf_release(&line);
 565                fclose(in);
 566        }
 567        else if (errno != ENOENT)
 568                die_errno(_("unable to read alternates file"));
 569
 570        if (found) {
 571                rollback_lock_file(&lock);
 572        } else {
 573                fprintf_or_die(out, "%s\n", reference);
 574                if (commit_lock_file(&lock))
 575                        die_errno(_("unable to move new alternates file into place"));
 576                if (the_repository->objects->loaded_alternates)
 577                        link_alt_odb_entries(the_repository, reference,
 578                                             '\n', NULL, 0);
 579        }
 580        free(alts);
 581}
 582
 583void add_to_alternates_memory(const char *reference)
 584{
 585        /*
 586         * Make sure alternates are initialized, or else our entry may be
 587         * overwritten when they are.
 588         */
 589        prepare_alt_odb(the_repository);
 590
 591        link_alt_odb_entries(the_repository, reference,
 592                             '\n', NULL, 0);
 593}
 594
 595/*
 596 * Compute the exact path an alternate is at and returns it. In case of
 597 * error NULL is returned and the human readable error is added to `err`
 598 * `path` may be relative and should point to $GIT_DIR.
 599 * `err` must not be null.
 600 */
 601char *compute_alternate_path(const char *path, struct strbuf *err)
 602{
 603        char *ref_git = NULL;
 604        const char *repo, *ref_git_s;
 605        int seen_error = 0;
 606
 607        ref_git_s = real_path_if_valid(path);
 608        if (!ref_git_s) {
 609                seen_error = 1;
 610                strbuf_addf(err, _("path '%s' does not exist"), path);
 611                goto out;
 612        } else
 613                /*
 614                 * Beware: read_gitfile(), real_path() and mkpath()
 615                 * return static buffer
 616                 */
 617                ref_git = xstrdup(ref_git_s);
 618
 619        repo = read_gitfile(ref_git);
 620        if (!repo)
 621                repo = read_gitfile(mkpath("%s/.git", ref_git));
 622        if (repo) {
 623                free(ref_git);
 624                ref_git = xstrdup(repo);
 625        }
 626
 627        if (!repo && is_directory(mkpath("%s/.git/objects", ref_git))) {
 628                char *ref_git_git = mkpathdup("%s/.git", ref_git);
 629                free(ref_git);
 630                ref_git = ref_git_git;
 631        } else if (!is_directory(mkpath("%s/objects", ref_git))) {
 632                struct strbuf sb = STRBUF_INIT;
 633                seen_error = 1;
 634                if (get_common_dir(&sb, ref_git)) {
 635                        strbuf_addf(err,
 636                                    _("reference repository '%s' as a linked "
 637                                      "checkout is not supported yet."),
 638                                    path);
 639                        goto out;
 640                }
 641
 642                strbuf_addf(err, _("reference repository '%s' is not a "
 643                                        "local repository."), path);
 644                goto out;
 645        }
 646
 647        if (!access(mkpath("%s/shallow", ref_git), F_OK)) {
 648                strbuf_addf(err, _("reference repository '%s' is shallow"),
 649                            path);
 650                seen_error = 1;
 651                goto out;
 652        }
 653
 654        if (!access(mkpath("%s/info/grafts", ref_git), F_OK)) {
 655                strbuf_addf(err,
 656                            _("reference repository '%s' is grafted"),
 657                            path);
 658                seen_error = 1;
 659                goto out;
 660        }
 661
 662out:
 663        if (seen_error) {
 664                FREE_AND_NULL(ref_git);
 665        }
 666
 667        return ref_git;
 668}
 669
 670int foreach_alt_odb(alt_odb_fn fn, void *cb)
 671{
 672        struct object_directory *ent;
 673        int r = 0;
 674
 675        prepare_alt_odb(the_repository);
 676        for (ent = the_repository->objects->odb->next; ent; ent = ent->next) {
 677                r = fn(ent, cb);
 678                if (r)
 679                        break;
 680        }
 681        return r;
 682}
 683
 684void prepare_alt_odb(struct repository *r)
 685{
 686        if (r->objects->loaded_alternates)
 687                return;
 688
 689        link_alt_odb_entries(r, r->objects->alternate_db, PATH_SEP, NULL, 0);
 690
 691        read_info_alternates(r, r->objects->odb->path, 0);
 692        r->objects->loaded_alternates = 1;
 693}
 694
 695/* Returns 1 if we have successfully freshened the file, 0 otherwise. */
 696static int freshen_file(const char *fn)
 697{
 698        struct utimbuf t;
 699        t.actime = t.modtime = time(NULL);
 700        return !utime(fn, &t);
 701}
 702
 703/*
 704 * All of the check_and_freshen functions return 1 if the file exists and was
 705 * freshened (if freshening was requested), 0 otherwise. If they return
 706 * 0, you should not assume that it is safe to skip a write of the object (it
 707 * either does not exist on disk, or has a stale mtime and may be subject to
 708 * pruning).
 709 */
 710int check_and_freshen_file(const char *fn, int freshen)
 711{
 712        if (access(fn, F_OK))
 713                return 0;
 714        if (freshen && !freshen_file(fn))
 715                return 0;
 716        return 1;
 717}
 718
 719static int check_and_freshen_odb(struct object_directory *odb,
 720                                 const struct object_id *oid,
 721                                 int freshen)
 722{
 723        static struct strbuf path = STRBUF_INIT;
 724        odb_loose_path(odb, &path, oid);
 725        return check_and_freshen_file(path.buf, freshen);
 726}
 727
 728static int check_and_freshen_local(const struct object_id *oid, int freshen)
 729{
 730        return check_and_freshen_odb(the_repository->objects->odb, oid, freshen);
 731}
 732
 733static int check_and_freshen_nonlocal(const struct object_id *oid, int freshen)
 734{
 735        struct object_directory *odb;
 736
 737        prepare_alt_odb(the_repository);
 738        for (odb = the_repository->objects->odb->next; odb; odb = odb->next) {
 739                if (check_and_freshen_odb(odb, oid, freshen))
 740                        return 1;
 741        }
 742        return 0;
 743}
 744
 745static int check_and_freshen(const struct object_id *oid, int freshen)
 746{
 747        return check_and_freshen_local(oid, freshen) ||
 748               check_and_freshen_nonlocal(oid, freshen);
 749}
 750
 751int has_loose_object_nonlocal(const struct object_id *oid)
 752{
 753        return check_and_freshen_nonlocal(oid, 0);
 754}
 755
 756static int has_loose_object(const struct object_id *oid)
 757{
 758        return check_and_freshen(oid, 0);
 759}
 760
 761static void mmap_limit_check(size_t length)
 762{
 763        static size_t limit = 0;
 764        if (!limit) {
 765                limit = git_env_ulong("GIT_MMAP_LIMIT", 0);
 766                if (!limit)
 767                        limit = SIZE_MAX;
 768        }
 769        if (length > limit)
 770                die(_("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX),
 771                    (uintmax_t)length, (uintmax_t)limit);
 772}
 773
 774void *xmmap_gently(void *start, size_t length,
 775                  int prot, int flags, int fd, off_t offset)
 776{
 777        void *ret;
 778
 779        mmap_limit_check(length);
 780        ret = mmap(start, length, prot, flags, fd, offset);
 781        if (ret == MAP_FAILED) {
 782                if (!length)
 783                        return NULL;
 784                release_pack_memory(length);
 785                ret = mmap(start, length, prot, flags, fd, offset);
 786        }
 787        return ret;
 788}
 789
 790void *xmmap(void *start, size_t length,
 791        int prot, int flags, int fd, off_t offset)
 792{
 793        void *ret = xmmap_gently(start, length, prot, flags, fd, offset);
 794        if (ret == MAP_FAILED)
 795                die_errno(_("mmap failed"));
 796        return ret;
 797}
 798
 799/*
 800 * With an in-core object data in "map", rehash it to make sure the
 801 * object name actually matches "oid" to detect object corruption.
 802 * With "map" == NULL, try reading the object named with "oid" using
 803 * the streaming interface and rehash it to do the same.
 804 */
 805int check_object_signature(const struct object_id *oid, void *map,
 806                           unsigned long size, const char *type)
 807{
 808        struct object_id real_oid;
 809        enum object_type obj_type;
 810        struct git_istream *st;
 811        git_hash_ctx c;
 812        char hdr[MAX_HEADER_LEN];
 813        int hdrlen;
 814
 815        if (map) {
 816                hash_object_file(map, size, type, &real_oid);
 817                return !oideq(oid, &real_oid) ? -1 : 0;
 818        }
 819
 820        st = open_istream(oid, &obj_type, &size, NULL);
 821        if (!st)
 822                return -1;
 823
 824        /* Generate the header */
 825        hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(obj_type), (uintmax_t)size) + 1;
 826
 827        /* Sha1.. */
 828        the_hash_algo->init_fn(&c);
 829        the_hash_algo->update_fn(&c, hdr, hdrlen);
 830        for (;;) {
 831                char buf[1024 * 16];
 832                ssize_t readlen = read_istream(st, buf, sizeof(buf));
 833
 834                if (readlen < 0) {
 835                        close_istream(st);
 836                        return -1;
 837                }
 838                if (!readlen)
 839                        break;
 840                the_hash_algo->update_fn(&c, buf, readlen);
 841        }
 842        the_hash_algo->final_fn(real_oid.hash, &c);
 843        close_istream(st);
 844        return !oideq(oid, &real_oid) ? -1 : 0;
 845}
 846
 847int git_open_cloexec(const char *name, int flags)
 848{
 849        int fd;
 850        static int o_cloexec = O_CLOEXEC;
 851
 852        fd = open(name, flags | o_cloexec);
 853        if ((o_cloexec & O_CLOEXEC) && fd < 0 && errno == EINVAL) {
 854                /* Try again w/o O_CLOEXEC: the kernel might not support it */
 855                o_cloexec &= ~O_CLOEXEC;
 856                fd = open(name, flags | o_cloexec);
 857        }
 858
 859#if defined(F_GETFD) && defined(F_SETFD) && defined(FD_CLOEXEC)
 860        {
 861                static int fd_cloexec = FD_CLOEXEC;
 862
 863                if (!o_cloexec && 0 <= fd && fd_cloexec) {
 864                        /* Opened w/o O_CLOEXEC?  try with fcntl(2) to add it */
 865                        int flags = fcntl(fd, F_GETFD);
 866                        if (fcntl(fd, F_SETFD, flags | fd_cloexec))
 867                                fd_cloexec = 0;
 868                }
 869        }
 870#endif
 871        return fd;
 872}
 873
 874/*
 875 * Find "oid" as a loose object in the local repository or in an alternate.
 876 * Returns 0 on success, negative on failure.
 877 *
 878 * The "path" out-parameter will give the path of the object we found (if any).
 879 * Note that it may point to static storage and is only valid until another
 880 * call to stat_loose_object().
 881 */
 882static int stat_loose_object(struct repository *r, const struct object_id *oid,
 883                             struct stat *st, const char **path)
 884{
 885        struct object_directory *odb;
 886        static struct strbuf buf = STRBUF_INIT;
 887
 888        prepare_alt_odb(r);
 889        for (odb = r->objects->odb; odb; odb = odb->next) {
 890                *path = odb_loose_path(odb, &buf, oid);
 891                if (!lstat(*path, st))
 892                        return 0;
 893        }
 894
 895        return -1;
 896}
 897
 898/*
 899 * Like stat_loose_object(), but actually open the object and return the
 900 * descriptor. See the caveats on the "path" parameter above.
 901 */
 902static int open_loose_object(struct repository *r,
 903                             const struct object_id *oid, const char **path)
 904{
 905        int fd;
 906        struct object_directory *odb;
 907        int most_interesting_errno = ENOENT;
 908        static struct strbuf buf = STRBUF_INIT;
 909
 910        prepare_alt_odb(r);
 911        for (odb = r->objects->odb; odb; odb = odb->next) {
 912                *path = odb_loose_path(odb, &buf, oid);
 913                fd = git_open(*path);
 914                if (fd >= 0)
 915                        return fd;
 916
 917                if (most_interesting_errno == ENOENT)
 918                        most_interesting_errno = errno;
 919        }
 920        errno = most_interesting_errno;
 921        return -1;
 922}
 923
 924static int quick_has_loose(struct repository *r,
 925                           const struct object_id *oid)
 926{
 927        struct object_directory *odb;
 928
 929        prepare_alt_odb(r);
 930        for (odb = r->objects->odb; odb; odb = odb->next) {
 931                if (oid_array_lookup(odb_loose_cache(odb, oid), oid) >= 0)
 932                        return 1;
 933        }
 934        return 0;
 935}
 936
 937/*
 938 * Map the loose object at "path" if it is not NULL, or the path found by
 939 * searching for a loose object named "oid".
 940 */
 941static void *map_loose_object_1(struct repository *r, const char *path,
 942                             const struct object_id *oid, unsigned long *size)
 943{
 944        void *map;
 945        int fd;
 946
 947        if (path)
 948                fd = git_open(path);
 949        else
 950                fd = open_loose_object(r, oid, &path);
 951        map = NULL;
 952        if (fd >= 0) {
 953                struct stat st;
 954
 955                if (!fstat(fd, &st)) {
 956                        *size = xsize_t(st.st_size);
 957                        if (!*size) {
 958                                /* mmap() is forbidden on empty files */
 959                                error(_("object file %s is empty"), path);
 960                                return NULL;
 961                        }
 962                        map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
 963                }
 964                close(fd);
 965        }
 966        return map;
 967}
 968
 969void *map_loose_object(struct repository *r,
 970                       const struct object_id *oid,
 971                       unsigned long *size)
 972{
 973        return map_loose_object_1(r, NULL, oid, size);
 974}
 975
 976static int unpack_loose_short_header(git_zstream *stream,
 977                                     unsigned char *map, unsigned long mapsize,
 978                                     void *buffer, unsigned long bufsiz)
 979{
 980        /* Get the data stream */
 981        memset(stream, 0, sizeof(*stream));
 982        stream->next_in = map;
 983        stream->avail_in = mapsize;
 984        stream->next_out = buffer;
 985        stream->avail_out = bufsiz;
 986
 987        git_inflate_init(stream);
 988        return git_inflate(stream, 0);
 989}
 990
 991int unpack_loose_header(git_zstream *stream,
 992                        unsigned char *map, unsigned long mapsize,
 993                        void *buffer, unsigned long bufsiz)
 994{
 995        int status = unpack_loose_short_header(stream, map, mapsize,
 996                                               buffer, bufsiz);
 997
 998        if (status < Z_OK)
 999                return status;
1000
1001        /* Make sure we have the terminating NUL */
1002        if (!memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1003                return -1;
1004        return 0;
1005}
1006
1007static int unpack_loose_header_to_strbuf(git_zstream *stream, unsigned char *map,
1008                                         unsigned long mapsize, void *buffer,
1009                                         unsigned long bufsiz, struct strbuf *header)
1010{
1011        int status;
1012
1013        status = unpack_loose_short_header(stream, map, mapsize, buffer, bufsiz);
1014        if (status < Z_OK)
1015                return -1;
1016
1017        /*
1018         * Check if entire header is unpacked in the first iteration.
1019         */
1020        if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1021                return 0;
1022
1023        /*
1024         * buffer[0..bufsiz] was not large enough.  Copy the partial
1025         * result out to header, and then append the result of further
1026         * reading the stream.
1027         */
1028        strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
1029        stream->next_out = buffer;
1030        stream->avail_out = bufsiz;
1031
1032        do {
1033                status = git_inflate(stream, 0);
1034                strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
1035                if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1036                        return 0;
1037                stream->next_out = buffer;
1038                stream->avail_out = bufsiz;
1039        } while (status != Z_STREAM_END);
1040        return -1;
1041}
1042
1043static void *unpack_loose_rest(git_zstream *stream,
1044                               void *buffer, unsigned long size,
1045                               const struct object_id *oid)
1046{
1047        int bytes = strlen(buffer) + 1;
1048        unsigned char *buf = xmallocz(size);
1049        unsigned long n;
1050        int status = Z_OK;
1051
1052        n = stream->total_out - bytes;
1053        if (n > size)
1054                n = size;
1055        memcpy(buf, (char *) buffer + bytes, n);
1056        bytes = n;
1057        if (bytes <= size) {
1058                /*
1059                 * The above condition must be (bytes <= size), not
1060                 * (bytes < size).  In other words, even though we
1061                 * expect no more output and set avail_out to zero,
1062                 * the input zlib stream may have bytes that express
1063                 * "this concludes the stream", and we *do* want to
1064                 * eat that input.
1065                 *
1066                 * Otherwise we would not be able to test that we
1067                 * consumed all the input to reach the expected size;
1068                 * we also want to check that zlib tells us that all
1069                 * went well with status == Z_STREAM_END at the end.
1070                 */
1071                stream->next_out = buf + bytes;
1072                stream->avail_out = size - bytes;
1073                while (status == Z_OK)
1074                        status = git_inflate(stream, Z_FINISH);
1075        }
1076        if (status == Z_STREAM_END && !stream->avail_in) {
1077                git_inflate_end(stream);
1078                return buf;
1079        }
1080
1081        if (status < 0)
1082                error(_("corrupt loose object '%s'"), oid_to_hex(oid));
1083        else if (stream->avail_in)
1084                error(_("garbage at end of loose object '%s'"),
1085                      oid_to_hex(oid));
1086        free(buf);
1087        return NULL;
1088}
1089
1090/*
1091 * We used to just use "sscanf()", but that's actually way
1092 * too permissive for what we want to check. So do an anal
1093 * object header parse by hand.
1094 */
1095static int parse_loose_header_extended(const char *hdr, struct object_info *oi,
1096                                       unsigned int flags)
1097{
1098        const char *type_buf = hdr;
1099        unsigned long size;
1100        int type, type_len = 0;
1101
1102        /*
1103         * The type can be of any size but is followed by
1104         * a space.
1105         */
1106        for (;;) {
1107                char c = *hdr++;
1108                if (!c)
1109                        return -1;
1110                if (c == ' ')
1111                        break;
1112                type_len++;
1113        }
1114
1115        type = type_from_string_gently(type_buf, type_len, 1);
1116        if (oi->type_name)
1117                strbuf_add(oi->type_name, type_buf, type_len);
1118        /*
1119         * Set type to 0 if its an unknown object and
1120         * we're obtaining the type using '--allow-unknown-type'
1121         * option.
1122         */
1123        if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE) && (type < 0))
1124                type = 0;
1125        else if (type < 0)
1126                die(_("invalid object type"));
1127        if (oi->typep)
1128                *oi->typep = type;
1129
1130        /*
1131         * The length must follow immediately, and be in canonical
1132         * decimal format (ie "010" is not valid).
1133         */
1134        size = *hdr++ - '0';
1135        if (size > 9)
1136                return -1;
1137        if (size) {
1138                for (;;) {
1139                        unsigned long c = *hdr - '0';
1140                        if (c > 9)
1141                                break;
1142                        hdr++;
1143                        size = size * 10 + c;
1144                }
1145        }
1146
1147        if (oi->sizep)
1148                *oi->sizep = size;
1149
1150        /*
1151         * The length must be followed by a zero byte
1152         */
1153        return *hdr ? -1 : type;
1154}
1155
1156int parse_loose_header(const char *hdr, unsigned long *sizep)
1157{
1158        struct object_info oi = OBJECT_INFO_INIT;
1159
1160        oi.sizep = sizep;
1161        return parse_loose_header_extended(hdr, &oi, 0);
1162}
1163
1164static int loose_object_info(struct repository *r,
1165                             const struct object_id *oid,
1166                             struct object_info *oi, int flags)
1167{
1168        int status = 0;
1169        unsigned long mapsize;
1170        void *map;
1171        git_zstream stream;
1172        char hdr[MAX_HEADER_LEN];
1173        struct strbuf hdrbuf = STRBUF_INIT;
1174        unsigned long size_scratch;
1175
1176        if (oi->delta_base_sha1)
1177                hashclr(oi->delta_base_sha1);
1178
1179        /*
1180         * If we don't care about type or size, then we don't
1181         * need to look inside the object at all. Note that we
1182         * do not optimize out the stat call, even if the
1183         * caller doesn't care about the disk-size, since our
1184         * return value implicitly indicates whether the
1185         * object even exists.
1186         */
1187        if (!oi->typep && !oi->type_name && !oi->sizep && !oi->contentp) {
1188                const char *path;
1189                struct stat st;
1190                if (!oi->disk_sizep && (flags & OBJECT_INFO_QUICK))
1191                        return quick_has_loose(r, oid) ? 0 : -1;
1192                if (stat_loose_object(r, oid, &st, &path) < 0)
1193                        return -1;
1194                if (oi->disk_sizep)
1195                        *oi->disk_sizep = st.st_size;
1196                return 0;
1197        }
1198
1199        map = map_loose_object(r, oid, &mapsize);
1200        if (!map)
1201                return -1;
1202
1203        if (!oi->sizep)
1204                oi->sizep = &size_scratch;
1205
1206        if (oi->disk_sizep)
1207                *oi->disk_sizep = mapsize;
1208        if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE)) {
1209                if (unpack_loose_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0)
1210                        status = error(_("unable to unpack %s header with --allow-unknown-type"),
1211                                       oid_to_hex(oid));
1212        } else if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
1213                status = error(_("unable to unpack %s header"),
1214                               oid_to_hex(oid));
1215        if (status < 0)
1216                ; /* Do nothing */
1217        else if (hdrbuf.len) {
1218                if ((status = parse_loose_header_extended(hdrbuf.buf, oi, flags)) < 0)
1219                        status = error(_("unable to parse %s header with --allow-unknown-type"),
1220                                       oid_to_hex(oid));
1221        } else if ((status = parse_loose_header_extended(hdr, oi, flags)) < 0)
1222                status = error(_("unable to parse %s header"), oid_to_hex(oid));
1223
1224        if (status >= 0 && oi->contentp) {
1225                *oi->contentp = unpack_loose_rest(&stream, hdr,
1226                                                  *oi->sizep, oid);
1227                if (!*oi->contentp) {
1228                        git_inflate_end(&stream);
1229                        status = -1;
1230                }
1231        } else
1232                git_inflate_end(&stream);
1233
1234        munmap(map, mapsize);
1235        if (status && oi->typep)
1236                *oi->typep = status;
1237        if (oi->sizep == &size_scratch)
1238                oi->sizep = NULL;
1239        strbuf_release(&hdrbuf);
1240        oi->whence = OI_LOOSE;
1241        return (status < 0) ? status : 0;
1242}
1243
1244int fetch_if_missing = 1;
1245
1246int oid_object_info_extended(struct repository *r, const struct object_id *oid,
1247                             struct object_info *oi, unsigned flags)
1248{
1249        static struct object_info blank_oi = OBJECT_INFO_INIT;
1250        struct pack_entry e;
1251        int rtype;
1252        const struct object_id *real = oid;
1253        int already_retried = 0;
1254
1255        if (flags & OBJECT_INFO_LOOKUP_REPLACE)
1256                real = lookup_replace_object(r, oid);
1257
1258        if (is_null_oid(real))
1259                return -1;
1260
1261        if (!oi)
1262                oi = &blank_oi;
1263
1264        if (!(flags & OBJECT_INFO_SKIP_CACHED)) {
1265                struct cached_object *co = find_cached_object(real);
1266                if (co) {
1267                        if (oi->typep)
1268                                *(oi->typep) = co->type;
1269                        if (oi->sizep)
1270                                *(oi->sizep) = co->size;
1271                        if (oi->disk_sizep)
1272                                *(oi->disk_sizep) = 0;
1273                        if (oi->delta_base_sha1)
1274                                hashclr(oi->delta_base_sha1);
1275                        if (oi->type_name)
1276                                strbuf_addstr(oi->type_name, type_name(co->type));
1277                        if (oi->contentp)
1278                                *oi->contentp = xmemdupz(co->buf, co->size);
1279                        oi->whence = OI_CACHED;
1280                        return 0;
1281                }
1282        }
1283
1284        while (1) {
1285                if (find_pack_entry(r, real, &e))
1286                        break;
1287
1288                if (flags & OBJECT_INFO_IGNORE_LOOSE)
1289                        return -1;
1290
1291                /* Most likely it's a loose object. */
1292                if (!loose_object_info(r, real, oi, flags))
1293                        return 0;
1294
1295                /* Not a loose object; someone else may have just packed it. */
1296                if (!(flags & OBJECT_INFO_QUICK)) {
1297                        reprepare_packed_git(r);
1298                        if (find_pack_entry(r, real, &e))
1299                                break;
1300                }
1301
1302                /* Check if it is a missing object */
1303                if (fetch_if_missing && repository_format_partial_clone &&
1304                    !already_retried && r == the_repository) {
1305                        /*
1306                         * TODO Investigate having fetch_object() return
1307                         * TODO error/success and stopping the music here.
1308                         * TODO Pass a repository struct through fetch_object,
1309                         * such that arbitrary repositories work.
1310                         */
1311                        fetch_objects(repository_format_partial_clone, real, 1);
1312                        already_retried = 1;
1313                        continue;
1314                }
1315
1316                return -1;
1317        }
1318
1319        if (oi == &blank_oi)
1320                /*
1321                 * We know that the caller doesn't actually need the
1322                 * information below, so return early.
1323                 */
1324                return 0;
1325        rtype = packed_object_info(r, e.p, e.offset, oi);
1326        if (rtype < 0) {
1327                mark_bad_packed_object(e.p, real->hash);
1328                return oid_object_info_extended(r, real, oi, 0);
1329        } else if (oi->whence == OI_PACKED) {
1330                oi->u.packed.offset = e.offset;
1331                oi->u.packed.pack = e.p;
1332                oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
1333                                         rtype == OBJ_OFS_DELTA);
1334        }
1335
1336        return 0;
1337}
1338
1339/* returns enum object_type or negative */
1340int oid_object_info(struct repository *r,
1341                    const struct object_id *oid,
1342                    unsigned long *sizep)
1343{
1344        enum object_type type;
1345        struct object_info oi = OBJECT_INFO_INIT;
1346
1347        oi.typep = &type;
1348        oi.sizep = sizep;
1349        if (oid_object_info_extended(r, oid, &oi,
1350                                      OBJECT_INFO_LOOKUP_REPLACE) < 0)
1351                return -1;
1352        return type;
1353}
1354
1355static void *read_object(const struct object_id *oid, enum object_type *type,
1356                         unsigned long *size)
1357{
1358        struct object_info oi = OBJECT_INFO_INIT;
1359        void *content;
1360        oi.typep = type;
1361        oi.sizep = size;
1362        oi.contentp = &content;
1363
1364        if (oid_object_info_extended(the_repository, oid, &oi, 0) < 0)
1365                return NULL;
1366        return content;
1367}
1368
1369int pretend_object_file(void *buf, unsigned long len, enum object_type type,
1370                        struct object_id *oid)
1371{
1372        struct cached_object *co;
1373
1374        hash_object_file(buf, len, type_name(type), oid);
1375        if (has_object_file(oid) || find_cached_object(oid))
1376                return 0;
1377        ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc);
1378        co = &cached_objects[cached_object_nr++];
1379        co->size = len;
1380        co->type = type;
1381        co->buf = xmalloc(len);
1382        memcpy(co->buf, buf, len);
1383        oidcpy(&co->oid, oid);
1384        return 0;
1385}
1386
1387/*
1388 * This function dies on corrupt objects; the callers who want to
1389 * deal with them should arrange to call read_object() and give error
1390 * messages themselves.
1391 */
1392void *read_object_file_extended(const struct object_id *oid,
1393                                enum object_type *type,
1394                                unsigned long *size,
1395                                int lookup_replace)
1396{
1397        void *data;
1398        const struct packed_git *p;
1399        const char *path;
1400        struct stat st;
1401        const struct object_id *repl = lookup_replace ?
1402                lookup_replace_object(the_repository, oid) : oid;
1403
1404        errno = 0;
1405        data = read_object(repl, type, size);
1406        if (data)
1407                return data;
1408
1409        if (errno && errno != ENOENT)
1410                die_errno(_("failed to read object %s"), oid_to_hex(oid));
1411
1412        /* die if we replaced an object with one that does not exist */
1413        if (repl != oid)
1414                die(_("replacement %s not found for %s"),
1415                    oid_to_hex(repl), oid_to_hex(oid));
1416
1417        if (!stat_loose_object(the_repository, repl, &st, &path))
1418                die(_("loose object %s (stored in %s) is corrupt"),
1419                    oid_to_hex(repl), path);
1420
1421        if ((p = has_packed_and_bad(repl->hash)) != NULL)
1422                die(_("packed object %s (stored in %s) is corrupt"),
1423                    oid_to_hex(repl), p->pack_name);
1424
1425        return NULL;
1426}
1427
1428void *read_object_with_reference(const struct object_id *oid,
1429                                 const char *required_type_name,
1430                                 unsigned long *size,
1431                                 struct object_id *actual_oid_return)
1432{
1433        enum object_type type, required_type;
1434        void *buffer;
1435        unsigned long isize;
1436        struct object_id actual_oid;
1437
1438        required_type = type_from_string(required_type_name);
1439        oidcpy(&actual_oid, oid);
1440        while (1) {
1441                int ref_length = -1;
1442                const char *ref_type = NULL;
1443
1444                buffer = read_object_file(&actual_oid, &type, &isize);
1445                if (!buffer)
1446                        return NULL;
1447                if (type == required_type) {
1448                        *size = isize;
1449                        if (actual_oid_return)
1450                                oidcpy(actual_oid_return, &actual_oid);
1451                        return buffer;
1452                }
1453                /* Handle references */
1454                else if (type == OBJ_COMMIT)
1455                        ref_type = "tree ";
1456                else if (type == OBJ_TAG)
1457                        ref_type = "object ";
1458                else {
1459                        free(buffer);
1460                        return NULL;
1461                }
1462                ref_length = strlen(ref_type);
1463
1464                if (ref_length + the_hash_algo->hexsz > isize ||
1465                    memcmp(buffer, ref_type, ref_length) ||
1466                    get_oid_hex((char *) buffer + ref_length, &actual_oid)) {
1467                        free(buffer);
1468                        return NULL;
1469                }
1470                free(buffer);
1471                /* Now we have the ID of the referred-to object in
1472                 * actual_oid.  Check again. */
1473        }
1474}
1475
1476static void write_object_file_prepare(const void *buf, unsigned long len,
1477                                      const char *type, struct object_id *oid,
1478                                      char *hdr, int *hdrlen)
1479{
1480        git_hash_ctx c;
1481
1482        /* Generate the header */
1483        *hdrlen = xsnprintf(hdr, *hdrlen, "%s %"PRIuMAX , type, (uintmax_t)len)+1;
1484
1485        /* Sha1.. */
1486        the_hash_algo->init_fn(&c);
1487        the_hash_algo->update_fn(&c, hdr, *hdrlen);
1488        the_hash_algo->update_fn(&c, buf, len);
1489        the_hash_algo->final_fn(oid->hash, &c);
1490}
1491
1492/*
1493 * Move the just written object into its final resting place.
1494 */
1495int finalize_object_file(const char *tmpfile, const char *filename)
1496{
1497        int ret = 0;
1498
1499        if (object_creation_mode == OBJECT_CREATION_USES_RENAMES)
1500                goto try_rename;
1501        else if (link(tmpfile, filename))
1502                ret = errno;
1503
1504        /*
1505         * Coda hack - coda doesn't like cross-directory links,
1506         * so we fall back to a rename, which will mean that it
1507         * won't be able to check collisions, but that's not a
1508         * big deal.
1509         *
1510         * The same holds for FAT formatted media.
1511         *
1512         * When this succeeds, we just return.  We have nothing
1513         * left to unlink.
1514         */
1515        if (ret && ret != EEXIST) {
1516        try_rename:
1517                if (!rename(tmpfile, filename))
1518                        goto out;
1519                ret = errno;
1520        }
1521        unlink_or_warn(tmpfile);
1522        if (ret) {
1523                if (ret != EEXIST) {
1524                        return error_errno(_("unable to write sha1 filename %s"), filename);
1525                }
1526                /* FIXME!!! Collision check here ? */
1527        }
1528
1529out:
1530        if (adjust_shared_perm(filename))
1531                return error(_("unable to set permission to '%s'"), filename);
1532        return 0;
1533}
1534
1535static int write_buffer(int fd, const void *buf, size_t len)
1536{
1537        if (write_in_full(fd, buf, len) < 0)
1538                return error_errno(_("file write error"));
1539        return 0;
1540}
1541
1542int hash_object_file(const void *buf, unsigned long len, const char *type,
1543                     struct object_id *oid)
1544{
1545        char hdr[MAX_HEADER_LEN];
1546        int hdrlen = sizeof(hdr);
1547        write_object_file_prepare(buf, len, type, oid, hdr, &hdrlen);
1548        return 0;
1549}
1550
1551/* Finalize a file on disk, and close it. */
1552static void close_loose_object(int fd)
1553{
1554        if (fsync_object_files)
1555                fsync_or_die(fd, "sha1 file");
1556        if (close(fd) != 0)
1557                die_errno(_("error when closing sha1 file"));
1558}
1559
1560/* Size of directory component, including the ending '/' */
1561static inline int directory_size(const char *filename)
1562{
1563        const char *s = strrchr(filename, '/');
1564        if (!s)
1565                return 0;
1566        return s - filename + 1;
1567}
1568
1569/*
1570 * This creates a temporary file in the same directory as the final
1571 * 'filename'
1572 *
1573 * We want to avoid cross-directory filename renames, because those
1574 * can have problems on various filesystems (FAT, NFS, Coda).
1575 */
1576static int create_tmpfile(struct strbuf *tmp, const char *filename)
1577{
1578        int fd, dirlen = directory_size(filename);
1579
1580        strbuf_reset(tmp);
1581        strbuf_add(tmp, filename, dirlen);
1582        strbuf_addstr(tmp, "tmp_obj_XXXXXX");
1583        fd = git_mkstemp_mode(tmp->buf, 0444);
1584        if (fd < 0 && dirlen && errno == ENOENT) {
1585                /*
1586                 * Make sure the directory exists; note that the contents
1587                 * of the buffer are undefined after mkstemp returns an
1588                 * error, so we have to rewrite the whole buffer from
1589                 * scratch.
1590                 */
1591                strbuf_reset(tmp);
1592                strbuf_add(tmp, filename, dirlen - 1);
1593                if (mkdir(tmp->buf, 0777) && errno != EEXIST)
1594                        return -1;
1595                if (adjust_shared_perm(tmp->buf))
1596                        return -1;
1597
1598                /* Try again */
1599                strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
1600                fd = git_mkstemp_mode(tmp->buf, 0444);
1601        }
1602        return fd;
1603}
1604
1605static int write_loose_object(const struct object_id *oid, char *hdr,
1606                              int hdrlen, const void *buf, unsigned long len,
1607                              time_t mtime)
1608{
1609        int fd, ret;
1610        unsigned char compressed[4096];
1611        git_zstream stream;
1612        git_hash_ctx c;
1613        struct object_id parano_oid;
1614        static struct strbuf tmp_file = STRBUF_INIT;
1615        static struct strbuf filename = STRBUF_INIT;
1616
1617        loose_object_path(the_repository, &filename, oid);
1618
1619        fd = create_tmpfile(&tmp_file, filename.buf);
1620        if (fd < 0) {
1621                if (errno == EACCES)
1622                        return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
1623                else
1624                        return error_errno(_("unable to create temporary file"));
1625        }
1626
1627        /* Set it up */
1628        git_deflate_init(&stream, zlib_compression_level);
1629        stream.next_out = compressed;
1630        stream.avail_out = sizeof(compressed);
1631        the_hash_algo->init_fn(&c);
1632
1633        /* First header.. */
1634        stream.next_in = (unsigned char *)hdr;
1635        stream.avail_in = hdrlen;
1636        while (git_deflate(&stream, 0) == Z_OK)
1637                ; /* nothing */
1638        the_hash_algo->update_fn(&c, hdr, hdrlen);
1639
1640        /* Then the data itself.. */
1641        stream.next_in = (void *)buf;
1642        stream.avail_in = len;
1643        do {
1644                unsigned char *in0 = stream.next_in;
1645                ret = git_deflate(&stream, Z_FINISH);
1646                the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
1647                if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
1648                        die(_("unable to write sha1 file"));
1649                stream.next_out = compressed;
1650                stream.avail_out = sizeof(compressed);
1651        } while (ret == Z_OK);
1652
1653        if (ret != Z_STREAM_END)
1654                die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
1655                    ret);
1656        ret = git_deflate_end_gently(&stream);
1657        if (ret != Z_OK)
1658                die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
1659                    ret);
1660        the_hash_algo->final_fn(parano_oid.hash, &c);
1661        if (!oideq(oid, &parano_oid))
1662                die(_("confused by unstable object source data for %s"),
1663                    oid_to_hex(oid));
1664
1665        close_loose_object(fd);
1666
1667        if (mtime) {
1668                struct utimbuf utb;
1669                utb.actime = mtime;
1670                utb.modtime = mtime;
1671                if (utime(tmp_file.buf, &utb) < 0)
1672                        warning_errno(_("failed utime() on %s"), tmp_file.buf);
1673        }
1674
1675        return finalize_object_file(tmp_file.buf, filename.buf);
1676}
1677
1678static int freshen_loose_object(const struct object_id *oid)
1679{
1680        return check_and_freshen(oid, 1);
1681}
1682
1683static int freshen_packed_object(const struct object_id *oid)
1684{
1685        struct pack_entry e;
1686        if (!find_pack_entry(the_repository, oid, &e))
1687                return 0;
1688        if (e.p->freshened)
1689                return 1;
1690        if (!freshen_file(e.p->pack_name))
1691                return 0;
1692        e.p->freshened = 1;
1693        return 1;
1694}
1695
1696int write_object_file(const void *buf, unsigned long len, const char *type,
1697                      struct object_id *oid)
1698{
1699        char hdr[MAX_HEADER_LEN];
1700        int hdrlen = sizeof(hdr);
1701
1702        /* Normally if we have it in the pack then we do not bother writing
1703         * it out into .git/objects/??/?{38} file.
1704         */
1705        write_object_file_prepare(buf, len, type, oid, hdr, &hdrlen);
1706        if (freshen_packed_object(oid) || freshen_loose_object(oid))
1707                return 0;
1708        return write_loose_object(oid, hdr, hdrlen, buf, len, 0);
1709}
1710
1711int hash_object_file_literally(const void *buf, unsigned long len,
1712                               const char *type, struct object_id *oid,
1713                               unsigned flags)
1714{
1715        char *header;
1716        int hdrlen, status = 0;
1717
1718        /* type string, SP, %lu of the length plus NUL must fit this */
1719        hdrlen = strlen(type) + MAX_HEADER_LEN;
1720        header = xmalloc(hdrlen);
1721        write_object_file_prepare(buf, len, type, oid, header, &hdrlen);
1722
1723        if (!(flags & HASH_WRITE_OBJECT))
1724                goto cleanup;
1725        if (freshen_packed_object(oid) || freshen_loose_object(oid))
1726                goto cleanup;
1727        status = write_loose_object(oid, header, hdrlen, buf, len, 0);
1728
1729cleanup:
1730        free(header);
1731        return status;
1732}
1733
1734int force_object_loose(const struct object_id *oid, time_t mtime)
1735{
1736        void *buf;
1737        unsigned long len;
1738        enum object_type type;
1739        char hdr[MAX_HEADER_LEN];
1740        int hdrlen;
1741        int ret;
1742
1743        if (has_loose_object(oid))
1744                return 0;
1745        buf = read_object(oid, &type, &len);
1746        if (!buf)
1747                return error(_("cannot read sha1_file for %s"), oid_to_hex(oid));
1748        hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1;
1749        ret = write_loose_object(oid, hdr, hdrlen, buf, len, mtime);
1750        free(buf);
1751
1752        return ret;
1753}
1754
1755int has_sha1_file_with_flags(const unsigned char *sha1, int flags)
1756{
1757        struct object_id oid;
1758        if (!startup_info->have_repository)
1759                return 0;
1760        hashcpy(oid.hash, sha1);
1761        return oid_object_info_extended(the_repository, &oid, NULL,
1762                                        flags | OBJECT_INFO_SKIP_CACHED) >= 0;
1763}
1764
1765int has_object_file(const struct object_id *oid)
1766{
1767        return has_sha1_file(oid->hash);
1768}
1769
1770int has_object_file_with_flags(const struct object_id *oid, int flags)
1771{
1772        return has_sha1_file_with_flags(oid->hash, flags);
1773}
1774
1775static void check_tree(const void *buf, size_t size)
1776{
1777        struct tree_desc desc;
1778        struct name_entry entry;
1779
1780        init_tree_desc(&desc, buf, size);
1781        while (tree_entry(&desc, &entry))
1782                /* do nothing
1783                 * tree_entry() will die() on malformed entries */
1784                ;
1785}
1786
1787static void check_commit(const void *buf, size_t size)
1788{
1789        struct commit c;
1790        memset(&c, 0, sizeof(c));
1791        if (parse_commit_buffer(the_repository, &c, buf, size, 0))
1792                die(_("corrupt commit"));
1793}
1794
1795static void check_tag(const void *buf, size_t size)
1796{
1797        struct tag t;
1798        memset(&t, 0, sizeof(t));
1799        if (parse_tag_buffer(the_repository, &t, buf, size))
1800                die(_("corrupt tag"));
1801}
1802
1803static int index_mem(struct index_state *istate,
1804                     struct object_id *oid, void *buf, size_t size,
1805                     enum object_type type,
1806                     const char *path, unsigned flags)
1807{
1808        int ret, re_allocated = 0;
1809        int write_object = flags & HASH_WRITE_OBJECT;
1810
1811        if (!type)
1812                type = OBJ_BLOB;
1813
1814        /*
1815         * Convert blobs to git internal format
1816         */
1817        if ((type == OBJ_BLOB) && path) {
1818                struct strbuf nbuf = STRBUF_INIT;
1819                if (convert_to_git(istate, path, buf, size, &nbuf,
1820                                   get_conv_flags(flags))) {
1821                        buf = strbuf_detach(&nbuf, &size);
1822                        re_allocated = 1;
1823                }
1824        }
1825        if (flags & HASH_FORMAT_CHECK) {
1826                if (type == OBJ_TREE)
1827                        check_tree(buf, size);
1828                if (type == OBJ_COMMIT)
1829                        check_commit(buf, size);
1830                if (type == OBJ_TAG)
1831                        check_tag(buf, size);
1832        }
1833
1834        if (write_object)
1835                ret = write_object_file(buf, size, type_name(type), oid);
1836        else
1837                ret = hash_object_file(buf, size, type_name(type), oid);
1838        if (re_allocated)
1839                free(buf);
1840        return ret;
1841}
1842
1843static int index_stream_convert_blob(struct index_state *istate,
1844                                     struct object_id *oid,
1845                                     int fd,
1846                                     const char *path,
1847                                     unsigned flags)
1848{
1849        int ret;
1850        const int write_object = flags & HASH_WRITE_OBJECT;
1851        struct strbuf sbuf = STRBUF_INIT;
1852
1853        assert(path);
1854        assert(would_convert_to_git_filter_fd(istate, path));
1855
1856        convert_to_git_filter_fd(istate, path, fd, &sbuf,
1857                                 get_conv_flags(flags));
1858
1859        if (write_object)
1860                ret = write_object_file(sbuf.buf, sbuf.len, type_name(OBJ_BLOB),
1861                                        oid);
1862        else
1863                ret = hash_object_file(sbuf.buf, sbuf.len, type_name(OBJ_BLOB),
1864                                       oid);
1865        strbuf_release(&sbuf);
1866        return ret;
1867}
1868
1869static int index_pipe(struct index_state *istate, struct object_id *oid,
1870                      int fd, enum object_type type,
1871                      const char *path, unsigned flags)
1872{
1873        struct strbuf sbuf = STRBUF_INIT;
1874        int ret;
1875
1876        if (strbuf_read(&sbuf, fd, 4096) >= 0)
1877                ret = index_mem(istate, oid, sbuf.buf, sbuf.len, type, path, flags);
1878        else
1879                ret = -1;
1880        strbuf_release(&sbuf);
1881        return ret;
1882}
1883
1884#define SMALL_FILE_SIZE (32*1024)
1885
1886static int index_core(struct index_state *istate,
1887                      struct object_id *oid, int fd, size_t size,
1888                      enum object_type type, const char *path,
1889                      unsigned flags)
1890{
1891        int ret;
1892
1893        if (!size) {
1894                ret = index_mem(istate, oid, "", size, type, path, flags);
1895        } else if (size <= SMALL_FILE_SIZE) {
1896                char *buf = xmalloc(size);
1897                ssize_t read_result = read_in_full(fd, buf, size);
1898                if (read_result < 0)
1899                        ret = error_errno(_("read error while indexing %s"),
1900                                          path ? path : "<unknown>");
1901                else if (read_result != size)
1902                        ret = error(_("short read while indexing %s"),
1903                                    path ? path : "<unknown>");
1904                else
1905                        ret = index_mem(istate, oid, buf, size, type, path, flags);
1906                free(buf);
1907        } else {
1908                void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
1909                ret = index_mem(istate, oid, buf, size, type, path, flags);
1910                munmap(buf, size);
1911        }
1912        return ret;
1913}
1914
1915/*
1916 * This creates one packfile per large blob unless bulk-checkin
1917 * machinery is "plugged".
1918 *
1919 * This also bypasses the usual "convert-to-git" dance, and that is on
1920 * purpose. We could write a streaming version of the converting
1921 * functions and insert that before feeding the data to fast-import
1922 * (or equivalent in-core API described above). However, that is
1923 * somewhat complicated, as we do not know the size of the filter
1924 * result, which we need to know beforehand when writing a git object.
1925 * Since the primary motivation for trying to stream from the working
1926 * tree file and to avoid mmaping it in core is to deal with large
1927 * binary blobs, they generally do not want to get any conversion, and
1928 * callers should avoid this code path when filters are requested.
1929 */
1930static int index_stream(struct object_id *oid, int fd, size_t size,
1931                        enum object_type type, const char *path,
1932                        unsigned flags)
1933{
1934        return index_bulk_checkin(oid, fd, size, type, path, flags);
1935}
1936
1937int index_fd(struct index_state *istate, struct object_id *oid,
1938             int fd, struct stat *st,
1939             enum object_type type, const char *path, unsigned flags)
1940{
1941        int ret;
1942
1943        /*
1944         * Call xsize_t() only when needed to avoid potentially unnecessary
1945         * die() for large files.
1946         */
1947        if (type == OBJ_BLOB && path && would_convert_to_git_filter_fd(istate, path))
1948                ret = index_stream_convert_blob(istate, oid, fd, path, flags);
1949        else if (!S_ISREG(st->st_mode))
1950                ret = index_pipe(istate, oid, fd, type, path, flags);
1951        else if (st->st_size <= big_file_threshold || type != OBJ_BLOB ||
1952                 (path && would_convert_to_git(istate, path)))
1953                ret = index_core(istate, oid, fd, xsize_t(st->st_size),
1954                                 type, path, flags);
1955        else
1956                ret = index_stream(oid, fd, xsize_t(st->st_size), type, path,
1957                                   flags);
1958        close(fd);
1959        return ret;
1960}
1961
1962int index_path(struct index_state *istate, struct object_id *oid,
1963               const char *path, struct stat *st, unsigned flags)
1964{
1965        int fd;
1966        struct strbuf sb = STRBUF_INIT;
1967        int rc = 0;
1968
1969        switch (st->st_mode & S_IFMT) {
1970        case S_IFREG:
1971                fd = open(path, O_RDONLY);
1972                if (fd < 0)
1973                        return error_errno("open(\"%s\")", path);
1974                if (index_fd(istate, oid, fd, st, OBJ_BLOB, path, flags) < 0)
1975                        return error(_("%s: failed to insert into database"),
1976                                     path);
1977                break;
1978        case S_IFLNK:
1979                if (strbuf_readlink(&sb, path, st->st_size))
1980                        return error_errno("readlink(\"%s\")", path);
1981                if (!(flags & HASH_WRITE_OBJECT))
1982                        hash_object_file(sb.buf, sb.len, blob_type, oid);
1983                else if (write_object_file(sb.buf, sb.len, blob_type, oid))
1984                        rc = error(_("%s: failed to insert into database"), path);
1985                strbuf_release(&sb);
1986                break;
1987        case S_IFDIR:
1988                return resolve_gitlink_ref(path, "HEAD", oid);
1989        default:
1990                return error(_("%s: unsupported file type"), path);
1991        }
1992        return rc;
1993}
1994
1995int read_pack_header(int fd, struct pack_header *header)
1996{
1997        if (read_in_full(fd, header, sizeof(*header)) != sizeof(*header))
1998                /* "eof before pack header was fully read" */
1999                return PH_ERROR_EOF;
2000
2001        if (header->hdr_signature != htonl(PACK_SIGNATURE))
2002                /* "protocol error (pack signature mismatch detected)" */
2003                return PH_ERROR_PACK_SIGNATURE;
2004        if (!pack_version_ok(header->hdr_version))
2005                /* "protocol error (pack version unsupported)" */
2006                return PH_ERROR_PROTOCOL;
2007        return 0;
2008}
2009
2010void assert_oid_type(const struct object_id *oid, enum object_type expect)
2011{
2012        enum object_type type = oid_object_info(the_repository, oid, NULL);
2013        if (type < 0)
2014                die(_("%s is not a valid object"), oid_to_hex(oid));
2015        if (type != expect)
2016                die(_("%s is not a valid '%s' object"), oid_to_hex(oid),
2017                    type_name(expect));
2018}
2019
2020int for_each_file_in_obj_subdir(unsigned int subdir_nr,
2021                                struct strbuf *path,
2022                                each_loose_object_fn obj_cb,
2023                                each_loose_cruft_fn cruft_cb,
2024                                each_loose_subdir_fn subdir_cb,
2025                                void *data)
2026{
2027        size_t origlen, baselen;
2028        DIR *dir;
2029        struct dirent *de;
2030        int r = 0;
2031        struct object_id oid;
2032
2033        if (subdir_nr > 0xff)
2034                BUG("invalid loose object subdirectory: %x", subdir_nr);
2035
2036        origlen = path->len;
2037        strbuf_complete(path, '/');
2038        strbuf_addf(path, "%02x", subdir_nr);
2039
2040        dir = opendir(path->buf);
2041        if (!dir) {
2042                if (errno != ENOENT)
2043                        r = error_errno(_("unable to open %s"), path->buf);
2044                strbuf_setlen(path, origlen);
2045                return r;
2046        }
2047
2048        oid.hash[0] = subdir_nr;
2049        strbuf_addch(path, '/');
2050        baselen = path->len;
2051
2052        while ((de = readdir(dir))) {
2053                size_t namelen;
2054                if (is_dot_or_dotdot(de->d_name))
2055                        continue;
2056
2057                namelen = strlen(de->d_name);
2058                strbuf_setlen(path, baselen);
2059                strbuf_add(path, de->d_name, namelen);
2060                if (namelen == the_hash_algo->hexsz - 2 &&
2061                    !hex_to_bytes(oid.hash + 1, de->d_name,
2062                                  the_hash_algo->rawsz - 1)) {
2063                        if (obj_cb) {
2064                                r = obj_cb(&oid, path->buf, data);
2065                                if (r)
2066                                        break;
2067                        }
2068                        continue;
2069                }
2070
2071                if (cruft_cb) {
2072                        r = cruft_cb(de->d_name, path->buf, data);
2073                        if (r)
2074                                break;
2075                }
2076        }
2077        closedir(dir);
2078
2079        strbuf_setlen(path, baselen - 1);
2080        if (!r && subdir_cb)
2081                r = subdir_cb(subdir_nr, path->buf, data);
2082
2083        strbuf_setlen(path, origlen);
2084
2085        return r;
2086}
2087
2088int for_each_loose_file_in_objdir_buf(struct strbuf *path,
2089                            each_loose_object_fn obj_cb,
2090                            each_loose_cruft_fn cruft_cb,
2091                            each_loose_subdir_fn subdir_cb,
2092                            void *data)
2093{
2094        int r = 0;
2095        int i;
2096
2097        for (i = 0; i < 256; i++) {
2098                r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb,
2099                                                subdir_cb, data);
2100                if (r)
2101                        break;
2102        }
2103
2104        return r;
2105}
2106
2107int for_each_loose_file_in_objdir(const char *path,
2108                                  each_loose_object_fn obj_cb,
2109                                  each_loose_cruft_fn cruft_cb,
2110                                  each_loose_subdir_fn subdir_cb,
2111                                  void *data)
2112{
2113        struct strbuf buf = STRBUF_INIT;
2114        int r;
2115
2116        strbuf_addstr(&buf, path);
2117        r = for_each_loose_file_in_objdir_buf(&buf, obj_cb, cruft_cb,
2118                                              subdir_cb, data);
2119        strbuf_release(&buf);
2120
2121        return r;
2122}
2123
2124int for_each_loose_object(each_loose_object_fn cb, void *data,
2125                          enum for_each_object_flags flags)
2126{
2127        struct object_directory *odb;
2128
2129        prepare_alt_odb(the_repository);
2130        for (odb = the_repository->objects->odb; odb; odb = odb->next) {
2131                int r = for_each_loose_file_in_objdir(odb->path, cb, NULL,
2132                                                      NULL, data);
2133                if (r)
2134                        return r;
2135
2136                if (flags & FOR_EACH_OBJECT_LOCAL_ONLY)
2137                        break;
2138        }
2139
2140        return 0;
2141}
2142
2143static int append_loose_object(const struct object_id *oid, const char *path,
2144                               void *data)
2145{
2146        oid_array_append(data, oid);
2147        return 0;
2148}
2149
2150struct oid_array *odb_loose_cache(struct object_directory *odb,
2151                                  const struct object_id *oid)
2152{
2153        int subdir_nr = oid->hash[0];
2154        struct strbuf buf = STRBUF_INIT;
2155
2156        if (subdir_nr < 0 ||
2157            subdir_nr >= ARRAY_SIZE(odb->loose_objects_subdir_seen))
2158                BUG("subdir_nr out of range");
2159
2160        if (odb->loose_objects_subdir_seen[subdir_nr])
2161                return &odb->loose_objects_cache[subdir_nr];
2162
2163        strbuf_addstr(&buf, odb->path);
2164        for_each_file_in_obj_subdir(subdir_nr, &buf,
2165                                    append_loose_object,
2166                                    NULL, NULL,
2167                                    &odb->loose_objects_cache[subdir_nr]);
2168        odb->loose_objects_subdir_seen[subdir_nr] = 1;
2169        strbuf_release(&buf);
2170        return &odb->loose_objects_cache[subdir_nr];
2171}
2172
2173void odb_clear_loose_cache(struct object_directory *odb)
2174{
2175        int i;
2176
2177        for (i = 0; i < ARRAY_SIZE(odb->loose_objects_cache); i++)
2178                oid_array_clear(&odb->loose_objects_cache[i]);
2179        memset(&odb->loose_objects_subdir_seen, 0,
2180               sizeof(odb->loose_objects_subdir_seen));
2181}
2182
2183static int check_stream_oid(git_zstream *stream,
2184                            const char *hdr,
2185                            unsigned long size,
2186                            const char *path,
2187                            const struct object_id *expected_oid)
2188{
2189        git_hash_ctx c;
2190        struct object_id real_oid;
2191        unsigned char buf[4096];
2192        unsigned long total_read;
2193        int status = Z_OK;
2194
2195        the_hash_algo->init_fn(&c);
2196        the_hash_algo->update_fn(&c, hdr, stream->total_out);
2197
2198        /*
2199         * We already read some bytes into hdr, but the ones up to the NUL
2200         * do not count against the object's content size.
2201         */
2202        total_read = stream->total_out - strlen(hdr) - 1;
2203
2204        /*
2205         * This size comparison must be "<=" to read the final zlib packets;
2206         * see the comment in unpack_loose_rest for details.
2207         */
2208        while (total_read <= size &&
2209               (status == Z_OK ||
2210                (status == Z_BUF_ERROR && !stream->avail_out))) {
2211                stream->next_out = buf;
2212                stream->avail_out = sizeof(buf);
2213                if (size - total_read < stream->avail_out)
2214                        stream->avail_out = size - total_read;
2215                status = git_inflate(stream, Z_FINISH);
2216                the_hash_algo->update_fn(&c, buf, stream->next_out - buf);
2217                total_read += stream->next_out - buf;
2218        }
2219        git_inflate_end(stream);
2220
2221        if (status != Z_STREAM_END) {
2222                error(_("corrupt loose object '%s'"), oid_to_hex(expected_oid));
2223                return -1;
2224        }
2225        if (stream->avail_in) {
2226                error(_("garbage at end of loose object '%s'"),
2227                      oid_to_hex(expected_oid));
2228                return -1;
2229        }
2230
2231        the_hash_algo->final_fn(real_oid.hash, &c);
2232        if (!oideq(expected_oid, &real_oid)) {
2233                error(_("sha1 mismatch for %s (expected %s)"), path,
2234                      oid_to_hex(expected_oid));
2235                return -1;
2236        }
2237
2238        return 0;
2239}
2240
2241int read_loose_object(const char *path,
2242                      const struct object_id *expected_oid,
2243                      enum object_type *type,
2244                      unsigned long *size,
2245                      void **contents)
2246{
2247        int ret = -1;
2248        void *map = NULL;
2249        unsigned long mapsize;
2250        git_zstream stream;
2251        char hdr[MAX_HEADER_LEN];
2252
2253        *contents = NULL;
2254
2255        map = map_loose_object_1(the_repository, path, NULL, &mapsize);
2256        if (!map) {
2257                error_errno(_("unable to mmap %s"), path);
2258                goto out;
2259        }
2260
2261        if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) {
2262                error(_("unable to unpack header of %s"), path);
2263                goto out;
2264        }
2265
2266        *type = parse_loose_header(hdr, size);
2267        if (*type < 0) {
2268                error(_("unable to parse header of %s"), path);
2269                git_inflate_end(&stream);
2270                goto out;
2271        }
2272
2273        if (*type == OBJ_BLOB && *size > big_file_threshold) {
2274                if (check_stream_oid(&stream, hdr, *size, path, expected_oid) < 0)
2275                        goto out;
2276        } else {
2277                *contents = unpack_loose_rest(&stream, hdr, *size, expected_oid);
2278                if (!*contents) {
2279                        error(_("unable to unpack contents of %s"), path);
2280                        git_inflate_end(&stream);
2281                        goto out;
2282                }
2283                if (check_object_signature(expected_oid, *contents,
2284                                         *size, type_name(*type))) {
2285                        error(_("sha1 mismatch for %s (expected %s)"), path,
2286                              oid_to_hex(expected_oid));
2287                        free(*contents);
2288                        goto out;
2289                }
2290        }
2291
2292        ret = 0; /* everything checks out */
2293
2294out:
2295        if (map)
2296                munmap(map, mapsize);
2297        return ret;
2298}