fetch-pack.c: use oidset to check existence of loose object
[gitweb.git] / sha1_file.c
index 59238f5bea9c666d3c4976fb16b510c9883b9f9c..c0a19794795caa8bf7c1e5fe8d4094ca57bde41f 100644 (file)
 #include "bulk-checkin.h"
 #include "streaming.h"
 #include "dir.h"
-#include "mru.h"
 #include "list.h"
 #include "mergesort.h"
 #include "quote.h"
 #include "packfile.h"
+#include "fetch-object.h"
 
 const unsigned char null_sha1[GIT_MAX_RAWSZ];
 const struct object_id null_oid;
@@ -39,32 +39,32 @@ const struct object_id empty_blob_oid = {
        EMPTY_BLOB_SHA1_BIN_LITERAL
 };
 
-static void git_hash_sha1_init(void *ctx)
+static void git_hash_sha1_init(git_hash_ctx *ctx)
 {
-       git_SHA1_Init((git_SHA_CTX *)ctx);
+       git_SHA1_Init(&ctx->sha1);
 }
 
-static void git_hash_sha1_update(void *ctx, const void *data, size_t len)
+static void git_hash_sha1_update(git_hash_ctx *ctx, const void *data, size_t len)
 {
-       git_SHA1_Update((git_SHA_CTX *)ctx, data, len);
+       git_SHA1_Update(&ctx->sha1, data, len);
 }
 
-static void git_hash_sha1_final(unsigned char *hash, void *ctx)
+static void git_hash_sha1_final(unsigned char *hash, git_hash_ctx *ctx)
 {
-       git_SHA1_Final(hash, (git_SHA_CTX *)ctx);
+       git_SHA1_Final(hash, &ctx->sha1);
 }
 
-static void git_hash_unknown_init(void *ctx)
+static void git_hash_unknown_init(git_hash_ctx *ctx)
 {
        die("trying to init unknown hash");
 }
 
-static void git_hash_unknown_update(void *ctx, const void *data, size_t len)
+static void git_hash_unknown_update(git_hash_ctx *ctx, const void *data, size_t len)
 {
        die("trying to update unknown hash");
 }
 
-static void git_hash_unknown_final(unsigned char *hash, void *ctx)
+static void git_hash_unknown_final(unsigned char *hash, git_hash_ctx *ctx)
 {
        die("trying to finalize unknown hash");
 }
@@ -75,7 +75,6 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = {
                0x00000000,
                0,
                0,
-               0,
                git_hash_unknown_init,
                git_hash_unknown_update,
                git_hash_unknown_final,
@@ -86,7 +85,6 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = {
                "sha-1",
                /* "sha1", big-endian */
                0x73686131,
-               sizeof(git_SHA_CTX),
                GIT_SHA1_RAWSZ,
                GIT_SHA1_HEXSZ,
                git_hash_sha1_init,
@@ -133,14 +131,14 @@ static struct cached_object *find_cached_object(const unsigned char *sha1)
 }
 
 
-static enum safe_crlf get_safe_crlf(unsigned flags)
+static int get_conv_flags(unsigned flags)
 {
        if (flags & HASH_RENORMALIZE)
-               return SAFE_CRLF_RENORMALIZE;
+               return CONV_EOL_RENORMALIZE;
        else if (flags & HASH_WRITE_OBJECT)
-               return safe_crlf;
+         return global_conv_flags_eol;
        else
-               return SAFE_CRLF_FALSE;
+               return 0;
 }
 
 
@@ -321,15 +319,11 @@ static void fill_sha1_path(struct strbuf *buf, const unsigned char *sha1)
        }
 }
 
-const char *sha1_file_name(const unsigned char *sha1)
+void sha1_file_name(struct strbuf *buf, const unsigned char *sha1)
 {
-       static struct strbuf buf = STRBUF_INIT;
-
-       strbuf_reset(&buf);
-       strbuf_addf(&buf, "%s/", get_object_directory());
-
-       fill_sha1_path(&buf, sha1);
-       return buf.buf;
+       strbuf_addstr(buf, get_object_directory());
+       strbuf_addch(buf, '/');
+       fill_sha1_path(buf, sha1);
 }
 
 struct strbuf *alt_scratch_buf(struct alternate_object_database *alt)
@@ -710,7 +704,12 @@ int check_and_freshen_file(const char *fn, int freshen)
 
 static int check_and_freshen_local(const unsigned char *sha1, int freshen)
 {
-       return check_and_freshen_file(sha1_file_name(sha1), freshen);
+       static struct strbuf buf = STRBUF_INIT;
+
+       strbuf_reset(&buf);
+       sha1_file_name(&buf, sha1);
+
+       return check_and_freshen_file(buf.buf, freshen);
 }
 
 static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen)
@@ -791,7 +790,7 @@ int check_sha1_signature(const unsigned char *sha1, void *map,
        struct object_id real_oid;
        enum object_type obj_type;
        struct git_istream *st;
-       git_SHA_CTX c;
+       git_hash_ctx c;
        char hdr[32];
        int hdrlen;
 
@@ -805,11 +804,11 @@ int check_sha1_signature(const unsigned char *sha1, void *map,
                return -1;
 
        /* Generate the header */
-       hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(obj_type), size) + 1;
+       hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", type_name(obj_type), size) + 1;
 
        /* Sha1.. */
-       git_SHA1_Init(&c);
-       git_SHA1_Update(&c, hdr, hdrlen);
+       the_hash_algo->init_fn(&c);
+       the_hash_algo->update_fn(&c, hdr, hdrlen);
        for (;;) {
                char buf[1024 * 16];
                ssize_t readlen = read_istream(st, buf, sizeof(buf));
@@ -820,9 +819,9 @@ int check_sha1_signature(const unsigned char *sha1, void *map,
                }
                if (!readlen)
                        break;
-               git_SHA1_Update(&c, buf, readlen);
+               the_hash_algo->update_fn(&c, buf, readlen);
        }
-       git_SHA1_Final(real_oid.hash, &c);
+       the_hash_algo->final_fn(real_oid.hash, &c);
        close_istream(st);
        return hashcmp(sha1, real_oid.hash) ? -1 : 0;
 }
@@ -866,8 +865,12 @@ static int stat_sha1_file(const unsigned char *sha1, struct stat *st,
                          const char **path)
 {
        struct alternate_object_database *alt;
+       static struct strbuf buf = STRBUF_INIT;
+
+       strbuf_reset(&buf);
+       sha1_file_name(&buf, sha1);
+       *path = buf.buf;
 
-       *path = sha1_file_name(sha1);
        if (!lstat(*path, st))
                return 0;
 
@@ -891,8 +894,12 @@ static int open_sha1_file(const unsigned char *sha1, const char **path)
        int fd;
        struct alternate_object_database *alt;
        int most_interesting_errno;
+       static struct strbuf buf = STRBUF_INIT;
+
+       strbuf_reset(&buf);
+       sha1_file_name(&buf, sha1);
+       *path = buf.buf;
 
-       *path = sha1_file_name(sha1);
        fd = git_open(*path);
        if (fd >= 0)
                return fd;
@@ -1087,8 +1094,8 @@ static int parse_sha1_header_extended(const char *hdr, struct object_info *oi,
        }
 
        type = type_from_string_gently(type_buf, type_len, 1);
-       if (oi->typename)
-               strbuf_add(oi->typename, type_buf, type_len);
+       if (oi->type_name)
+               strbuf_add(oi->type_name, type_buf, type_len);
        /*
         * Set type to 0 if its an unknown object and
         * we're obtaining the type using '--allow-unknown-type'
@@ -1158,7 +1165,7 @@ static int sha1_loose_object_info(const unsigned char *sha1,
         * return value implicitly indicates whether the
         * object even exists.
         */
-       if (!oi->typep && !oi->typename && !oi->sizep && !oi->contentp) {
+       if (!oi->typep && !oi->type_name && !oi->sizep && !oi->contentp) {
                const char *path;
                struct stat st;
                if (stat_sha1_file(sha1, &st, &path) < 0)
@@ -1213,6 +1220,8 @@ static int sha1_loose_object_info(const unsigned char *sha1,
        return (status < 0) ? status : 0;
 }
 
+int fetch_if_missing = 1;
+
 int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags)
 {
        static struct object_info blank_oi = OBJECT_INFO_INIT;
@@ -1221,6 +1230,7 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi,
        const unsigned char *real = (flags & OBJECT_INFO_LOOKUP_REPLACE) ?
                                    lookup_replace_object(sha1) :
                                    sha1;
+       int already_retried = 0;
 
        if (is_null_sha1(real))
                return -1;
@@ -1239,8 +1249,8 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi,
                                *(oi->disk_sizep) = 0;
                        if (oi->delta_base_sha1)
                                hashclr(oi->delta_base_sha1);
-                       if (oi->typename)
-                               strbuf_addstr(oi->typename, typename(co->type));
+                       if (oi->type_name)
+                               strbuf_addstr(oi->type_name, type_name(co->type));
                        if (oi->contentp)
                                *oi->contentp = xmemdupz(co->buf, co->size);
                        oi->whence = OI_CACHED;
@@ -1248,19 +1258,35 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi,
                }
        }
 
-       if (!find_pack_entry(real, &e)) {
+       while (1) {
+               if (find_pack_entry(real, &e))
+                       break;
+
+               if (flags & OBJECT_INFO_IGNORE_LOOSE)
+                       return -1;
+
                /* Most likely it's a loose object. */
                if (!sha1_loose_object_info(real, oi, flags))
                        return 0;
 
                /* Not a loose object; someone else may have just packed it. */
-               if (flags & OBJECT_INFO_QUICK) {
-                       return -1;
-               } else {
-                       reprepare_packed_git();
-                       if (!find_pack_entry(real, &e))
-                               return -1;
+               reprepare_packed_git();
+               if (find_pack_entry(real, &e))
+                       break;
+
+               /* Check if it is a missing object */
+               if (fetch_if_missing && repository_format_partial_clone &&
+                   !already_retried) {
+                       /*
+                        * TODO Investigate haveing fetch_object() return
+                        * TODO error/success and stopping the music here.
+                        */
+                       fetch_object(repository_format_partial_clone, real);
+                       already_retried = 1;
+                       continue;
                }
+
+               return -1;
        }
 
        if (oi == &blank_oi)
@@ -1269,7 +1295,6 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi,
                 * information below, so return early.
                 */
                return 0;
-
        rtype = packed_object_info(e.p, e.offset, oi);
        if (rtype < 0) {
                mark_bad_packed_object(e.p, real);
@@ -1317,7 +1342,7 @@ int pretend_object_file(void *buf, unsigned long len, enum object_type type,
 {
        struct cached_object *co;
 
-       hash_object_file(buf, len, typename(type), oid);
+       hash_object_file(buf, len, type_name(type), oid);
        if (has_sha1_file(oid->hash) || find_cached_object(oid->hash))
                return 0;
        ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc);
@@ -1423,16 +1448,16 @@ static void write_object_file_prepare(const void *buf, unsigned long len,
                                      const char *type, struct object_id *oid,
                                      char *hdr, int *hdrlen)
 {
-       git_SHA_CTX c;
+       git_hash_ctx c;
 
        /* Generate the header */
        *hdrlen = xsnprintf(hdr, *hdrlen, "%s %lu", type, len)+1;
 
        /* Sha1.. */
-       git_SHA1_Init(&c);
-       git_SHA1_Update(&c, hdr, *hdrlen);
-       git_SHA1_Update(&c, buf, len);
-       git_SHA1_Final(oid->hash, &c);
+       the_hash_algo->init_fn(&c);
+       the_hash_algo->update_fn(&c, hdr, *hdrlen);
+       the_hash_algo->update_fn(&c, buf, len);
+       the_hash_algo->final_fn(oid->hash, &c);
 }
 
 /*
@@ -1555,12 +1580,15 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
        int fd, ret;
        unsigned char compressed[4096];
        git_zstream stream;
-       git_SHA_CTX c;
+       git_hash_ctx c;
        struct object_id parano_oid;
        static struct strbuf tmp_file = STRBUF_INIT;
-       const char *filename = sha1_file_name(oid->hash);
+       static struct strbuf filename = STRBUF_INIT;
+
+       strbuf_reset(&filename);
+       sha1_file_name(&filename, oid->hash);
 
-       fd = create_tmpfile(&tmp_file, filename);
+       fd = create_tmpfile(&tmp_file, filename.buf);
        if (fd < 0) {
                if (errno == EACCES)
                        return error("insufficient permission for adding an object to repository database %s", get_object_directory());
@@ -1572,14 +1600,14 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
        git_deflate_init(&stream, zlib_compression_level);
        stream.next_out = compressed;
        stream.avail_out = sizeof(compressed);
-       git_SHA1_Init(&c);
+       the_hash_algo->init_fn(&c);
 
        /* First header.. */
        stream.next_in = (unsigned char *)hdr;
        stream.avail_in = hdrlen;
        while (git_deflate(&stream, 0) == Z_OK)
                ; /* nothing */
-       git_SHA1_Update(&c, hdr, hdrlen);
+       the_hash_algo->update_fn(&c, hdr, hdrlen);
 
        /* Then the data itself.. */
        stream.next_in = (void *)buf;
@@ -1587,7 +1615,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
        do {
                unsigned char *in0 = stream.next_in;
                ret = git_deflate(&stream, Z_FINISH);
-               git_SHA1_Update(&c, in0, stream.next_in - in0);
+               the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
                if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
                        die("unable to write sha1 file");
                stream.next_out = compressed;
@@ -1601,7 +1629,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
        if (ret != Z_OK)
                die("deflateEnd on object %s failed (%d)", oid_to_hex(oid),
                    ret);
-       git_SHA1_Final(parano_oid.hash, &c);
+       the_hash_algo->final_fn(parano_oid.hash, &c);
        if (oidcmp(oid, &parano_oid) != 0)
                die("confused by unstable object source data for %s",
                    oid_to_hex(oid));
@@ -1616,7 +1644,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
                        warning_errno("failed utime() on %s", tmp_file.buf);
        }
 
-       return finalize_object_file(tmp_file.buf, filename);
+       return finalize_object_file(tmp_file.buf, filename.buf);
 }
 
 static int freshen_loose_object(const unsigned char *sha1)
@@ -1652,8 +1680,9 @@ int write_object_file(const void *buf, unsigned long len, const char *type,
        return write_loose_object(oid, hdr, hdrlen, buf, len, 0);
 }
 
-int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type,
-                            struct object_id *oid, unsigned flags)
+int hash_object_file_literally(const void *buf, unsigned long len,
+                              const char *type, struct object_id *oid,
+                              unsigned flags)
 {
        char *header;
        int hdrlen, status = 0;
@@ -1688,7 +1717,7 @@ int force_object_loose(const struct object_id *oid, time_t mtime)
        buf = read_object(oid->hash, &type, &len);
        if (!buf)
                return error("cannot read sha1_file for %s", oid_to_hex(oid));
-       hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(type), len) + 1;
+       hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", type_name(type), len) + 1;
        ret = write_loose_object(oid, hdr, hdrlen, buf, len, mtime);
        free(buf);
 
@@ -1757,7 +1786,7 @@ static int index_mem(struct object_id *oid, void *buf, size_t size,
        if ((type == OBJ_BLOB) && path) {
                struct strbuf nbuf = STRBUF_INIT;
                if (convert_to_git(&the_index, path, buf, size, &nbuf,
-                                  get_safe_crlf(flags))) {
+                                  get_conv_flags(flags))) {
                        buf = strbuf_detach(&nbuf, &size);
                        re_allocated = 1;
                }
@@ -1772,9 +1801,9 @@ static int index_mem(struct object_id *oid, void *buf, size_t size,
        }
 
        if (write_object)
-               ret = write_object_file(buf, size, typename(type), oid);
+               ret = write_object_file(buf, size, type_name(type), oid);
        else
-               ret = hash_object_file(buf, size, typename(type), oid);
+               ret = hash_object_file(buf, size, type_name(type), oid);
        if (re_allocated)
                free(buf);
        return ret;
@@ -1791,13 +1820,13 @@ static int index_stream_convert_blob(struct object_id *oid, int fd,
        assert(would_convert_to_git_filter_fd(path));
 
        convert_to_git_filter_fd(&the_index, path, fd, &sbuf,
-                                get_safe_crlf(flags));
+                                get_conv_flags(flags));
 
        if (write_object)
-               ret = write_object_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
+               ret = write_object_file(sbuf.buf, sbuf.len, type_name(OBJ_BLOB),
                                        oid);
        else
-               ret = hash_object_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
+               ret = hash_object_file(sbuf.buf, sbuf.len, type_name(OBJ_BLOB),
                                       oid);
        strbuf_release(&sbuf);
        return ret;
@@ -1947,7 +1976,7 @@ void assert_sha1_type(const unsigned char *sha1, enum object_type expect)
                die("%s is not a valid object", sha1_to_hex(sha1));
        if (type != expect)
                die("%s is not a valid '%s' object", sha1_to_hex(sha1),
-                   typename(expect));
+                   type_name(expect));
 }
 
 int for_each_file_in_obj_subdir(unsigned int subdir_nr,
@@ -2098,14 +2127,14 @@ static int check_stream_sha1(git_zstream *stream,
                             const char *path,
                             const unsigned char *expected_sha1)
 {
-       git_SHA_CTX c;
+       git_hash_ctx c;
        unsigned char real_sha1[GIT_MAX_RAWSZ];
        unsigned char buf[4096];
        unsigned long total_read;
        int status = Z_OK;
 
-       git_SHA1_Init(&c);
-       git_SHA1_Update(&c, hdr, stream->total_out);
+       the_hash_algo->init_fn(&c);
+       the_hash_algo->update_fn(&c, hdr, stream->total_out);
 
        /*
         * We already read some bytes into hdr, but the ones up to the NUL
@@ -2124,7 +2153,7 @@ static int check_stream_sha1(git_zstream *stream,
                if (size - total_read < stream->avail_out)
                        stream->avail_out = size - total_read;
                status = git_inflate(stream, Z_FINISH);
-               git_SHA1_Update(&c, buf, stream->next_out - buf);
+               the_hash_algo->update_fn(&c, buf, stream->next_out - buf);
                total_read += stream->next_out - buf;
        }
        git_inflate_end(stream);
@@ -2139,7 +2168,7 @@ static int check_stream_sha1(git_zstream *stream,
                return -1;
        }
 
-       git_SHA1_Final(real_sha1, &c);
+       the_hash_algo->final_fn(real_sha1, &c);
        if (hashcmp(expected_sha1, real_sha1)) {
                error("sha1 mismatch for %s (expected %s)", path,
                      sha1_to_hex(expected_sha1));
@@ -2192,7 +2221,7 @@ int read_loose_object(const char *path,
                        goto out;
                }
                if (check_sha1_signature(expected_sha1, *contents,
-                                        *size, typename(*type))) {
+                                        *size, type_name(*type))) {
                        error("sha1 mismatch for %s (expected %s)", path,
                              sha1_to_hex(expected_sha1));
                        free(*contents);