Loop over pack_windows when inflating/accessing data.

When multiple mmaps start getting used for all pack file access it
is not possible to get all data associated with a specific object
in one contiguous memory region.  This limitation prevents simply
passing a single address and length to SHA1_Update or to inflate.

Instead we need to loop until we have processed all data of interest.

As we loop over the data we are always interested in reusing the same
window 'cursor', as the prior window will no longer be of any use
to us.  This allows the use_pack() call to automatically decrement
the use count of the prior window before setting up access for us
to the next window.

Within each loop we need to make use of the available length output
parameter of use_pack() to tell us how many bytes are available in
the current memory region, as we cannot tell otherwise.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c
index 6d7ae7f1aef758cbcd2bbb931287826fc8267dd8..afb926a34c53a2221fabe38cb5003acdcc9ffee6 100644 (file)
--- a/builtin-pack-objects.c
+++ b/builtin-pack-objects.c
@@ -276,7 +276,52 @@ static int encode_header(enum object_type type, unsigned long size, unsigned cha
  * we are going to reuse the existing object data as is.  make
  * sure it is not corrupt.
  */
  * we are going to reuse the existing object data as is.  make
  * sure it is not corrupt.
  */

-static int check_inflate(unsigned char *data, unsigned long len, unsigned long expect)
+static int check_pack_inflate(struct packed_git *p,
+               struct pack_window **w_curs,
+               unsigned long offset,
+               unsigned long len,
+               unsigned long expect)
+{
+       z_stream stream;
+       unsigned char fakebuf[4096], *in;
+       int st;
+
+       memset(&stream, 0, sizeof(stream));
+       inflateInit(&stream);
+       do {
+               in = use_pack(p, w_curs, offset, &stream.avail_in);
+               stream.next_in = in;
+               stream.next_out = fakebuf;
+               stream.avail_out = sizeof(fakebuf);
+               st = inflate(&stream, Z_FINISH);
+               offset += stream.next_in - in;
+       } while (st == Z_OK || st == Z_BUF_ERROR);
+       inflateEnd(&stream);
+       return (st == Z_STREAM_END &&
+               stream.total_out == expect &&
+               stream.total_in == len) ? 0 : -1;
+}
+
+static void copy_pack_data(struct sha1file *f,
+               struct packed_git *p,
+               struct pack_window **w_curs,
+               unsigned long offset,
+               unsigned long len)
+{
+       unsigned char *in;
+       unsigned int avail;
+
+       while (len) {
+               in = use_pack(p, w_curs, offset, &avail);
+               if (avail > len)
+                       avail = len;
+               sha1write(f, in, avail);
+               offset += avail;
+               len -= avail;
+       }
+}
+
+static int check_loose_inflate(unsigned char *data, unsigned long len, unsigned long expect)

 {
        z_stream stream;
        unsigned char fakebuf[4096];
 {
        z_stream stream;
        unsigned char fakebuf[4096];


@@ -323,7 +368,7 @@ static int revalidate_loose_object(struct object_entry *entry,
                return -1;
        map += used;
        mapsize -= used;
                return -1;
        map += used;
        mapsize -= used;

-       return check_inflate(map, mapsize, size);
+       return check_loose_inflate(map, mapsize, size);

 }
 
 static unsigned long write_object(struct sha1file *f,
 }
 
 static unsigned long write_object(struct sha1file *f,


@@ -417,6 +462,7 @@ static unsigned long write_object(struct sha1file *f,
        else {
                struct packed_git *p = entry->in_pack;
                struct pack_window *w_curs = NULL;
        else {
                struct packed_git *p = entry->in_pack;
                struct pack_window *w_curs = NULL;

+               unsigned long offset;

 
                if (entry->delta) {
                        obj_type = (allow_ofs_delta && entry->delta->offset) ?
 
                if (entry->delta) {
                        obj_type = (allow_ofs_delta && entry->delta->offset) ?


@@ -438,13 +484,13 @@ static unsigned long write_object(struct sha1file *f,
                        hdrlen += 20;
                }
 
                        hdrlen += 20;
                }
 

-               buf = use_pack(p, &w_curs, entry->in_pack_offset
-                       + entry->in_pack_header_size, NULL);
+               offset = entry->in_pack_offset + entry->in_pack_header_size;

                datalen = find_packed_object_size(p, entry->in_pack_offset)
                                - entry->in_pack_header_size;
                datalen = find_packed_object_size(p, entry->in_pack_offset)
                                - entry->in_pack_header_size;

-               if (!pack_to_stdout && check_inflate(buf, datalen, entry->size))
+               if (!pack_to_stdout && check_pack_inflate(p, &w_curs,
+                               offset, datalen, entry->size))

                        die("corrupt delta in pack %s", sha1_to_hex(entry->sha1));
                        die("corrupt delta in pack %s", sha1_to_hex(entry->sha1));

-               sha1write(f, buf, datalen);
+               copy_pack_data(f, p, &w_curs, offset, datalen);

                unuse_pack(&w_curs);
                reused++;
        }
                unuse_pack(&w_curs);
                reused++;
        }

diff --git a/pack-check.c b/pack-check.c
index 972916f4064386e033ada16e6badc3a174ef5b23..08a9fd8dc09056c1b21cb7414f2467cffcdef077 100644 (file)
--- a/pack-check.c
+++ b/pack-check.c
@@ -8,39 +8,38 @@ static int verify_packfile(struct packed_git *p,
        void *index_base = p->index_base;
        SHA_CTX ctx;
        unsigned char sha1[20];
        void *index_base = p->index_base;
        SHA_CTX ctx;
        unsigned char sha1[20];

-       unsigned long pack_size = p->pack_size;
-       void *pack_base;
-       struct pack_header *hdr;
+       unsigned long offset = 0, pack_sig = p->pack_size - 20;

        int nr_objects, err, i;
 
        int nr_objects, err, i;
 

-       /* Header consistency check */
-       pack_base = use_pack(p, w_curs, 0, NULL);
-       hdr = (struct pack_header*)pack_base;
-       if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
-               return error("Packfile %s signature mismatch", p->pack_name);
-       if (!pack_version_ok(hdr->hdr_version))
-               return error("Packfile version %d unsupported",
-                            ntohl(hdr->hdr_version));
-       nr_objects = ntohl(hdr->hdr_entries);
-       if (num_packed_objects(p) != nr_objects)
-               return error("Packfile claims to have %d objects, "
-                            "while idx size expects %d", nr_objects,
-                            num_packed_objects(p));
+       /* Note that the pack header checks are actually performed by
+        * use_pack when it first opens the pack file.  If anything
+        * goes wrong during those checks then the call will die out
+        * immediately.
+        */

 
        SHA1_Init(&ctx);
 
        SHA1_Init(&ctx);

-       SHA1_Update(&ctx, pack_base, pack_size - 20);
+       while (offset < pack_sig) {
+               unsigned int remaining;
+               unsigned char *in = use_pack(p, w_curs, offset, &remaining);
+               offset += remaining;
+               if (offset > pack_sig)
+                       remaining -= offset - pack_sig;
+               SHA1_Update(&ctx, in, remaining);
+       }

        SHA1_Final(sha1, &ctx);
        SHA1_Final(sha1, &ctx);

-       if (hashcmp(sha1, (unsigned char *)pack_base + pack_size - 20))
+       if (hashcmp(sha1, use_pack(p, w_curs, pack_sig, NULL)))

                return error("Packfile %s SHA1 mismatch with itself",
                             p->pack_name);
        if (hashcmp(sha1, (unsigned char *)index_base + index_size - 40))
                return error("Packfile %s SHA1 mismatch with idx",
                             p->pack_name);
                return error("Packfile %s SHA1 mismatch with itself",
                             p->pack_name);
        if (hashcmp(sha1, (unsigned char *)index_base + index_size - 40))
                return error("Packfile %s SHA1 mismatch with idx",
                             p->pack_name);

+       unuse_pack(w_curs);

 
        /* Make sure everything reachable from idx is valid.  Since we
         * have verified that nr_objects matches between idx and pack,
         * we do not do scan-streaming check on the pack file.
         */
 
        /* Make sure everything reachable from idx is valid.  Since we
         * have verified that nr_objects matches between idx and pack,
         * we do not do scan-streaming check on the pack file.
         */

+       nr_objects = num_packed_objects(p);

        for (i = err = 0; i < nr_objects; i++) {
                unsigned char sha1[20];
                void *data;
        for (i = err = 0; i < nr_objects; i++) {
                unsigned char sha1[20];
                void *data;


@@ -73,15 +72,12 @@ static int verify_packfile(struct packed_git *p,
 
 #define MAX_CHAIN 40
 
 
 #define MAX_CHAIN 40
 

-static void show_pack_info(struct packed_git *p,
-               struct pack_window **w_curs)
+static void show_pack_info(struct packed_git *p)

 {
 {

-       struct pack_header *hdr;

        int nr_objects, i;
        unsigned int chain_histogram[MAX_CHAIN];
 
        int nr_objects, i;
        unsigned int chain_histogram[MAX_CHAIN];
 

-       hdr = (struct pack_header*)use_pack(p, w_curs, 0, NULL);
-       nr_objects = ntohl(hdr->hdr_entries);
+       nr_objects = num_packed_objects(p);

        memset(chain_histogram, 0, sizeof(chain_histogram));
 
        for (i = 0; i < nr_objects; i++) {
        memset(chain_histogram, 0, sizeof(chain_histogram));
 
        for (i = 0; i < nr_objects; i++) {


@@ -153,9 +149,7 @@ int verify_pack(struct packed_git *p, int verbose)
                if (ret)
                        printf("%s: bad\n", p->pack_name);
                else {
                if (ret)
                        printf("%s: bad\n", p->pack_name);
                else {

-                       struct pack_window *w_curs = NULL;
-                       show_pack_info(p, &w_curs);
-                       unuse_pack(&w_curs);
+                       show_pack_info(p);

                        printf("%s: ok\n", p->pack_name);
                }
        }
                        printf("%s: ok\n", p->pack_name);
                }
        }

diff --git a/sha1_file.c b/sha1_file.c
index 4d80527baf92a667ca6934406193389ca2c0821d..c30452251943bbb058bc5064aaee4a0033ae22c3 100644 (file)
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -962,19 +962,23 @@ static int packed_delta_info(struct packed_git *p,
 
        if (sizep) {
                const unsigned char *data;
 
        if (sizep) {
                const unsigned char *data;

-               unsigned char delta_head[20];
+               unsigned char delta_head[20], *in;

                unsigned long result_size;
                z_stream stream;
                int st;
 
                memset(&stream, 0, sizeof(stream));
                unsigned long result_size;
                z_stream stream;
                int st;
 
                memset(&stream, 0, sizeof(stream));

-
-               stream.next_in = use_pack(p, w_curs, offset, &stream.avail_in);

                stream.next_out = delta_head;
                stream.avail_out = sizeof(delta_head);
 
                inflateInit(&stream);
                stream.next_out = delta_head;
                stream.avail_out = sizeof(delta_head);
 
                inflateInit(&stream);

-               st = inflate(&stream, Z_FINISH);
+               do {
+                       in = use_pack(p, w_curs, offset, &stream.avail_in);
+                       stream.next_in = in;
+                       st = inflate(&stream, Z_FINISH);
+                       offset += stream.next_in - in;
+               } while ((st == Z_OK || st == Z_BUF_ERROR)
+                       && stream.total_out < sizeof(delta_head));

                inflateEnd(&stream);
                if ((st != Z_STREAM_END) &&
                    stream.total_out != sizeof(delta_head))
                inflateEnd(&stream);
                if ((st != Z_STREAM_END) &&
                    stream.total_out != sizeof(delta_head))


@@ -1103,17 +1107,21 @@ static void *unpack_compressed_entry(struct packed_git *p,
 {
        int st;
        z_stream stream;
 {
        int st;
        z_stream stream;

-       unsigned char *buffer;
+       unsigned char *buffer, *in;

 
        buffer = xmalloc(size + 1);
        buffer[size] = 0;
        memset(&stream, 0, sizeof(stream));
 
        buffer = xmalloc(size + 1);
        buffer[size] = 0;
        memset(&stream, 0, sizeof(stream));

-       stream.next_in = use_pack(p, w_curs, offset, &stream.avail_in);

        stream.next_out = buffer;
        stream.avail_out = size;
 
        inflateInit(&stream);
        stream.next_out = buffer;
        stream.avail_out = size;
 
        inflateInit(&stream);

-       st = inflate(&stream, Z_FINISH);
+       do {
+               in = use_pack(p, w_curs, offset, &stream.avail_in);
+               stream.next_in = in;
+               st = inflate(&stream, Z_FINISH);
+               offset += stream.next_in - in;
+       } while (st == Z_OK || st == Z_BUF_ERROR);

        inflateEnd(&stream);
        if ((st != Z_STREAM_END) || stream.total_out != size) {
                free(buffer);
        inflateEnd(&stream);
        if ((st != Z_STREAM_END) || stream.total_out != size) {
                free(buffer);