index-pack: factor out unpack core from get_data_from_pack

This allows caller to consume large inflated object with a fixed
amount of memory.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/builtin/index-pack.c b/builtin/index-pack.c
index c7c2b886567d4e0b6e4eea27ee86d452a8e89189..9129299cddbe1f868914d90221cedbe252aa6cac 100644 (file)
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@ -504,7 +504,9 @@ static void *unpack_raw_entry(struct object_entry *obj,
        return data;
 }
 
        return data;
 }
 

-static void *get_data_from_pack(struct object_entry *obj)
+static void *unpack_data(struct object_entry *obj,
+                        int (*consume)(const unsigned char *, unsigned long, void *),
+                        void *cb_data)

 {
        off_t from = obj[0].idx.offset + obj[0].hdr_size;
        unsigned long len = obj[1].idx.offset - from;
 {
        off_t from = obj[0].idx.offset + obj[0].hdr_size;
        unsigned long len = obj[1].idx.offset - from;


@@ -512,15 +514,16 @@ static void *get_data_from_pack(struct object_entry *obj)
        git_zstream stream;
        int status;
 
        git_zstream stream;
        int status;
 

-       data = xmalloc(obj->size);
+       data = xmalloc(consume ? 64*1024 : obj->size);

        inbuf = xmalloc((len < 64*1024) ? len : 64*1024);
 
        memset(&stream, 0, sizeof(stream));
        git_inflate_init(&stream);
        stream.next_out = data;
        inbuf = xmalloc((len < 64*1024) ? len : 64*1024);
 
        memset(&stream, 0, sizeof(stream));
        git_inflate_init(&stream);
        stream.next_out = data;

-       stream.avail_out = obj->size;
+       stream.avail_out = consume ? 64*1024 : obj->size;

 
        do {
 
        do {

+               unsigned char *last_out = stream.next_out;

                ssize_t n = (len < 64*1024) ? len : 64*1024;
                n = pread(pack_fd, inbuf, n, from);
                if (n < 0)
                ssize_t n = (len < 64*1024) ? len : 64*1024;
                n = pread(pack_fd, inbuf, n, from);
                if (n < 0)


@@ -535,6 +538,15 @@ static void *get_data_from_pack(struct object_entry *obj)
                stream.next_in = inbuf;
                stream.avail_in = n;
                status = git_inflate(&stream, 0);
                stream.next_in = inbuf;
                stream.avail_in = n;
                status = git_inflate(&stream, 0);

+               if (consume) {
+                       if (consume(last_out, stream.next_out - last_out, cb_data)) {
+                               free(inbuf);
+                               free(data);
+                               return NULL;
+                       }
+                       stream.next_out = data;
+                       stream.avail_out = 64*1024;
+               }

        } while (len && status == Z_OK && !stream.avail_in);
 
        /* This has been inflated OK when first encountered, so... */
        } while (len && status == Z_OK && !stream.avail_in);
 
        /* This has been inflated OK when first encountered, so... */


@@ -543,9 +555,18 @@ static void *get_data_from_pack(struct object_entry *obj)
 
        git_inflate_end(&stream);
        free(inbuf);
 
        git_inflate_end(&stream);
        free(inbuf);

+       if (consume) {
+               free(data);
+               data = NULL;
+       }

        return data;
 }
 
        return data;
 }
 

+static void *get_data_from_pack(struct object_entry *obj)
+{
+       return unpack_data(obj, NULL, NULL);
+}
+

 static int compare_delta_bases(const union delta_base *base1,
                               const union delta_base *base2,
                               enum object_type type1,
 static int compare_delta_bases(const union delta_base *base1,
                               const union delta_base *base2,
                               enum object_type type1,