Merge branch 'jc/zlib-wrap'

* jc/zlib-wrap:
zlib: allow feeding more than 4GB in one go
zlib: zlib can only process 4GB at a time
zlib: wrap deflateBound() too
zlib: wrap deflate side of the API
zlib: wrap inflateInit2 used to accept only for gzip format
zlib: wrap remaining calls to direct inflate/inflateEnd
zlib wrapper: refactor error message formatter

Conflicts:
sha1_file.c

diff --git a/archive-zip.c b/archive-zip.c
index cf285044e3576d0127c3215cb1253443d67517dc..72d55a58ac60019f9f6175d4c86af47348630762 100644 (file)
--- a/archive-zip.c
+++ b/archive-zip.c
@@ -90,14 +90,14 @@ static void copy_le32(unsigned char *dest, unsigned int n)
 static void *zlib_deflate(void *data, unsigned long size,
                int compression_level, unsigned long *compressed_size)
 {
-       z_stream stream;
+       git_zstream stream;
        unsigned long maxsize;
        void *buffer;
        int result;
 
        memset(&stream, 0, sizeof(stream));
-       deflateInit(&stream, compression_level);
-       maxsize = deflateBound(&stream, size);
+       git_deflate_init(&stream, compression_level);
+       maxsize = git_deflate_bound(&stream, size);
        buffer = xmalloc(maxsize);
 
        stream.next_in = data;
@@ -106,7 +106,7 @@ static void *zlib_deflate(void *data, unsigned long size,
        stream.avail_out = maxsize;
 
        do {
-               result = deflate(&stream, Z_FINISH);
+               result = git_deflate(&stream, Z_FINISH);
        } while (result == Z_OK);
 
        if (result != Z_STREAM_END) {
@@ -114,7 +114,7 @@ static void *zlib_deflate(void *data, unsigned long size,
                return NULL;
        }
 
-       deflateEnd(&stream);
+       git_deflate_end(&stream);
        *compressed_size = stream.total_out;
 
        return buffer;

diff --git a/builtin/apply.c b/builtin/apply.c
index 530d4bb7e76da8e402f78ab7f981a080d25d6e78..f2edc52818e817e83717f992a6bbd66dad4ef24d 100644 (file)
--- a/builtin/apply.c
+++ b/builtin/apply.c
@@ -1634,7 +1634,7 @@ static inline int metadata_changes(struct patch *patch)
 static char *inflate_it(const void *data, unsigned long size,
                        unsigned long inflated_size)
 {
-       z_stream stream;
+       git_zstream stream;
        void *out;
        int st;
 

diff --git a/builtin/index-pack.c b/builtin/index-pack.c
index e40451ffb4ecd31107bfe51817cdce18e1dfe859..81cdc28b30731e722eaa2af045a3464bc83f0e16 100644 (file)
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@ -265,7 +265,7 @@ static void unlink_base_data(struct base_data *c)
 static void *unpack_entry_data(unsigned long offset, unsigned long size)
 {
        int status;
-       z_stream stream;
+       git_zstream stream;
        void *buf = xmalloc(size);
 
        memset(&stream, 0, sizeof(stream));
@@ -355,7 +355,7 @@ static void *get_data_from_pack(struct object_entry *obj)
        off_t from = obj[0].idx.offset + obj[0].hdr_size;
        unsigned long len = obj[1].idx.offset - from;
        unsigned char *data, *inbuf;
-       z_stream stream;
+       git_zstream stream;
        int status;
 
        data = xmalloc(obj->size);
@@ -666,26 +666,26 @@ static void parse_pack_objects(unsigned char *sha1)
 
 static int write_compressed(struct sha1file *f, void *in, unsigned int size)
 {
-       z_stream stream;
+       git_zstream stream;
        int status;
        unsigned char outbuf[4096];
 
        memset(&stream, 0, sizeof(stream));
-       deflateInit(&stream, zlib_compression_level);
+       git_deflate_init(&stream, zlib_compression_level);
        stream.next_in = in;
        stream.avail_in = size;
 
        do {
                stream.next_out = outbuf;
                stream.avail_out = sizeof(outbuf);
-               status = deflate(&stream, Z_FINISH);
+               status = git_deflate(&stream, Z_FINISH);
                sha1write(f, outbuf, sizeof(outbuf) - stream.avail_out);
        } while (status == Z_OK);
 
        if (status != Z_STREAM_END)
                die("unable to deflate appended object (%d)", status);
        size = stream.total_out;
-       deflateEnd(&stream);
+       git_deflate_end(&stream);
        return size;
 }
 

diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c
index f402a843bb5ee23360d29bdbbc6cc9c5b9c72008..c6e2d8766b0ec15fcfe9dc0a60ee81db6750b527 100644 (file)
--- a/builtin/pack-objects.c
+++ b/builtin/pack-objects.c
@@ -126,13 +126,13 @@ static void *get_delta(struct object_entry *entry)
 
 static unsigned long do_compress(void **pptr, unsigned long size)
 {
-       z_stream stream;
+       git_zstream stream;
        void *in, *out;
        unsigned long maxsize;
 
        memset(&stream, 0, sizeof(stream));
-       deflateInit(&stream, pack_compression_level);
-       maxsize = deflateBound(&stream, size);
+       git_deflate_init(&stream, pack_compression_level);
+       maxsize = git_deflate_bound(&stream, size);
 
        in = *pptr;
        out = xmalloc(maxsize);
@@ -142,9 +142,9 @@ static unsigned long do_compress(void **pptr, unsigned long size)
        stream.avail_in = size;
        stream.next_out = out;
        stream.avail_out = maxsize;
-       while (deflate(&stream, Z_FINISH) == Z_OK)
+       while (git_deflate(&stream, Z_FINISH) == Z_OK)
                ; /* nothing */
-       deflateEnd(&stream);
+       git_deflate_end(&stream);
 
        free(in);
        return stream.total_out;
@@ -160,7 +160,7 @@ static int check_pack_inflate(struct packed_git *p,
                off_t len,
                unsigned long expect)
 {
-       z_stream stream;
+       git_zstream stream;
        unsigned char fakebuf[4096], *in;
        int st;
 
@@ -187,12 +187,12 @@ static void copy_pack_data(struct sha1file *f,
                off_t len)
 {
        unsigned char *in;
-       unsigned int avail;
+       unsigned long avail;
 
        while (len) {
                in = use_pack(p, w_curs, offset, &avail);
                if (avail > len)
-                       avail = (unsigned int)len;
+                       avail = (unsigned long)len;
                sha1write(f, in, avail);
                offset += avail;
                len -= avail;
@@ -994,7 +994,7 @@ static void check_object(struct object_entry *entry)
                const unsigned char *base_ref = NULL;
                struct object_entry *base_entry;
                unsigned long used, used_0;
-               unsigned int avail;
+               unsigned long avail;
                off_t ofs;
                unsigned char *buf, c;
 

diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c
index f63973c9143c4109a3d1e40df9ec66a32498fe00..14e04e6795bd514fb1fec506073d8a9a71668cfa 100644 (file)
--- a/builtin/unpack-objects.c
+++ b/builtin/unpack-objects.c
@@ -90,7 +90,7 @@ static void use(int bytes)
 
 static void *get_data(unsigned long size)
 {
-       z_stream stream;
+       git_zstream stream;
        void *buf = xmalloc(size);
 
        memset(&stream, 0, sizeof(stream));

diff --git a/cache.h b/cache.h
index bc9e5eb15e23c23b41b76e1f353e078ea3858344..30be900911fc3985211e80f79dbe86b739103718 100644 (file)
--- a/cache.h
+++ b/cache.h
@@ -17,13 +17,27 @@
 #endif
 
 #include <zlib.h>
-#if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200
-#define deflateBound(c,s)  ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11)
-#endif
-
-void git_inflate_init(z_streamp strm);
-void git_inflate_end(z_streamp strm);
-int git_inflate(z_streamp strm, int flush);
+typedef struct git_zstream {
+       z_stream z;
+       unsigned long avail_in;
+       unsigned long avail_out;
+       unsigned long total_in;
+       unsigned long total_out;
+       unsigned char *next_in;
+       unsigned char *next_out;
+} git_zstream;
+
+void git_inflate_init(git_zstream *);
+void git_inflate_init_gzip_only(git_zstream *);
+void git_inflate_end(git_zstream *);
+int git_inflate(git_zstream *, int flush);
+
+void git_deflate_init(git_zstream *, int level);
+void git_deflate_init_gzip(git_zstream *, int level);
+void git_deflate_end(git_zstream *);
+int git_deflate_end_gently(git_zstream *);
+int git_deflate(git_zstream *, int flush);
+unsigned long git_deflate_bound(git_zstream *, unsigned long);
 
 #if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT)
 #define DTYPE(de)      ((de)->d_type)
@@ -753,7 +767,7 @@ extern int write_sha1_file(const void *buf, unsigned long len, const char *type,
 extern int pretend_sha1_file(void *, unsigned long, enum object_type, unsigned char *);
 extern int force_object_loose(const unsigned char *sha1, time_t mtime);
 extern void *map_sha1_file(const unsigned char *sha1, unsigned long *size);
-extern int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz);
+extern int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz);
 extern int parse_sha1_header(const char *hdr, unsigned long *sizep);
 
 /* global flag to enable extra checks when accessing packed objects */
@@ -984,7 +998,7 @@ extern struct packed_git *find_sha1_pack(const unsigned char *sha1,
 extern void pack_report(void);
 extern int open_pack_index(struct packed_git *);
 extern void close_pack_index(struct packed_git *);
-extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *);
+extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *);
 extern void close_pack_windows(struct packed_git *);
 extern void unuse_pack(struct pack_window **);
 extern void free_pack_by_name(const char *);

diff --git a/diff.c b/diff.c
index 918c70ca457bcdca54b9d524ff14b60d88b5a631..93ef9a265ca6b52b644468979dba732c70b5097a 100644 (file)
--- a/diff.c
+++ b/diff.c
@@ -1861,20 +1861,20 @@ static unsigned char *deflate_it(char *data,
 {
        int bound;
        unsigned char *deflated;
-       z_stream stream;
+       git_zstream stream;
 
        memset(&stream, 0, sizeof(stream));
-       deflateInit(&stream, zlib_compression_level);
-       bound = deflateBound(&stream, size);
+       git_deflate_init(&stream, zlib_compression_level);
+       bound = git_deflate_bound(&stream, size);
        deflated = xmalloc(bound);
        stream.next_out = deflated;
        stream.avail_out = bound;
 
        stream.next_in = (unsigned char *)data;
        stream.avail_in = size;
-       while (deflate(&stream, Z_FINISH) == Z_OK)
+       while (git_deflate(&stream, Z_FINISH) == Z_OK)
                ; /* nothing */
-       deflateEnd(&stream);
+       git_deflate_end(&stream);
        *result_size = stream.total_out;
        return deflated;
 }

diff --git a/fast-import.c b/fast-import.c
index 78d978684da2e2b79340c138e1b6300100df1221..1d5e3336a51a4bf19c0ab700565826b438c43266 100644 (file)
--- a/fast-import.c
+++ b/fast-import.c
@@ -1017,7 +1017,7 @@ static int store_object(
        unsigned char sha1[20];
        unsigned long hdrlen, deltalen;
        git_SHA_CTX c;
-       z_stream s;
+       git_zstream s;
 
        hdrlen = sprintf((char *)hdr,"%s %lu", typename(type),
                (unsigned long)dat->len) + 1;
@@ -1050,7 +1050,7 @@ static int store_object(
                delta = NULL;
 
        memset(&s, 0, sizeof(s));
-       deflateInit(&s, pack_compression_level);
+       git_deflate_init(&s, pack_compression_level);
        if (delta) {
                s.next_in = delta;
                s.avail_in = deltalen;
@@ -1058,11 +1058,11 @@ static int store_object(
                s.next_in = (void *)dat->buf;
                s.avail_in = dat->len;
        }
-       s.avail_out = deflateBound(&s, s.avail_in);
+       s.avail_out = git_deflate_bound(&s, s.avail_in);
        s.next_out = out = xmalloc(s.avail_out);
-       while (deflate(&s, Z_FINISH) == Z_OK)
-               /* nothing */;
-       deflateEnd(&s);
+       while (git_deflate(&s, Z_FINISH) == Z_OK)
+               ; /* nothing */
+       git_deflate_end(&s);
 
        /* Determine if we should auto-checkpoint. */
        if ((max_packsize && (pack_size + 60 + s.total_out) > max_packsize)
@@ -1078,14 +1078,14 @@ static int store_object(
                        delta = NULL;
 
                        memset(&s, 0, sizeof(s));
-                       deflateInit(&s, pack_compression_level);
+                       git_deflate_init(&s, pack_compression_level);
                        s.next_in = (void *)dat->buf;
                        s.avail_in = dat->len;
-                       s.avail_out = deflateBound(&s, s.avail_in);
+                       s.avail_out = git_deflate_bound(&s, s.avail_in);
                        s.next_out = out = xrealloc(out, s.avail_out);
-                       while (deflate(&s, Z_FINISH) == Z_OK)
-                               /* nothing */;
-                       deflateEnd(&s);
+                       while (git_deflate(&s, Z_FINISH) == Z_OK)
+                               ; /* nothing */
+                       git_deflate_end(&s);
                }
        }
 
@@ -1163,7 +1163,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark)
        off_t offset;
        git_SHA_CTX c;
        git_SHA_CTX pack_file_ctx;
-       z_stream s;
+       git_zstream s;
        int status = Z_OK;
 
        /* Determine if we should auto-checkpoint. */
@@ -1187,7 +1187,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark)
        crc32_begin(pack_file);
 
        memset(&s, 0, sizeof(s));
-       deflateInit(&s, pack_compression_level);
+       git_deflate_init(&s, pack_compression_level);
 
        hdrlen = encode_in_pack_object_header(OBJ_BLOB, len, out_buf);
        if (out_sz <= hdrlen)
@@ -1209,7 +1209,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark)
                        len -= n;
                }
 
-               status = deflate(&s, len ? 0 : Z_FINISH);
+               status = git_deflate(&s, len ? 0 : Z_FINISH);
 
                if (!s.avail_out || status == Z_STREAM_END) {
                        size_t n = s.next_out - out_buf;
@@ -1228,7 +1228,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark)
                        die("unexpected deflate failure: %d", status);
                }
        }
-       deflateEnd(&s);
+       git_deflate_end(&s);
        git_SHA1_Final(sha1, &c);
 
        if (sha1out)

diff --git a/http-backend.c b/http-backend.c
index 85015048dd466fb2f2afe12086379ecf579d0a56..59ad7da605f711af6970d6832db32efd62b6ea97 100644 (file)
--- a/http-backend.c
+++ b/http-backend.c
@@ -271,16 +271,13 @@ static struct rpc_service *select_service(const char *name)
 
 static void inflate_request(const char *prog_name, int out)
 {
-       z_stream stream;
+       git_zstream stream;
        unsigned char in_buf[8192];
        unsigned char out_buf[8192];
        unsigned long cnt = 0;
-       int ret;
 
        memset(&stream, 0, sizeof(stream));
-       ret = inflateInit2(&stream, (15 + 16));
-       if (ret != Z_OK)
-               die("cannot start zlib inflater, zlib err %d", ret);
+       git_inflate_init_gzip_only(&stream);
 
        while (1) {
                ssize_t n = xread(0, in_buf, sizeof(in_buf));
@@ -296,7 +293,7 @@ static void inflate_request(const char *prog_name, int out)
                        stream.next_out = out_buf;
                        stream.avail_out = sizeof(out_buf);
 
-                       ret = inflate(&stream, Z_NO_FLUSH);
+                       ret = git_inflate(&stream, Z_NO_FLUSH);
                        if (ret != Z_OK && ret != Z_STREAM_END)
                                die("zlib error inflating request, result %d", ret);
 
@@ -311,7 +308,7 @@ static void inflate_request(const char *prog_name, int out)
        }
 
 done:
-       inflateEnd(&stream);
+       git_inflate_end(&stream);
        close(out);
 }
 

diff --git a/http-push.c b/http-push.c
index 28bfe768f7749e455f522fcfedeb2350da646416..6e8f6d09abb6397f1782fa32d9d273fc4fc740fe 100644 (file)
--- a/http-push.c
+++ b/http-push.c
@@ -377,15 +377,15 @@ static void start_put(struct transfer_request *request)
        unsigned long len;
        int hdrlen;
        ssize_t size;
-       z_stream stream;
+       git_zstream stream;
 
        unpacked = read_sha1_file(request->obj->sha1, &type, &len);
        hdrlen = sprintf(hdr, "%s %lu", typename(type), len) + 1;
 
        /* Set it up */
        memset(&stream, 0, sizeof(stream));
-       deflateInit(&stream, zlib_compression_level);
-       size = deflateBound(&stream, len + hdrlen);
+       git_deflate_init(&stream, zlib_compression_level);
+       size = git_deflate_bound(&stream, len + hdrlen);
        strbuf_init(&request->buffer.buf, size);
        request->buffer.posn = 0;
 
@@ -396,15 +396,15 @@ static void start_put(struct transfer_request *request)
        /* First header.. */
        stream.next_in = (void *)hdr;
        stream.avail_in = hdrlen;
-       while (deflate(&stream, 0) == Z_OK)
-               /* nothing */;
+       while (git_deflate(&stream, 0) == Z_OK)
+               ; /* nothing */
 
        /* Then the data itself.. */
        stream.next_in = unpacked;
        stream.avail_in = len;
-       while (deflate(&stream, Z_FINISH) == Z_OK)
-               /* nothing */;
-       deflateEnd(&stream);
+       while (git_deflate(&stream, Z_FINISH) == Z_OK)
+               ; /* nothing */
+       git_deflate_end(&stream);
        free(unpacked);
 
        request->buffer.buf.len = stream.total_out;

diff --git a/http.h b/http.h
index 19b7134fa5c7ff9b65565dcbddccae1354f024c7..0bf8592dc45209c9be5b8ddac3a53f6fc11e29be 100644 (file)
--- a/http.h
+++ b/http.h
@@ -172,7 +172,7 @@ struct http_object_request {
        unsigned char sha1[20];
        unsigned char real_sha1[20];
        git_SHA_CTX c;
-       z_stream stream;
+       git_zstream stream;
        int zret;
        int rename;
        struct active_request_slot *slot;

diff --git a/pack-check.c b/pack-check.c
index a1a521648deee8e7e82c4e2cb7fe8fe03605dc06..0c19b6e5a5677bd14989175abddc119381fac4ef 100644 (file)
--- a/pack-check.c
+++ b/pack-check.c
@@ -26,7 +26,7 @@ int check_pack_crc(struct packed_git *p, struct pack_window **w_curs,
        uint32_t data_crc = crc32(0, NULL, 0);
 
        do {
-               unsigned int avail;
+               unsigned long avail;
                void *data = use_pack(p, w_curs, offset, &avail);
                if (avail > len)
                        avail = len;
@@ -61,7 +61,7 @@ static int verify_packfile(struct packed_git *p,
 
        git_SHA1_Init(&ctx);
        do {
-               unsigned int remaining;
+               unsigned long remaining;
                unsigned char *in = use_pack(p, w_curs, offset, &remaining);
                offset += remaining;
                if (!pack_sig_ofs)

diff --git a/remote-curl.c b/remote-curl.c
index b5be25ce9623936e42566ca52b9e4d1d33153111..faaeda44a9225ff56d45e9566cccf1a2278abb27 100644 (file)
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -471,16 +471,12 @@ static int post_rpc(struct rpc_state *rpc)
                 * the transfer time.
                 */
                size_t size;
-               z_stream stream;
+               git_zstream stream;
                int ret;
 
                memset(&stream, 0, sizeof(stream));
-               ret = deflateInit2(&stream, Z_BEST_COMPRESSION,
-                               Z_DEFLATED, (15 + 16),
-                               8, Z_DEFAULT_STRATEGY);
-               if (ret != Z_OK)
-                       die("cannot deflate request; zlib init error %d", ret);
-               size = deflateBound(&stream, rpc->len);
+               git_deflate_init_gzip(&stream, Z_BEST_COMPRESSION);
+               size = git_deflate_bound(&stream, rpc->len);
                gzip_body = xmalloc(size);
 
                stream.next_in = (unsigned char *)rpc->buf;
@@ -488,11 +484,11 @@ static int post_rpc(struct rpc_state *rpc)
                stream.next_out = (unsigned char *)gzip_body;
                stream.avail_out = size;
 
-               ret = deflate(&stream, Z_FINISH);
+               ret = git_deflate(&stream, Z_FINISH);
                if (ret != Z_STREAM_END)
                        die("cannot deflate request; zlib deflate error %d", ret);
 
-               ret = deflateEnd(&stream);
+               ret = git_deflate_end_gently(&stream);
                if (ret != Z_OK)
                        die("cannot deflate request; zlib end error %d", ret);
 

diff --git a/sha1_file.c b/sha1_file.c
index bd6fb2f95dda17adcc56ae288af971a781cb7cdf..a6aac70923cad6ed3f2afec631dcc67d5f28a019 100644 (file)
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -834,7 +834,7 @@ static int in_window(struct pack_window *win, off_t offset)
 unsigned char *use_pack(struct packed_git *p,
                struct pack_window **w_cursor,
                off_t offset,
-               unsigned int *left)
+               unsigned long *left)
 {
        struct pack_window *win = *w_cursor;
 
@@ -1254,7 +1254,7 @@ unsigned long unpack_object_header_buffer(const unsigned char *buf,
        return used;
 }
 
-int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
+int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
 {
        unsigned long size, used;
        static const char valid_loose_object_type[8] = {
@@ -1296,7 +1296,7 @@ int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsi
        return git_inflate(stream, 0);
 }
 
-static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
+static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
 {
        int bytes = strlen(buffer) + 1;
        unsigned char *buf = xmallocz(size);
@@ -1395,7 +1395,7 @@ int parse_sha1_header(const char *hdr, unsigned long *sizep)
 static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
 {
        int ret;
-       z_stream stream;
+       git_zstream stream;
        char hdr[8192];
 
        ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr));
@@ -1411,7 +1411,7 @@ unsigned long get_size_from_delta(struct packed_git *p,
 {
        const unsigned char *data;
        unsigned char delta_head[20], *in;
-       z_stream stream;
+       git_zstream stream;
        int st;
 
        memset(&stream, 0, sizeof(stream));
@@ -1533,7 +1533,7 @@ int unpack_object_header(struct packed_git *p,
                         unsigned long *sizep)
 {
        unsigned char *base;
-       unsigned int left;
+       unsigned long left;
        unsigned long used;
        enum object_type type;
 
@@ -1648,7 +1648,7 @@ static void *unpack_compressed_entry(struct packed_git *p,
                                    unsigned long size)
 {
        int st;
-       z_stream stream;
+       git_zstream stream;
        unsigned char *buffer, *in;
 
        buffer = xmallocz(size);
@@ -2088,7 +2088,7 @@ static int sha1_loose_object_info(const unsigned char *sha1, unsigned long *size
        int status;
        unsigned long mapsize, size;
        void *map;
-       z_stream stream;
+       git_zstream stream;
        char hdr[32];
 
        map = map_sha1_file(sha1, &mapsize);
@@ -2457,7 +2457,7 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
 {
        int fd, ret;
        unsigned char compressed[4096];
-       z_stream stream;
+       git_zstream stream;
        git_SHA_CTX c;
        unsigned char parano_sha1[20];
        char *filename;
@@ -2474,7 +2474,7 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
 
        /* Set it up */
        memset(&stream, 0, sizeof(stream));
-       deflateInit(&stream, zlib_compression_level);
+       git_deflate_init(&stream, zlib_compression_level);
        stream.next_out = compressed;
        stream.avail_out = sizeof(compressed);
        git_SHA1_Init(&c);
@@ -2482,8 +2482,8 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
        /* First header.. */
        stream.next_in = (unsigned char *)hdr;
        stream.avail_in = hdrlen;
-       while (deflate(&stream, 0) == Z_OK)
-               /* nothing */;
+       while (git_deflate(&stream, 0) == Z_OK)
+               ; /* nothing */
        git_SHA1_Update(&c, hdr, hdrlen);
 
        /* Then the data itself.. */
@@ -2491,7 +2491,7 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
        stream.avail_in = len;
        do {
                unsigned char *in0 = stream.next_in;
-               ret = deflate(&stream, Z_FINISH);
+               ret = git_deflate(&stream, Z_FINISH);
                git_SHA1_Update(&c, in0, stream.next_in - in0);
                if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
                        die("unable to write sha1 file");
@@ -2501,7 +2501,7 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
 
        if (ret != Z_STREAM_END)
                die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret);
-       ret = deflateEnd(&stream);
+       ret = git_deflate_end_gently(&stream);
        if (ret != Z_OK)
                die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret);
        git_SHA1_Final(parano_sha1, &c);

diff --git a/streaming.c b/streaming.c
index 91414f459231a83e6d22ee5a91bd0f6bc65ff5b0..25c9a209bb1ba9cec8bb49623ae45fc19fdca2c7 100644 (file)
--- a/streaming.c
+++ b/streaming.c
@@ -66,7 +66,7 @@ struct filtered_istream {
 struct git_istream {
        const struct stream_vtbl *vtbl;
        unsigned long size; /* inflated size of full object */
-       z_stream z;
+       git_zstream z;
        enum { z_unused, z_used, z_done, z_error } z_state;
 
        union {

diff --git a/zlib.c b/zlib.c
index c4d58da4e9050c6330ff145914cc379f0600f703..3c63d480c75e9939fb3a047f595b032e9509d681 100644 (file)
--- a/zlib.c
+++ b/zlib.c
@@ -4,58 +4,248 @@
  */
 #include "cache.h"
 
-void git_inflate_init(z_streamp strm)
+static const char *zerr_to_string(int status)
 {
-       const char *err;
+       switch (status) {
+       case Z_MEM_ERROR:
+               return "out of memory";
+       case Z_VERSION_ERROR:
+               return "wrong version";
+       case Z_NEED_DICT:
+               return "needs dictionary";
+       case Z_DATA_ERROR:
+               return "data stream error";
+       case Z_STREAM_ERROR:
+               return "stream consistency error";
+       default:
+               return "unknown error";
+       }
+}
 
-       switch (inflateInit(strm)) {
-       case Z_OK:
+/*
+ * avail_in and avail_out in zlib are counted in uInt, which typically
+ * limits the size of the buffer we can use to 4GB when interacting
+ * with zlib in a single call to inflate/deflate.
+ */
+/* #define ZLIB_BUF_MAX ((uInt)-1) */
+#define ZLIB_BUF_MAX ((uInt) 1024 * 1024 * 1024) /* 1GB */
+static inline uInt zlib_buf_cap(unsigned long len)
+{
+       return (ZLIB_BUF_MAX < len) ? ZLIB_BUF_MAX : len;
+}
+
+static void zlib_pre_call(git_zstream *s)
+{
+       s->z.next_in = s->next_in;
+       s->z.next_out = s->next_out;
+       s->z.total_in = s->total_in;
+       s->z.total_out = s->total_out;
+       s->z.avail_in = zlib_buf_cap(s->avail_in);
+       s->z.avail_out = zlib_buf_cap(s->avail_out);
+}
+
+static void zlib_post_call(git_zstream *s)
+{
+       unsigned long bytes_consumed;
+       unsigned long bytes_produced;
+
+       bytes_consumed = s->z.next_in - s->next_in;
+       bytes_produced = s->z.next_out - s->next_out;
+       if (s->z.total_out != s->total_out + bytes_produced)
+               die("BUG: total_out mismatch");
+       if (s->z.total_in != s->total_in + bytes_consumed)
+               die("BUG: total_in mismatch");
+
+       s->total_out = s->z.total_out;
+       s->total_in = s->z.total_in;
+       s->next_in = s->z.next_in;
+       s->next_out = s->z.next_out;
+       s->avail_in -= bytes_consumed;
+       s->avail_out -= bytes_produced;
+}
+
+void git_inflate_init(git_zstream *strm)
+{
+       int status;
+
+       zlib_pre_call(strm);
+       status = inflateInit(&strm->z);
+       zlib_post_call(strm);
+       if (status == Z_OK)
                return;
+       die("inflateInit: %s (%s)", zerr_to_string(status),
+           strm->z.msg ? strm->z.msg : "no message");
+}
 
-       case Z_MEM_ERROR:
-               err = "out of memory";
-               break;
-       case Z_VERSION_ERROR:
-               err = "wrong version";
+void git_inflate_init_gzip_only(git_zstream *strm)
+{
+       /*
+        * Use default 15 bits, +16 is to accept only gzip and to
+        * yield Z_DATA_ERROR when fed zlib format.
+        */
+       const int windowBits = 15 + 16;
+       int status;
+
+       zlib_pre_call(strm);
+       status = inflateInit2(&strm->z, windowBits);
+       zlib_post_call(strm);
+       if (status == Z_OK)
+               return;
+       die("inflateInit2: %s (%s)", zerr_to_string(status),
+           strm->z.msg ? strm->z.msg : "no message");
+}
+
+void git_inflate_end(git_zstream *strm)
+{
+       int status;
+
+       zlib_pre_call(strm);
+       status = inflateEnd(&strm->z);
+       zlib_post_call(strm);
+       if (status == Z_OK)
+               return;
+       error("inflateEnd: %s (%s)", zerr_to_string(status),
+             strm->z.msg ? strm->z.msg : "no message");
+}
+
+int git_inflate(git_zstream *strm, int flush)
+{
+       int status;
+
+       for (;;) {
+               zlib_pre_call(strm);
+               /* Never say Z_FINISH unless we are feeding everything */
+               status = inflate(&strm->z,
+                                (strm->z.avail_in != strm->avail_in)
+                                ? 0 : flush);
+               if (status == Z_MEM_ERROR)
+                       die("inflate: out of memory");
+               zlib_post_call(strm);
+
+               /*
+                * Let zlib work another round, while we can still
+                * make progress.
+                */
+               if ((strm->avail_out && !strm->z.avail_out) &&
+                   (status == Z_OK || status == Z_BUF_ERROR))
+                       continue;
                break;
+       }
+
+       switch (status) {
+       /* Z_BUF_ERROR: normal, needs more space in the output buffer */
+       case Z_BUF_ERROR:
+       case Z_OK:
+       case Z_STREAM_END:
+               return status;
        default:
-               err = "error";
+               break;
        }
-       die("inflateInit: %s (%s)", err, strm->msg ? strm->msg : "no message");
+       error("inflate: %s (%s)", zerr_to_string(status),
+             strm->z.msg ? strm->z.msg : "no message");
+       return status;
 }
 
-void git_inflate_end(z_streamp strm)
+#if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200
+#define deflateBound(c,s)  ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11)
+#endif
+
+unsigned long git_deflate_bound(git_zstream *strm, unsigned long size)
 {
-       if (inflateEnd(strm) != Z_OK)
-               error("inflateEnd: %s", strm->msg ? strm->msg : "failed");
+       return deflateBound(&strm->z, size);
 }
 
-int git_inflate(z_streamp strm, int flush)
+void git_deflate_init(git_zstream *strm, int level)
 {
-       int ret = inflate(strm, flush);
-       const char *err;
+       int status;
 
-       switch (ret) {
-       /* Out of memory is fatal. */
-       case Z_MEM_ERROR:
-               die("inflate: out of memory");
+       zlib_pre_call(strm);
+       status = deflateInit(&strm->z, level);
+       zlib_post_call(strm);
+       if (status == Z_OK)
+               return;
+       die("deflateInit: %s (%s)", zerr_to_string(status),
+           strm->z.msg ? strm->z.msg : "no message");
+}
 
-       /* Data corruption errors: we may want to recover from them (fsck) */
-       case Z_NEED_DICT:
-               err = "needs dictionary"; break;
-       case Z_DATA_ERROR:
-               err = "data stream error"; break;
-       case Z_STREAM_ERROR:
-               err = "stream consistency error"; break;
-       default:
-               err = "unknown error"; break;
+void git_deflate_init_gzip(git_zstream *strm, int level)
+{
+       /*
+        * Use default 15 bits, +16 is to generate gzip header/trailer
+        * instead of the zlib wrapper.
+        */
+       const int windowBits = 15 + 16;
+       int status;
 
+       zlib_pre_call(strm);
+       status = deflateInit2(&strm->z, level,
+                                 Z_DEFLATED, windowBits,
+                                 8, Z_DEFAULT_STRATEGY);
+       zlib_post_call(strm);
+       if (status == Z_OK)
+               return;
+       die("deflateInit2: %s (%s)", zerr_to_string(status),
+           strm->z.msg ? strm->z.msg : "no message");
+}
+
+void git_deflate_end(git_zstream *strm)
+{
+       int status;
+
+       zlib_pre_call(strm);
+       status = deflateEnd(&strm->z);
+       zlib_post_call(strm);
+       if (status == Z_OK)
+               return;
+       error("deflateEnd: %s (%s)", zerr_to_string(status),
+             strm->z.msg ? strm->z.msg : "no message");
+}
+
+int git_deflate_end_gently(git_zstream *strm)
+{
+       int status;
+
+       zlib_pre_call(strm);
+       status = deflateEnd(&strm->z);
+       zlib_post_call(strm);
+       return status;
+}
+
+int git_deflate(git_zstream *strm, int flush)
+{
+       int status;
+
+       for (;;) {
+               zlib_pre_call(strm);
+
+               /* Never say Z_FINISH unless we are feeding everything */
+               status = deflate(&strm->z,
+                                (strm->z.avail_in != strm->avail_in)
+                                ? 0 : flush);
+               if (status == Z_MEM_ERROR)
+                       die("deflate: out of memory");
+               zlib_post_call(strm);
+
+               /*
+                * Let zlib work another round, while we can still
+                * make progress.
+                */
+               if ((strm->avail_out && !strm->z.avail_out) &&
+                   (status == Z_OK || status == Z_BUF_ERROR))
+                       continue;
+               break;
+       }
+
+       switch (status) {
        /* Z_BUF_ERROR: normal, needs more space in the output buffer */
        case Z_BUF_ERROR:
        case Z_OK:
        case Z_STREAM_END:
-               return ret;
+               return status;
+       default:
+               break;
        }
-       error("inflate: %s (%s)", err, strm->msg ? strm->msg : "no message");
-       return ret;
+       error("deflate: %s (%s)", zerr_to_string(status),
+             strm->z.msg ? strm->z.msg : "no message");
+       return status;
 }