git-repack --max-pack-size: write_{object,one}() respect pack limit

With --max-pack-size,  generate the appropriate write limit
for each object and check against it before each group of writes.
Update delta usability rules to handle base being in a previously-
written pack.  Inline sha1write_compress() so we know the
exact size of the written data when it needs to be compressed.
Detect and return write "failure".

Signed-off-by: Dana L. How <danahow@gmail.com>
Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c
index 7c17ea04d33743dc67476a14525b1aea4f22268b..3023aace4d8725374f935dd92b63aa34154dc191 100644 (file)
--- a/builtin-pack-objects.c
+++ b/builtin-pack-objects.c
@@ -352,16 +352,31 @@ static void copy_pack_data(struct sha1file *f,
 }
 
 static unsigned long write_object(struct sha1file *f,
 }
 
 static unsigned long write_object(struct sha1file *f,

-                                 struct object_entry *entry)
+                                 struct object_entry *entry,
+                                 off_t write_offset)

 {
        unsigned long size;
        enum object_type type;
        void *buf;
        unsigned char header[10];
 {
        unsigned long size;
        enum object_type type;
        void *buf;
        unsigned char header[10];

+       unsigned char dheader[10];

        unsigned hdrlen;
        off_t datalen;
        enum object_type obj_type;
        int to_reuse = 0;
        unsigned hdrlen;
        off_t datalen;
        enum object_type obj_type;
        int to_reuse = 0;

+       /* write limit if limited packsize and not first object */
+       unsigned long limit = pack_size_limit && nr_written ?
+                               pack_size_limit - write_offset : 0;
+                               /* no if no delta */
+       int usable_delta =      !entry->delta ? 0 :
+                               /* yes if unlimited packfile */
+                               !pack_size_limit ? 1 :
+                               /* no if base written to previous pack */
+                               entry->delta->offset == (off_t)-1 ? 0 :
+                               /* otherwise double-check written to this
+                                * pack,  like we do below
+                                */
+                               entry->delta->offset ? 1 : 0;

 
        if (!pack_to_stdout)
                crc32_begin(f);
 
        if (!pack_to_stdout)
                crc32_begin(f);


@@ -372,7 +387,9 @@ static unsigned long write_object(struct sha1file *f,
        else if (!entry->in_pack)
                to_reuse = 0;   /* can't reuse what we don't have */
        else if (obj_type == OBJ_REF_DELTA || obj_type == OBJ_OFS_DELTA)
        else if (!entry->in_pack)
                to_reuse = 0;   /* can't reuse what we don't have */
        else if (obj_type == OBJ_REF_DELTA || obj_type == OBJ_OFS_DELTA)

-               to_reuse = 1;   /* check_object() decided it for us */
+                               /* check_object() decided it for us ... */
+               to_reuse = usable_delta;
+                               /* ... but pack split may override that */

        else if (obj_type != entry->in_pack_type)
                to_reuse = 0;   /* pack has delta which is unusable */
        else if (entry->delta)
        else if (obj_type != entry->in_pack_type)
                to_reuse = 0;   /* pack has delta which is unusable */
        else if (entry->delta)


@@ -383,24 +400,48 @@ static unsigned long write_object(struct sha1file *f,
                                 */
 
        if (!to_reuse) {
                                 */
 
        if (!to_reuse) {

+               z_stream stream;
+               unsigned long maxsize;
+               void *out;

                buf = read_sha1_file(entry->sha1, &type, &size);
                if (!buf)
                        die("unable to read %s", sha1_to_hex(entry->sha1));
                if (size != entry->size)
                        die("object %s size inconsistency (%lu vs %lu)",
                            sha1_to_hex(entry->sha1), size, entry->size);
                buf = read_sha1_file(entry->sha1, &type, &size);
                if (!buf)
                        die("unable to read %s", sha1_to_hex(entry->sha1));
                if (size != entry->size)
                        die("object %s size inconsistency (%lu vs %lu)",
                            sha1_to_hex(entry->sha1), size, entry->size);

-               if (entry->delta) {
+               if (usable_delta) {

                        buf = delta_against(buf, size, entry);
                        size = entry->delta_size;
                        obj_type = (allow_ofs_delta && entry->delta->offset) ?
                                OBJ_OFS_DELTA : OBJ_REF_DELTA;
                        buf = delta_against(buf, size, entry);
                        size = entry->delta_size;
                        obj_type = (allow_ofs_delta && entry->delta->offset) ?
                                OBJ_OFS_DELTA : OBJ_REF_DELTA;

+               } else {
+                       /*
+                        * recover real object type in case
+                        * check_object() wanted to re-use a delta,
+                        * but we couldn't since base was in previous split pack
+                        */
+                       obj_type = type;

                }
                }

+               /* compress the data to store and put compressed length in datalen */
+               memset(&stream, 0, sizeof(stream));
+               deflateInit(&stream, pack_compression_level);
+               maxsize = deflateBound(&stream, size);
+               out = xmalloc(maxsize);
+               /* Compress it */
+               stream.next_in = buf;
+               stream.avail_in = size;
+               stream.next_out = out;
+               stream.avail_out = maxsize;
+               while (deflate(&stream, Z_FINISH) == Z_OK)
+                       /* nothing */;
+               deflateEnd(&stream);
+               datalen = stream.total_out;
+               deflateEnd(&stream);

                /*
                 * The object header is a byte of 'type' followed by zero or
                 * more bytes of length.
                 */
                hdrlen = encode_header(obj_type, size, header);
                /*
                 * The object header is a byte of 'type' followed by zero or
                 * more bytes of length.
                 */
                hdrlen = encode_header(obj_type, size, header);

-               sha1write(f, header, hdrlen);

 
                if (obj_type == OBJ_OFS_DELTA) {
                        /*
 
                if (obj_type == OBJ_OFS_DELTA) {
                        /*


@@ -409,21 +450,41 @@ static unsigned long write_object(struct sha1file *f,
                         * base from this object's position in the pack.
                         */
                        off_t ofs = entry->offset - entry->delta->offset;
                         * base from this object's position in the pack.
                         */
                        off_t ofs = entry->offset - entry->delta->offset;

-                       unsigned pos = sizeof(header) - 1;
-                       header[pos] = ofs & 127;
+                       unsigned pos = sizeof(dheader) - 1;
+                       dheader[pos] = ofs & 127;

                        while (ofs >>= 7)
                        while (ofs >>= 7)

-                               header[--pos] = 128 | (--ofs & 127);
-                       sha1write(f, header + pos, sizeof(header) - pos);
-                       hdrlen += sizeof(header) - pos;
+                               dheader[--pos] = 128 | (--ofs & 127);
+                       if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) {
+                               free(out);
+                               free(buf);
+                               return 0;
+                       }
+                       sha1write(f, header, hdrlen);
+                       sha1write(f, dheader + pos, sizeof(dheader) - pos);
+                       hdrlen += sizeof(dheader) - pos;

                } else if (obj_type == OBJ_REF_DELTA) {
                        /*
                         * Deltas with a base reference contain
                         * an additional 20 bytes for the base sha1.
                         */
                } else if (obj_type == OBJ_REF_DELTA) {
                        /*
                         * Deltas with a base reference contain
                         * an additional 20 bytes for the base sha1.
                         */

+                       if (limit && hdrlen + 20 + datalen + 20 >= limit) {
+                               free(out);
+                               free(buf);
+                               return 0;
+                       }
+                       sha1write(f, header, hdrlen);

                        sha1write(f, entry->delta->sha1, 20);
                        hdrlen += 20;
                        sha1write(f, entry->delta->sha1, 20);
                        hdrlen += 20;

+               } else {
+                       if (limit && hdrlen + datalen + 20 >= limit) {
+                               free(out);
+                               free(buf);
+                               return 0;
+                       }
+                       sha1write(f, header, hdrlen);

                }
                }

-               datalen = sha1write_compressed(f, buf, size, pack_compression_level);
+               sha1write(f, out, datalen);
+               free(out);

                free(buf);
        }
        else {
                free(buf);
        }
        else {


@@ -438,20 +499,6 @@ static unsigned long write_object(struct sha1file *f,
                        reused_delta++;
                }
                hdrlen = encode_header(obj_type, entry->size, header);
                        reused_delta++;
                }
                hdrlen = encode_header(obj_type, entry->size, header);

-               sha1write(f, header, hdrlen);
-               if (obj_type == OBJ_OFS_DELTA) {
-                       off_t ofs = entry->offset - entry->delta->offset;
-                       unsigned pos = sizeof(header) - 1;
-                       header[pos] = ofs & 127;
-                       while (ofs >>= 7)
-                               header[--pos] = 128 | (--ofs & 127);
-                       sha1write(f, header + pos, sizeof(header) - pos);
-                       hdrlen += sizeof(header) - pos;
-               } else if (obj_type == OBJ_REF_DELTA) {
-                       sha1write(f, entry->delta->sha1, 20);
-                       hdrlen += 20;
-               }
-

                offset = entry->in_pack_offset;
                revidx = find_packed_object(p, offset);
                datalen = revidx[1].offset - offset;
                offset = entry->in_pack_offset;
                revidx = find_packed_object(p, offset);
                datalen = revidx[1].offset - offset;


@@ -460,6 +507,29 @@ static unsigned long write_object(struct sha1file *f,
                        die("bad packed object CRC for %s", sha1_to_hex(entry->sha1));
                offset += entry->in_pack_header_size;
                datalen -= entry->in_pack_header_size;
                        die("bad packed object CRC for %s", sha1_to_hex(entry->sha1));
                offset += entry->in_pack_header_size;
                datalen -= entry->in_pack_header_size;

+               if (obj_type == OBJ_OFS_DELTA) {
+                       off_t ofs = entry->offset - entry->delta->offset;
+                       unsigned pos = sizeof(dheader) - 1;
+                       dheader[pos] = ofs & 127;
+                       while (ofs >>= 7)
+                               dheader[--pos] = 128 | (--ofs & 127);
+                       if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit)
+                               return 0;
+                       sha1write(f, header, hdrlen);
+                       sha1write(f, dheader + pos, sizeof(dheader) - pos);
+                       hdrlen += sizeof(dheader) - pos;
+               } else if (obj_type == OBJ_REF_DELTA) {
+                       if (limit && hdrlen + 20 + datalen + 20 >= limit)
+                               return 0;
+                       sha1write(f, header, hdrlen);
+                       sha1write(f, entry->delta->sha1, 20);
+                       hdrlen += 20;
+               } else {
+                       if (limit && hdrlen + datalen + 20 >= limit)
+                               return 0;
+                       sha1write(f, header, hdrlen);
+               }
+

                if (!pack_to_stdout && p->index_version == 1 &&
                    check_pack_inflate(p, &w_curs, offset, datalen, entry->size))
                        die("corrupt packed object for %s", sha1_to_hex(entry->sha1));
                if (!pack_to_stdout && p->index_version == 1 &&
                    check_pack_inflate(p, &w_curs, offset, datalen, entry->size))
                        die("corrupt packed object for %s", sha1_to_hex(entry->sha1));


@@ -467,7 +537,7 @@ static unsigned long write_object(struct sha1file *f,
                unuse_pack(&w_curs);
                reused++;
        }
                unuse_pack(&w_curs);
                reused++;
        }

-       if (entry->delta)
+       if (usable_delta)

                written_delta++;
        written++;
        if (!pack_to_stdout)
                written_delta++;
        written++;
        if (!pack_to_stdout)


@@ -486,11 +556,18 @@ static off_t write_one(struct sha1file *f,
                return offset;
 
        /* if we are deltified, write out base object first. */
                return offset;
 
        /* if we are deltified, write out base object first. */

-       if (e->delta)
+       if (e->delta) {

                offset = write_one(f, e->delta, offset);
                offset = write_one(f, e->delta, offset);

+               if (!offset)
+                       return 0;
+       }

 
        e->offset = offset;
 
        e->offset = offset;

-       size = write_object(f, e);
+       size = write_object(f, e, offset);
+       if (!size) {
+               e->offset = 0;
+               return 0;
+       }

 
        /* make sure off_t is sufficiently large not to wrap */
        if (offset > offset + size)
 
        /* make sure off_t is sufficiently large not to wrap */
        if (offset > offset + size)