trim_common_tail: brown paper bag fix.
[gitweb.git] / index-pack.c
index 58c4a9c41dd7a05b86d40e6eeee33ba0a3fb6c4f..9fd6982a979a40e701dcb6bcaf117eafbf6ff161 100644 (file)
@@ -13,13 +13,11 @@ static const char index_pack_usage[] =
 
 struct object_entry
 {
-       off_t offset;
+       struct pack_idx_entry idx;
        unsigned long size;
        unsigned int hdr_size;
-       uint32_t crc32;
        enum object_type type;
        enum object_type real_type;
-       unsigned char sha1[20];
 };
 
 union delta_base {
@@ -48,7 +46,7 @@ static int nr_resolved_deltas;
 static int from_stdin;
 static int verbose;
 
-static struct progress progress;
+static struct progress *progress;
 
 /* We always read in 4kB chunks. */
 static unsigned char input_buffer[4096];
@@ -90,6 +88,8 @@ static void *fill(int min)
                        die("read error on input: %s", strerror(errno));
                }
                input_len += ret;
+               if (from_stdin)
+                       display_throughput(progress, consumed_bytes + input_len);
        } while (input_len < min);
        return input_buffer;
 }
@@ -108,7 +108,7 @@ static void use(int bytes)
        consumed_bytes += bytes;
 }
 
-static const char *open_pack_file(const char *pack_name)
+static char *open_pack_file(char *pack_name)
 {
        if (from_stdin) {
                input_fd = 0;
@@ -116,7 +116,7 @@ static const char *open_pack_file(const char *pack_name)
                        static char tmpfile[PATH_MAX];
                        snprintf(tmpfile, sizeof(tmpfile),
                                 "%s/tmp_pack_XXXXXX", get_object_directory());
-                       output_fd = mkstemp(tmpfile);
+                       output_fd = xmkstemp(tmpfile);
                        pack_name = xstrdup(tmpfile);
                } else
                        output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600);
@@ -197,7 +197,7 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
        unsigned shift;
        void *data;
 
-       obj->offset = consumed_bytes;
+       obj->idx.offset = consumed_bytes;
        input_crc32 = crc32(0, Z_NULL, 0);
 
        p = fill(1);
@@ -229,15 +229,15 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
                while (c & 128) {
                        base_offset += 1;
                        if (!base_offset || MSB(base_offset, 7))
-                               bad_object(obj->offset, "offset value overflow for delta base object");
+                               bad_object(obj->idx.offset, "offset value overflow for delta base object");
                        p = fill(1);
                        c = *p;
                        use(1);
                        base_offset = (base_offset << 7) + (c & 127);
                }
-               delta_base->offset = obj->offset - base_offset;
-               if (delta_base->offset >= obj->offset)
-                       bad_object(obj->offset, "delta base offset is out of bound");
+               delta_base->offset = obj->idx.offset - base_offset;
+               if (delta_base->offset >= obj->idx.offset)
+                       bad_object(obj->idx.offset, "delta base offset is out of bound");
                break;
        case OBJ_COMMIT:
        case OBJ_TREE:
@@ -245,19 +245,19 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
        case OBJ_TAG:
                break;
        default:
-               bad_object(obj->offset, "unknown object type %d", obj->type);
+               bad_object(obj->idx.offset, "unknown object type %d", obj->type);
        }
-       obj->hdr_size = consumed_bytes - obj->offset;
+       obj->hdr_size = consumed_bytes - obj->idx.offset;
 
-       data = unpack_entry_data(obj->offset, obj->size);
-       obj->crc32 = input_crc32;
+       data = unpack_entry_data(obj->idx.offset, obj->size);
+       obj->idx.crc32 = input_crc32;
        return data;
 }
 
 static void *get_data_from_pack(struct object_entry *obj)
 {
-       unsigned long from = obj[0].offset + obj[0].hdr_size;
-       unsigned long len = obj[1].offset - from;
+       off_t from = obj[0].idx.offset + obj[0].hdr_size;
+       unsigned long len = obj[1].idx.offset - from;
        unsigned long rdy = 0;
        unsigned char *src, *data;
        z_stream stream;
@@ -360,11 +360,11 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data,
                             &result_size);
        free(delta_data);
        if (!result)
-               bad_object(delta_obj->offset, "failed to apply delta");
-       sha1_object(result, result_size, type, delta_obj->sha1);
+               bad_object(delta_obj->idx.offset, "failed to apply delta");
+       sha1_object(result, result_size, type, delta_obj->idx.sha1);
        nr_resolved_deltas++;
 
-       hashcpy(delta_base.sha1, delta_obj->sha1);
+       hashcpy(delta_base.sha1, delta_obj->idx.sha1);
        if (!find_delta_children(&delta_base, &first, &last)) {
                for (j = first; j <= last; j++) {
                        struct object_entry *child = objects + deltas[j].obj_no;
@@ -374,7 +374,7 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data,
        }
 
        memset(&delta_base, 0, sizeof(delta_base));
-       delta_base.offset = delta_obj->offset;
+       delta_base.offset = delta_obj->idx.offset;
        if (!find_delta_children(&delta_base, &first, &last)) {
                for (j = first; j <= last; j++) {
                        struct object_entry *child = objects + deltas[j].obj_no;
@@ -408,7 +408,9 @@ static void parse_pack_objects(unsigned char *sha1)
         * - remember base (SHA1 or offset) for all deltas.
         */
        if (verbose)
-               start_progress(&progress, "Indexing %u objects...", "", nr_objects);
+               progress = start_progress(
+                               from_stdin ? "Receiving objects" : "Indexing objects",
+                               nr_objects);
        for (i = 0; i < nr_objects; i++) {
                struct object_entry *obj = &objects[i];
                data = unpack_raw_entry(obj, &delta->base);
@@ -418,14 +420,12 @@ static void parse_pack_objects(unsigned char *sha1)
                        delta->obj_no = i;
                        delta++;
                } else
-                       sha1_object(data, obj->size, obj->type, obj->sha1);
+                       sha1_object(data, obj->size, obj->type, obj->idx.sha1);
                free(data);
-               if (verbose)
-                       display_progress(&progress, i+1);
+               display_progress(progress, i+1);
        }
-       objects[i].offset = consumed_bytes;
-       if (verbose)
-               stop_progress(&progress);
+       objects[i].idx.offset = consumed_bytes;
+       stop_progress(&progress);
 
        /* Check pack integrity */
        flush();
@@ -457,7 +457,7 @@ static void parse_pack_objects(unsigned char *sha1)
         *   for some more deltas.
         */
        if (verbose)
-               start_progress(&progress, "Resolving %u deltas...", "", nr_deltas);
+               progress = start_progress("Resolving deltas", nr_deltas);
        for (i = 0; i < nr_objects; i++) {
                struct object_entry *obj = &objects[i];
                union delta_base base;
@@ -465,10 +465,10 @@ static void parse_pack_objects(unsigned char *sha1)
 
                if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA)
                        continue;
-               hashcpy(base.sha1, obj->sha1);
+               hashcpy(base.sha1, obj->idx.sha1);
                ref = !find_delta_children(&base, &ref_first, &ref_last);
                memset(&base, 0, sizeof(base));
-               base.offset = obj->offset;
+               base.offset = obj->idx.offset;
                ofs = !find_delta_children(&base, &ofs_first, &ofs_last);
                if (!ref && !ofs)
                        continue;
@@ -488,8 +488,7 @@ static void parse_pack_objects(unsigned char *sha1)
                                                      obj->size, obj->type);
                        }
                free(data);
-               if (verbose)
-                       display_progress(&progress, nr_resolved_deltas);
+               display_progress(progress, nr_resolved_deltas);
        }
 }
 
@@ -535,11 +534,11 @@ static void append_obj_to_pack(const unsigned char *sha1, void *buf,
        }
        header[n++] = c;
        write_or_die(output_fd, header, n);
-       obj[0].crc32 = crc32(0, Z_NULL, 0);
-       obj[0].crc32 = crc32(obj[0].crc32, header, n);
-       obj[1].offset = obj[0].offset + n;
-       obj[1].offset += write_compressed(output_fd, buf, size, &obj[0].crc32);
-       hashcpy(obj->sha1, sha1);
+       obj[0].idx.crc32 = crc32(0, Z_NULL, 0);
+       obj[0].idx.crc32 = crc32(obj[0].idx.crc32, header, n);
+       obj[1].idx.offset = obj[0].idx.offset + n;
+       obj[1].idx.offset += write_compressed(output_fd, buf, size, &obj[0].idx.crc32);
+       hashcpy(obj->idx.sha1, sha1);
 }
 
 static int delta_pos_compare(const void *_a, const void *_b)
@@ -596,151 +595,11 @@ static void fix_unresolved_deltas(int nr_unresolved)
                        die("local object %s is corrupt", sha1_to_hex(d->base.sha1));
                append_obj_to_pack(d->base.sha1, data, size, type);
                free(data);
-               if (verbose)
-                       display_progress(&progress, nr_resolved_deltas);
+               display_progress(progress, nr_resolved_deltas);
        }
        free(sorted_by_pos);
 }
 
-static uint32_t index_default_version = 1;
-static uint32_t index_off32_limit = 0x7fffffff;
-
-static int sha1_compare(const void *_a, const void *_b)
-{
-       struct object_entry *a = *(struct object_entry **)_a;
-       struct object_entry *b = *(struct object_entry **)_b;
-       return hashcmp(a->sha1, b->sha1);
-}
-
-/*
- * On entry *sha1 contains the pack content SHA1 hash, on exit it is
- * the SHA1 hash of sorted object names.
- */
-static const char *write_index_file(const char *index_name, unsigned char *sha1)
-{
-       struct sha1file *f;
-       struct object_entry **sorted_by_sha, **list, **last;
-       uint32_t array[256];
-       int i, fd;
-       SHA_CTX ctx;
-       uint32_t index_version;
-
-       if (nr_objects) {
-               sorted_by_sha =
-                       xcalloc(nr_objects, sizeof(struct object_entry *));
-               list = sorted_by_sha;
-               last = sorted_by_sha + nr_objects;
-               for (i = 0; i < nr_objects; ++i)
-                       sorted_by_sha[i] = &objects[i];
-               qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
-                     sha1_compare);
-       }
-       else
-               sorted_by_sha = list = last = NULL;
-
-       if (!index_name) {
-               static char tmpfile[PATH_MAX];
-               snprintf(tmpfile, sizeof(tmpfile),
-                        "%s/tmp_idx_XXXXXX", get_object_directory());
-               fd = mkstemp(tmpfile);
-               index_name = xstrdup(tmpfile);
-       } else {
-               unlink(index_name);
-               fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
-       }
-       if (fd < 0)
-               die("unable to create %s: %s", index_name, strerror(errno));
-       f = sha1fd(fd, index_name);
-
-       /* if last object's offset is >= 2^31 we should use index V2 */
-       index_version = (objects[nr_objects-1].offset >> 31) ? 2 : index_default_version;
-
-       /* index versions 2 and above need a header */
-       if (index_version >= 2) {
-               struct pack_idx_header hdr;
-               hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
-               hdr.idx_version = htonl(index_version);
-               sha1write(f, &hdr, sizeof(hdr));
-       }
-
-       /*
-        * Write the first-level table (the list is sorted,
-        * but we use a 256-entry lookup to be able to avoid
-        * having to do eight extra binary search iterations).
-        */
-       for (i = 0; i < 256; i++) {
-               struct object_entry **next = list;
-               while (next < last) {
-                       struct object_entry *obj = *next;
-                       if (obj->sha1[0] != i)
-                               break;
-                       next++;
-               }
-               array[i] = htonl(next - sorted_by_sha);
-               list = next;
-       }
-       sha1write(f, array, 256 * 4);
-
-       /* compute the SHA1 hash of sorted object names. */
-       SHA1_Init(&ctx);
-
-       /*
-        * Write the actual SHA1 entries..
-        */
-       list = sorted_by_sha;
-       for (i = 0; i < nr_objects; i++) {
-               struct object_entry *obj = *list++;
-               if (index_version < 2) {
-                       uint32_t offset = htonl(obj->offset);
-                       sha1write(f, &offset, 4);
-               }
-               sha1write(f, obj->sha1, 20);
-               SHA1_Update(&ctx, obj->sha1, 20);
-       }
-
-       if (index_version >= 2) {
-               unsigned int nr_large_offset = 0;
-
-               /* write the crc32 table */
-               list = sorted_by_sha;
-               for (i = 0; i < nr_objects; i++) {
-                       struct object_entry *obj = *list++;
-                       uint32_t crc32_val = htonl(obj->crc32);
-                       sha1write(f, &crc32_val, 4);
-               }
-
-               /* write the 32-bit offset table */
-               list = sorted_by_sha;
-               for (i = 0; i < nr_objects; i++) {
-                       struct object_entry *obj = *list++;
-                       uint32_t offset = (obj->offset <= index_off32_limit) ?
-                               obj->offset : (0x80000000 | nr_large_offset++);
-                       offset = htonl(offset);
-                       sha1write(f, &offset, 4);
-               }
-
-               /* write the large offset table */
-               list = sorted_by_sha;
-               while (nr_large_offset) {
-                       struct object_entry *obj = *list++;
-                       uint64_t offset = obj->offset;
-                       if (offset > index_off32_limit) {
-                               uint32_t split[2];
-                               split[0]        = htonl(offset >> 32);
-                               split[1] = htonl(offset & 0xffffffff);
-                               sha1write(f, split, 8);
-                               nr_large_offset--;
-                       }
-               }
-       }
-
-       sha1write(f, sha1, 20);
-       sha1close(f, NULL, 1);
-       free(sorted_by_sha);
-       SHA1_Final(sha1, &ctx);
-       return index_name;
-}
-
 static void final(const char *final_pack_name, const char *curr_pack_name,
                  const char *final_index_name, const char *curr_index_name,
                  const char *keep_name, const char *keep_msg,
@@ -775,7 +634,8 @@ static void final(const char *final_pack_name, const char *curr_pack_name,
                                write_or_die(keep_fd, keep_msg, keep_msg_len);
                                write_or_die(keep_fd, "\n", 1);
                        }
-                       close(keep_fd);
+                       if (close(keep_fd) != 0)
+                               die("cannot write keep file");
                        report = "keep";
                }
        }
@@ -823,17 +683,31 @@ static void final(const char *final_pack_name, const char *curr_pack_name,
        }
 }
 
+static int git_index_pack_config(const char *k, const char *v)
+{
+       if (!strcmp(k, "pack.indexversion")) {
+               pack_idx_default_version = git_config_int(k, v);
+               if (pack_idx_default_version > 2)
+                       die("bad pack.indexversion=%d", pack_idx_default_version);
+               return 0;
+       }
+       return git_default_config(k, v);
+}
+
 int main(int argc, char **argv)
 {
        int i, fix_thin_pack = 0;
-       const char *curr_pack, *pack_name = NULL;
-       const char *curr_index, *index_name = NULL;
+       char *curr_pack, *pack_name = NULL;
+       char *curr_index, *index_name = NULL;
        const char *keep_name = NULL, *keep_msg = NULL;
        char *index_name_buf = NULL, *keep_name_buf = NULL;
+       struct pack_idx_entry **idx_objects;
        unsigned char sha1[20];
 
+       git_config(git_index_pack_config);
+
        for (i = 1; i < argc; i++) {
-               const char *arg = argv[i];
+               char *arg = argv[i];
 
                if (*arg == '-') {
                        if (!strcmp(arg, "--stdin")) {
@@ -865,12 +739,12 @@ int main(int argc, char **argv)
                                index_name = argv[++i];
                        } else if (!prefixcmp(arg, "--index-version=")) {
                                char *c;
-                               index_default_version = strtoul(arg + 16, &c, 10);
-                               if (index_default_version > 2)
+                               pack_idx_default_version = strtoul(arg + 16, &c, 10);
+                               if (pack_idx_default_version > 2)
                                        die("bad %s", arg);
                                if (*c == ',')
-                                       index_off32_limit = strtoul(c+1, &c, 0);
-                               if (*c || index_off32_limit & 0x80000000)
+                                       pack_idx_off32_limit = strtoul(c+1, &c, 0);
+                               if (*c || pack_idx_off32_limit & 0x80000000)
                                        die("bad %s", arg);
                        } else
                                usage(index_pack_usage);
@@ -913,12 +787,12 @@ int main(int argc, char **argv)
        deltas = xmalloc(nr_objects * sizeof(struct delta_entry));
        parse_pack_objects(sha1);
        if (nr_deltas == nr_resolved_deltas) {
-               if (verbose)
-                       stop_progress(&progress);
+               stop_progress(&progress);
                /* Flush remaining pack final 20-byte SHA1. */
                flush();
        } else {
                if (fix_thin_pack) {
+                       char msg[48];
                        int nr_unresolved = nr_deltas - nr_resolved_deltas;
                        int nr_objects_initial = nr_objects;
                        if (nr_unresolved <= 0)
@@ -927,20 +801,24 @@ int main(int argc, char **argv)
                                           (nr_objects + nr_unresolved + 1)
                                           * sizeof(*objects));
                        fix_unresolved_deltas(nr_unresolved);
-                       if (verbose) {
-                               stop_progress(&progress);
-                               fprintf(stderr, "%d objects were added to complete this thin pack.\n",
-                                       nr_objects - nr_objects_initial);
-                       }
+                       sprintf(msg, "completed with %d local objects",
+                               nr_objects - nr_objects_initial);
+                       stop_progress_msg(&progress, msg);
                        fixup_pack_header_footer(output_fd, sha1,
-                               curr_pack, nr_objects);
+                                                curr_pack, nr_objects);
                }
                if (nr_deltas != nr_resolved_deltas)
                        die("pack has %d unresolved deltas",
                            nr_deltas - nr_resolved_deltas);
        }
        free(deltas);
-       curr_index = write_index_file(index_name, sha1);
+
+       idx_objects = xmalloc((nr_objects) * sizeof(struct pack_idx_entry *));
+       for (i = 0; i < nr_objects; i++)
+               idx_objects[i] = &objects[i].idx;
+       curr_index = write_idx_file(index_name, idx_objects, nr_objects, sha1);
+       free(idx_objects);
+
        final(pack_name, curr_pack,
                index_name, curr_index,
                keep_name, keep_msg,
@@ -948,6 +826,10 @@ int main(int argc, char **argv)
        free(objects);
        free(index_name_buf);
        free(keep_name_buf);
+       if (pack_name == NULL)
+               free(curr_pack);
+       if (index_name == NULL)
+               free(curr_index);
 
        return 0;
 }