Implemented tree delta compression in fast-import.
[gitweb.git] / fast-import.c
index e692f6b430429d8452d2f6f1793f7d360055f410..6b011204150111d40cf8eb50117b24236eb214c8 100644 (file)
@@ -4,9 +4,9 @@ Format of STDIN stream:
   stream ::= cmd*;
 
   cmd ::= new_blob
-        | new_branch
         | new_commit
         | new_tag
+        | reset_branch
         ;
 
   new_blob ::= 'blob' lf
@@ -14,15 +14,12 @@ Format of STDIN stream:
     file_content;
   file_content ::= data;
 
-  new_branch ::= 'branch' sp ref_str lf
-    ('from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)?
-    lf;
-
   new_commit ::= 'commit' sp ref_str lf
-       mark?
-       ('author' sp name '<' email '>' ts tz lf)?
-       'committer' sp name '<' email '>' ts tz lf
-       commit_msg
+    mark?
+    ('author' sp name '<' email '>' ts tz lf)?
+    'committer' sp name '<' email '>' ts tz lf
+    commit_msg
+    ('from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)?
     file_change*
     lf;
   commit_msg ::= data;
@@ -38,6 +35,8 @@ Format of STDIN stream:
     tag_msg;
   tag_msg ::= data;
 
+  reset_branch ::= 'reset' sp ref_str lf;
+
      # note: the first idnum in a stream should be 1 and subsequent
      # idnums should not have gaps between values as this will cause
      # the stream parser to reserve space for the gapped values.  An
@@ -133,7 +132,7 @@ struct mark_set
 struct last_object
 {
        void *data;
-       unsigned int len;
+       unsigned long len;
        unsigned int depth;
        unsigned char sha1[20];
 };
@@ -158,14 +157,18 @@ struct tree_entry
 {
        struct tree_content *tree;
        struct atom_str* name;
-       unsigned int mode;
-       unsigned char sha1[20];
+       struct tree_entry_ms
+       {
+               unsigned int mode;
+               unsigned char sha1[20];
+       } versions[2];
 };
 
 struct tree_content
 {
        unsigned int entry_capacity; /* must match avail_tree_content */
        unsigned int entry_count;
+       unsigned int delta_depth;
        struct tree_entry *entries[FLEX_ARRAY]; /* more */
 };
 
@@ -198,11 +201,13 @@ static unsigned long max_depth = 10;
 static unsigned long alloc_count;
 static unsigned long branch_count;
 static unsigned long branch_load_count;
+static unsigned long remap_count;
 static unsigned long object_count;
 static unsigned long duplicate_count;
 static unsigned long marks_set_count;
 static unsigned long object_count_by_type[9];
 static unsigned long duplicate_count_by_type[9];
+static unsigned long delta_count_by_type[9];
 
 /* Memory pools */
 static size_t mem_pool_alloc = 2*1024*1024 - sizeof(struct mem_pool);
@@ -216,14 +221,19 @@ static struct atom_str **atom_table;
 
 /* The .pack file being generated */
 static int pack_fd;
-static unsigned long pack_offset;
+static unsigned long pack_size;
 static unsigned char pack_sha1[20];
+static unsigned char* pack_base;
+static unsigned long pack_moff;
+static unsigned long pack_mlen = 128*1024*1024;
+static unsigned long page_size;
 
 /* Table of objects we've written. */
-static unsigned int object_entry_alloc = 1000;
+static unsigned int object_entry_alloc = 5000;
 static struct object_entry_pool *blocks;
 static struct object_entry *object_table[1 << 16];
 static struct mark_set *marks;
+static const char* mark_file;
 
 /* Our last blob */
 static struct last_object last_blob;
@@ -248,6 +258,7 @@ static struct tag *last_tag;
 /* Input stream parsing */
 static struct strbuf command_buf;
 static unsigned long next_mark;
+static FILE* branch_log;
 
 
 static void alloc_objects(int cnt)
@@ -271,7 +282,7 @@ static struct object_entry* new_object(unsigned char *sha1)
                alloc_objects(object_entry_alloc);
 
        e = blocks->next_free++;
-       memcpy(e->sha1, sha1, sizeof(e->sha1));
+       hashcpy(e->sha1, sha1);
        return e;
 }
 
@@ -280,7 +291,7 @@ static struct object_entry* find_object(unsigned char *sha1)
        unsigned int h = sha1[0] << 8 | sha1[1];
        struct object_entry *e;
        for (e = object_table[h]; e; e = e->next)
-               if (!memcmp(sha1, e->sha1, sizeof(e->sha1)))
+               if (!hashcmp(sha1, e->sha1))
                        return e;
        return NULL;
 }
@@ -292,7 +303,7 @@ static struct object_entry* insert_object(unsigned char *sha1)
        struct object_entry *p = NULL;
 
        while (e) {
-               if (!memcmp(sha1, e->sha1, sizeof(e->sha1)))
+               if (!hashcmp(sha1, e->sha1))
                        return e;
                p = e;
                e = e->next;
@@ -339,6 +350,9 @@ static void* pool_alloc(size_t len)
        }
 
        r = p->next_free;
+       /* round out to a pointer alignment */
+       if (len & (sizeof(void*) - 1))
+               len += sizeof(void*) - (len & (sizeof(void*) - 1));
        p->next_free += len;
        return r;
 }
@@ -477,6 +491,7 @@ static struct tree_content* new_tree_content(unsigned int cnt)
 
        t = (struct tree_content*)f;
        t->entry_count = 0;
+       t->delta_depth = 0;
        return t;
 }
 
@@ -503,6 +518,7 @@ static struct tree_content* grow_tree_content(
 {
        struct tree_content *r = new_tree_content(t->entry_count + amt);
        r->entry_count = t->entry_count;
+       r->delta_depth = t->delta_depth;
        memcpy(r->entries,t->entries,t->entry_count*sizeof(t->entries[0]));
        release_tree_content(t);
        return r;
@@ -514,11 +530,13 @@ static struct tree_entry* new_tree_entry()
 
        if (!avail_tree_entry) {
                unsigned int n = tree_entry_alloc;
+               total_allocd += n * sizeof(struct tree_entry);
                avail_tree_entry = e = xmalloc(n * sizeof(struct tree_entry));
-               while (n--) {
+               while (n-- > 1) {
                        *((void**)e) = e + 1;
                        e++;
                }
+               *((void**)e) = NULL;
        }
 
        e = avail_tree_entry;
@@ -605,7 +623,7 @@ static int store_object(
        SHA1_Update(&c, dat, datlen);
        SHA1_Final(sha1, &c);
        if (sha1out)
-               memcpy(sha1out, sha1, sizeof(sha1));
+               hashcpy(sha1out, sha1);
 
        e = insert_object(sha1);
        if (mark)
@@ -616,7 +634,7 @@ static int store_object(
                return 1;
        }
        e->type = type;
-       e->offset = pack_offset;
+       e->offset = pack_size;
        object_count++;
        object_count_by_type[type]++;
 
@@ -631,13 +649,14 @@ static int store_object(
        deflateInit(&s, zlib_compression_level);
 
        if (delta) {
+               delta_count_by_type[type]++;
                last->depth++;
                s.next_in = delta;
                s.avail_in = deltalen;
                hdrlen = encode_header(OBJ_DELTA, deltalen, hdr);
                ywrite(pack_fd, hdr, hdrlen);
                ywrite(pack_fd, last->sha1, sizeof(sha1));
-               pack_offset += hdrlen + sizeof(sha1);
+               pack_size += hdrlen + sizeof(sha1);
        } else {
                if (last)
                        last->depth = 0;
@@ -645,7 +664,7 @@ static int store_object(
                s.avail_in = datlen;
                hdrlen = encode_header(type, datlen, hdr);
                ywrite(pack_fd, hdr, hdrlen);
-               pack_offset += hdrlen;
+               pack_size += hdrlen;
        }
 
        s.avail_out = deflateBound(&s, s.avail_in);
@@ -655,7 +674,7 @@ static int store_object(
        deflateEnd(&s);
 
        ywrite(pack_fd, out, s.total_out);
-       pack_offset += s.total_out;
+       pack_size += s.total_out;
 
        free(out);
        if (delta)
@@ -665,11 +684,140 @@ static int store_object(
                        free(last->data);
                last->data = dat;
                last->len = datlen;
-               memcpy(last->sha1, sha1, sizeof(sha1));
+               hashcpy(last->sha1, sha1);
        }
        return 0;
 }
 
+static unsigned char* map_pack(unsigned long offset, unsigned int *left)
+{
+       if (offset >= pack_size)
+               die("object offset outside of pack file");
+       if (!pack_base
+                       || offset < pack_moff
+                       || (offset + 20) >= (pack_moff + pack_mlen)) {
+               if (pack_base)
+                       munmap(pack_base, pack_mlen);
+               pack_moff = (offset / page_size) * page_size;
+               pack_base = mmap(NULL,pack_mlen,PROT_READ,MAP_SHARED,
+                       pack_fd,pack_moff);
+               if (pack_base == MAP_FAILED)
+                       die("Failed to map generated pack: %s", strerror(errno));
+               remap_count++;
+       }
+       offset -= pack_moff;
+       if (left)
+               *left = pack_mlen - offset;
+       return pack_base + offset;
+}
+
+static unsigned long unpack_object_header(unsigned long offset,
+       enum object_type *type,
+       unsigned long *sizep)
+{
+       unsigned shift;
+       unsigned char c;
+       unsigned long size;
+
+       c = *map_pack(offset++, NULL);
+       *type = (c >> 4) & 7;
+       size = c & 15;
+       shift = 4;
+       while (c & 0x80) {
+               c = *map_pack(offset++, NULL);
+               size += (c & 0x7f) << shift;
+               shift += 7;
+       }
+       *sizep = size;
+       return offset;
+}
+
+static void *unpack_non_delta_entry(unsigned long o, unsigned long sz)
+{
+       z_stream stream;
+       unsigned char *result;
+
+       result = xmalloc(sz + 1);
+       result[sz] = 0;
+
+       memset(&stream, 0, sizeof(stream));
+       stream.next_in = map_pack(o, &stream.avail_in);
+       stream.next_out = result;
+       stream.avail_out = sz;
+
+       inflateInit(&stream);
+       for (;;) {
+               int st = inflate(&stream, Z_FINISH);
+               if (st == Z_STREAM_END)
+                       break;
+               if (st == Z_OK || st == Z_BUF_ERROR) {
+                       o = stream.next_in - pack_base + pack_moff;
+                       stream.next_in = map_pack(o, &stream.avail_in);
+                       continue;
+               }
+               die("Error %i from zlib during inflate.", st);
+       }
+       inflateEnd(&stream);
+       if (stream.total_out != sz)
+               die("Error after inflate: sizes mismatch");
+       return result;
+}
+
+static void *unpack_entry(unsigned long offset,
+       unsigned long *sizep,
+       unsigned int *delta_depth);
+
+static void *unpack_delta_entry(unsigned long offset,
+       unsigned long delta_size,
+       unsigned long *sizep,
+       unsigned int *delta_depth)
+{
+       struct object_entry *base_oe;
+       unsigned char *base_sha1;
+       void *delta_data, *base, *result;
+       unsigned long base_size, result_size;
+
+       base_sha1 = map_pack(offset, NULL);
+       base_oe = find_object(base_sha1);
+       if (!base_oe)
+               die("I'm broken; I can't find a base I know must be here.");
+       base = unpack_entry(base_oe->offset, &base_size, delta_depth);
+       delta_data = unpack_non_delta_entry(offset + 20, delta_size);
+       result = patch_delta(base, base_size,
+                            delta_data, delta_size,
+                            &result_size);
+       if (!result)
+               die("failed to apply delta");
+       free(delta_data);
+       free(base);
+       *sizep = result_size;
+       (*delta_depth)++;
+       return result;
+}
+
+static void *unpack_entry(unsigned long offset,
+       unsigned long *sizep,
+       unsigned int *delta_depth)
+{
+       unsigned long size;
+       enum object_type kind;
+
+       offset = unpack_object_header(offset, &kind, &size);
+       switch (kind) {
+       case OBJ_DELTA:
+               return unpack_delta_entry(offset, size, sizep, delta_depth);
+       case OBJ_COMMIT:
+       case OBJ_TREE:
+       case OBJ_BLOB:
+       case OBJ_TAG:
+               *sizep = size;
+               *delta_depth = 0;
+               return unpack_non_delta_entry(offset, size);
+       default:
+               die("I created an object I can't read!");
+       }
+}
+
 static const char *get_mode(const char *str, unsigned int *modep)
 {
        unsigned char c;
@@ -686,24 +834,27 @@ static const char *get_mode(const char *str, unsigned int *modep)
 
 static void load_tree(struct tree_entry *root)
 {
+       unsigned char* sha1 = root->versions[1].sha1;
        struct object_entry *myoe;
        struct tree_content *t;
        unsigned long size;
        char *buf;
        const char *c;
-       char type[20];
 
        root->tree = t = new_tree_content(8);
-       if (!memcmp(root->sha1, null_sha1, 20))
+       if (is_null_sha1(sha1))
                return;
 
-       myoe = find_object(root->sha1);
+       myoe = find_object(sha1);
        if (myoe) {
-               die("FIXME");
+               if (myoe->type != OBJ_TREE)
+                       die("Not a tree: %s", sha1_to_hex(sha1));
+               buf = unpack_entry(myoe->offset, &size, &t->delta_depth);
        } else {
-               buf = read_sha1_file(root->sha1, type, &size);
+               char type[20];
+               buf = read_sha1_file(sha1, type, &size);
                if (!buf || strcmp(type, tree_type))
-                       die("Can't load existing tree %s", sha1_to_hex(root->sha1));
+                       die("Can't load tree %s", sha1_to_hex(sha1));
        }
 
        c = buf;
@@ -715,56 +866,116 @@ static void load_tree(struct tree_entry *root)
                t->entries[t->entry_count++] = e;
 
                e->tree = NULL;
-               c = get_mode(c, &e->mode);
+               c = get_mode(c, &e->versions[1].mode);
                if (!c)
-                       die("Corrupt mode in %s", sha1_to_hex(root->sha1));
+                       die("Corrupt mode in %s", sha1_to_hex(sha1));
+               e->versions[0].mode = e->versions[1].mode;
                e->name = to_atom(c, strlen(c));
                c += e->name->str_len + 1;
-               memcpy(e->sha1, c, sizeof(e->sha1));
+               hashcpy(e->versions[0].sha1, (unsigned char*)c);
+               hashcpy(e->versions[1].sha1, (unsigned char*)c);
                c += 20;
        }
        free(buf);
 }
 
-static int tecmp (const void *_a, const void *_b)
+static int tecmp0 (const void *_a, const void *_b)
 {
        struct tree_entry *a = *((struct tree_entry**)_a);
        struct tree_entry *b = *((struct tree_entry**)_b);
        return base_name_compare(
-               a->name->str_dat, a->name->str_len, a->mode,
-               b->name->str_dat, b->name->str_len, b->mode);
+               a->name->str_dat, a->name->str_len, a->versions[0].mode,
+               b->name->str_dat, b->name->str_len, b->versions[0].mode);
 }
 
-static void store_tree(struct tree_entry *root)
+static int tecmp1 (const void *_a, const void *_b)
 {
-       struct tree_content *t = root->tree;
+       struct tree_entry *a = *((struct tree_entry**)_a);
+       struct tree_entry *b = *((struct tree_entry**)_b);
+       return base_name_compare(
+               a->name->str_dat, a->name->str_len, a->versions[1].mode,
+               b->name->str_dat, b->name->str_len, b->versions[1].mode);
+}
+
+static void* mktree(struct tree_content *t, int v, unsigned long *szp)
+{
+       size_t maxlen = 0;
        unsigned int i;
-       size_t maxlen;
        char *buf, *c;
 
-       if (memcmp(root->sha1, null_sha1, 20))
-               return;
+       if (!v)
+               qsort(t->entries,t->entry_count,sizeof(t->entries[0]),tecmp0);
+       else
+               qsort(t->entries,t->entry_count,sizeof(t->entries[0]),tecmp1);
 
-       maxlen = 0;
        for (i = 0; i < t->entry_count; i++) {
-               maxlen += t->entries[i]->name->str_len + 34;
-               if (t->entries[i]->tree)
-                       store_tree(t->entries[i]);
+               if (t->entries[i]->versions[v].mode)
+                       maxlen += t->entries[i]->name->str_len + 34;
        }
 
-       qsort(t->entries, t->entry_count, sizeof(t->entries[0]), tecmp);
        buf = c = xmalloc(maxlen);
        for (i = 0; i < t->entry_count; i++) {
                struct tree_entry *e = t->entries[i];
-               c += sprintf(c, "%o", e->mode);
+               if (!e->versions[v].mode)
+                       continue;
+               c += sprintf(c, "%o", e->versions[v].mode);
                *c++ = ' ';
                strcpy(c, e->name->str_dat);
                c += e->name->str_len + 1;
-               memcpy(c, e->sha1, 20);
+               hashcpy((unsigned char*)c, e->versions[v].sha1);
                c += 20;
        }
-       store_object(OBJ_TREE, buf, c - buf, NULL, root->sha1, 0);
-       free(buf);
+
+       *szp = c - buf;
+       return buf;
+}
+
+static void store_tree(struct tree_entry *root)
+{
+       struct tree_content *t = root->tree;
+       unsigned int i, j, del;
+       unsigned long vers1len;
+       void **vers1dat;
+       struct last_object lo;
+
+       if (!is_null_sha1(root->versions[1].sha1))
+               return;
+
+       for (i = 0; i < t->entry_count; i++) {
+               if (t->entries[i]->tree)
+                       store_tree(t->entries[i]);
+       }
+
+       if (is_null_sha1(root->versions[0].sha1)
+                       || !find_object(root->versions[0].sha1)) {
+               lo.data = NULL;
+               lo.depth = 0;
+       } else {
+               lo.data = mktree(t, 0, &lo.len);
+               lo.depth = t->delta_depth;
+               hashcpy(lo.sha1, root->versions[0].sha1);
+       }
+       vers1dat = mktree(t, 1, &vers1len);
+
+       store_object(OBJ_TREE, vers1dat, vers1len,
+               &lo, root->versions[1].sha1, 0);
+       /* note: lo.dat (if created) was freed by store_object */
+       free(vers1dat);
+
+       t->delta_depth = lo.depth;
+       hashcpy(root->versions[0].sha1, root->versions[1].sha1);
+       for (i = 0, j = 0, del = 0; i < t->entry_count; i++) {
+               struct tree_entry *e = t->entries[i];
+               if (e->versions[1].mode) {
+                       e->versions[0].mode = e->versions[1].mode;
+                       hashcpy(e->versions[0].sha1, e->versions[1].sha1);
+                       t->entries[j++] = e;
+               } else {
+                       release_tree_entry(e);
+                       del++;
+               }
+       }
+       t->entry_count -= del;
 }
 
 static int tree_content_set(
@@ -788,25 +999,26 @@ static int tree_content_set(
                e = t->entries[i];
                if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) {
                        if (!slash1) {
-                               if (e->mode == mode && !memcmp(e->sha1, sha1, 20))
+                               if (e->versions[1].mode == mode
+                                               && !hashcmp(e->versions[1].sha1, sha1))
                                        return 0;
-                               e->mode = mode;
-                               memcpy(e->sha1, sha1, 20);
+                               e->versions[1].mode = mode;
+                               hashcpy(e->versions[1].sha1, sha1);
                                if (e->tree) {
                                        release_tree_content_recursive(e->tree);
                                        e->tree = NULL;
                                }
-                               memcpy(root->sha1, null_sha1, 20);
+                               hashclr(root->versions[1].sha1);
                                return 1;
                        }
-                       if (!S_ISDIR(e->mode)) {
+                       if (!S_ISDIR(e->versions[1].mode)) {
                                e->tree = new_tree_content(8);
-                               e->mode = S_IFDIR;
+                               e->versions[1].mode = S_IFDIR;
                        }
                        if (!e->tree)
                                load_tree(e);
                        if (tree_content_set(e, slash1 + 1, sha1, mode)) {
-                               memcpy(root->sha1, null_sha1, 20);
+                               hashclr(root->versions[1].sha1);
                                return 1;
                        }
                        return 0;
@@ -817,17 +1029,19 @@ static int tree_content_set(
                root->tree = t = grow_tree_content(t, 8);
        e = new_tree_entry();
        e->name = to_atom(p, n);
+       e->versions[0].mode = 0;
+       hashclr(e->versions[0].sha1);
        t->entries[t->entry_count++] = e;
        if (slash1) {
                e->tree = new_tree_content(8);
-               e->mode = S_IFDIR;
+               e->versions[1].mode = S_IFDIR;
                tree_content_set(e, slash1 + 1, sha1, mode);
        } else {
                e->tree = NULL;
-               e->mode = mode;
-               memcpy(e->sha1, sha1, 20);
+               e->versions[1].mode = mode;
+               hashcpy(e->versions[1].sha1, sha1);
        }
-       memcpy(root->sha1, null_sha1, 20);
+       hashclr(root->versions[1].sha1);
        return 1;
 }
 
@@ -847,14 +1061,14 @@ static int tree_content_remove(struct tree_entry *root, const char *p)
        for (i = 0; i < t->entry_count; i++) {
                e = t->entries[i];
                if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) {
-                       if (!slash1 || !S_ISDIR(e->mode))
+                       if (!slash1 || !S_ISDIR(e->versions[1].mode))
                                goto del_entry;
                        if (!e->tree)
                                load_tree(e);
                        if (tree_content_remove(e, slash1 + 1)) {
                                if (!e->tree->entry_count)
                                        goto del_entry;
-                               memcpy(root->sha1, null_sha1, 20);
+                               hashclr(root->versions[1].sha1);
                                return 1;
                        }
                        return 0;
@@ -863,11 +1077,13 @@ static int tree_content_remove(struct tree_entry *root, const char *p)
        return 0;
 
 del_entry:
-       for (i++; i < t->entry_count; i++)
-               t->entries[i-1] = t->entries[i];
-       t->entry_count--;
-       release_tree_entry(e);
-       memcpy(root->sha1, null_sha1, 20);
+       if (e->tree) {
+               release_tree_content_recursive(e->tree);
+               e->tree = NULL;
+       }
+       e->versions[1].mode = 0;
+       hashclr(e->versions[1].sha1);
+       hashclr(root->versions[1].sha1);
        return 1;
 }
 
@@ -880,7 +1096,7 @@ static void init_pack_header()
        hdr.hdr_entries = 0;
 
        ywrite(pack_fd, &hdr, sizeof(hdr));
-       pack_offset = sizeof(hdr);
+       pack_size = sizeof(hdr);
 }
 
 static void fixup_header_footer()
@@ -919,7 +1135,7 @@ static int oecmp (const void *_a, const void *_b)
 {
        struct object_entry *a = *((struct object_entry**)_a);
        struct object_entry *b = *((struct object_entry**)_b);
-       return memcmp(a->sha1, b->sha1, sizeof(a->sha1));
+       return hashcmp(a->sha1, b->sha1);
 }
 
 static void write_index(const char *idx_name)
@@ -996,6 +1212,36 @@ static void dump_tags()
        }
 }
 
+static void dump_marks_helper(FILE *f,
+       unsigned long base,
+       struct mark_set *m)
+{
+       int k;
+       if (m->shift) {
+               for (k = 0; k < 1024; k++) {
+                       if (m->data.sets[k])
+                               dump_marks_helper(f, (base + k) << m->shift,
+                                       m->data.sets[k]);
+               }
+       } else {
+               for (k = 0; k < 1024; k++) {
+                       if (m->data.marked[k])
+                               fprintf(f, ":%lu %s\n", base + k,
+                                       sha1_to_hex(m->data.marked[k]->sha1));
+               }
+       }
+}
+
+static void dump_marks()
+{
+       if (mark_file)
+       {
+               FILE *f = fopen(mark_file, "w");
+               dump_marks_helper(f, 0, marks);
+               fclose(f);
+       }
+}
+
 static void read_next_command()
 {
        read_line(&command_buf, stdin, '\n');
@@ -1052,7 +1298,8 @@ static void cmd_new_blob()
 
 static void unload_one_branch()
 {
-       while (cur_active_branches >= max_active_branches) {
+       while (cur_active_branches
+               && cur_active_branches >= max_active_branches) {
                unsigned long min_commit = ULONG_MAX;
                struct branch *e, *l = NULL, *p = NULL;
 
@@ -1169,6 +1416,77 @@ static void file_change_d(struct branch *b)
                free(p_uq);
 }
 
+static void cmd_from(struct branch *b)
+{
+       const char *from, *endp;
+       char *str_uq;
+       struct branch *s;
+
+       if (strncmp("from ", command_buf.buf, 5))
+               return;
+
+       if (b->last_commit)
+               die("Can't reinitailize branch %s", b->name);
+
+       from = strchr(command_buf.buf, ' ') + 1;
+       str_uq = unquote_c_style(from, &endp);
+       if (str_uq) {
+               if (*endp)
+                       die("Garbage after string in: %s", command_buf.buf);
+               from = str_uq;
+       }
+
+       s = lookup_branch(from);
+       if (b == s)
+               die("Can't create a branch from itself: %s", b->name);
+       else if (s) {
+               unsigned char *t = s->branch_tree.versions[1].sha1;
+               hashcpy(b->sha1, s->sha1);
+               hashcpy(b->branch_tree.versions[0].sha1, t);
+               hashcpy(b->branch_tree.versions[1].sha1, t);
+       } else if (*from == ':') {
+               unsigned long idnum = strtoul(from + 1, NULL, 10);
+               struct object_entry *oe = find_mark(idnum);
+               unsigned long size;
+               unsigned int depth;
+               char *buf;
+               if (oe->type != OBJ_COMMIT)
+                       die("Mark :%lu not a commit", idnum);
+               hashcpy(b->sha1, oe->sha1);
+               buf = unpack_entry(oe->offset, &size, &depth);
+               if (!buf || size < 46)
+                       die("Not a valid commit: %s", from);
+               if (memcmp("tree ", buf, 5)
+                       || get_sha1_hex(buf + 5, b->branch_tree.versions[1].sha1))
+                       die("The commit %s is corrupt", sha1_to_hex(b->sha1));
+               free(buf);
+               hashcpy(b->branch_tree.versions[0].sha1,
+                       b->branch_tree.versions[1].sha1);
+       } else if (!get_sha1(from, b->sha1)) {
+               if (is_null_sha1(b->sha1)) {
+                       hashclr(b->branch_tree.versions[0].sha1);
+                       hashclr(b->branch_tree.versions[1].sha1);
+               } else {
+                       unsigned long size;
+                       char *buf;
+
+                       buf = read_object_with_reference(b->sha1,
+                               type_names[OBJ_COMMIT], &size, b->sha1);
+                       if (!buf || size < 46)
+                               die("Not a valid commit: %s", from);
+                       if (memcmp("tree ", buf, 5)
+                               || get_sha1_hex(buf + 5, b->branch_tree.versions[1].sha1))
+                               die("The commit %s is corrupt", sha1_to_hex(b->sha1));
+                       free(buf);
+                       hashcpy(b->branch_tree.versions[0].sha1,
+                               b->branch_tree.versions[1].sha1);
+               }
+       } else
+               die("Invalid ref name or SHA1 expression: %s", from);
+
+       read_next_command();
+}
+
 static void cmd_new_commit()
 {
        struct branch *b;
@@ -1191,7 +1509,7 @@ static void cmd_new_commit()
        }
        b = lookup_branch(sp);
        if (!b)
-               die("Branch not declared: %s", sp);
+               b = new_branch(sp);
        if (str_uq)
                free(str_uq);
 
@@ -1208,16 +1526,17 @@ static void cmd_new_commit()
        if (!committer)
                die("Expected committer but didn't get one");
        msg = cmd_data(&msglen);
+       read_next_command();
+       cmd_from(b);
 
        /* ensure the branch is active/loaded */
-       if (!b->branch_tree.tree) {
+       if (!b->branch_tree.tree || !max_active_branches) {
                unload_one_branch();
                load_branch(b);
        }
 
        /* file_change* */
        for (;;) {
-               read_next_command();
                if (1 == command_buf.len)
                        break;
                else if (!strncmp("M ", command_buf.buf, 2))
@@ -1226,6 +1545,7 @@ static void cmd_new_commit()
                        file_change_d(b);
                else
                        die("Unsupported file_change: %s", command_buf.buf);
+               read_next_command();
        }
 
        /* build the tree and the commit */
@@ -1235,8 +1555,9 @@ static void cmd_new_commit()
                        ? strlen(author) + strlen(committer)
                        : 2 * strlen(committer)));
        sp = body;
-       sp += sprintf(sp, "tree %s\n", sha1_to_hex(b->branch_tree.sha1));
-       if (memcmp(b->sha1, null_sha1, 20))
+       sp += sprintf(sp, "tree %s\n",
+               sha1_to_hex(b->branch_tree.versions[1].sha1));
+       if (!is_null_sha1(b->sha1))
                sp += sprintf(sp, "parent %s\n", sha1_to_hex(b->sha1));
        if (author)
                sp += sprintf(sp, "%s\n", author);
@@ -1253,83 +1574,18 @@ static void cmd_new_commit()
        store_object(OBJ_COMMIT, body, sp - body, NULL, b->sha1, next_mark);
        free(body);
        b->last_commit = object_count_by_type[OBJ_COMMIT];
-}
-
-static void cmd_new_branch()
-{
-       struct branch *b;
-       char *str_uq;
-       const char *endp;
-       char *sp;
-
-       /* Obtain the new branch name from the rest of our command */
-       sp = strchr(command_buf.buf, ' ') + 1;
-       str_uq = unquote_c_style(sp, &endp);
-       if (str_uq) {
-               if (*endp)
-                       die("Garbage after ref in: %s", command_buf.buf);
-               sp = str_uq;
-       }
-       b = new_branch(sp);
-       if (str_uq)
-               free(str_uq);
-       read_next_command();
-
-       /* from ... */
-       if (!strncmp("from ", command_buf.buf, 5)) {
-               const char *from;
-               struct branch *s;
-
-               from = strchr(command_buf.buf, ' ') + 1;
-               str_uq = unquote_c_style(from, &endp);
-               if (str_uq) {
-                       if (*endp)
-                               die("Garbage after string in: %s", command_buf.buf);
-                       from = str_uq;
-               }
 
-               s = lookup_branch(from);
-               if (b == s)
-                       die("Can't create a branch from itself: %s", b->name);
-               else if (s) {
-                       memcpy(b->sha1, s->sha1, 20);
-                       memcpy(b->branch_tree.sha1, s->branch_tree.sha1, 20);
-               } else if (*from == ':') {
-                       unsigned long idnum = strtoul(from + 1, NULL, 10);
-                       struct object_entry *oe = find_mark(idnum);
-                       if (oe->type != OBJ_COMMIT)
-                               die("Mark :%lu not a commit", idnum);
-                       memcpy(b->sha1, oe->sha1, 20);
-                       memcpy(b->branch_tree.sha1, null_sha1, 20);
-               } else if (!get_sha1(from, b->sha1)) {
-                       if (!memcmp(b->sha1, null_sha1, 20))
-                               memcpy(b->branch_tree.sha1, null_sha1, 20);
-                       else {
-                               unsigned long size;
-                               char *buf;
-
-                               buf = read_object_with_reference(b->sha1,
-                                       type_names[OBJ_COMMIT], &size, b->sha1);
-                               if (!buf || size < 46)
-                                       die("Not a valid commit: %s", from);
-                               if (memcmp("tree ", buf, 5)
-                                       || get_sha1_hex(buf + 5, b->branch_tree.sha1))
-                                       die("The commit %s is corrupt", sha1_to_hex(b->sha1));
-                               free(buf);
-                       }
+       if (branch_log) {
+               int need_dq = quote_c_style(b->name, NULL, NULL, 0);
+               fprintf(branch_log, "commit ");
+               if (need_dq) {
+                       fputc('"', branch_log);
+                       quote_c_style(b->name, NULL, branch_log, 0);
+                       fputc('"', branch_log);
                } else
-                       die("Invalid ref name or SHA1 expression: %s", from);
-
-               if (str_uq)
-                       free(str_uq);
-               read_next_command();
-       } else {
-               memcpy(b->sha1, null_sha1, 20);
-               memcpy(b->branch_tree.sha1, null_sha1, 20);
+                       fprintf(branch_log, "%s", b->name);
+               fprintf(branch_log," :%lu %s\n",next_mark,sha1_to_hex(b->sha1));
        }
-
-       if (command_buf.eof || command_buf.len > 1)
-               die("An lf did not terminate the branch command as expected.");
 }
 
 static void cmd_new_tag()
@@ -1344,6 +1600,7 @@ static void cmd_new_tag()
        size_t msglen;
        char *body;
        struct tag *t;
+       unsigned long from_mark = 0;
        unsigned char sha1[20];
 
        /* Obtain the new tag name from the rest of our command */
@@ -1380,13 +1637,13 @@ static void cmd_new_tag()
 
        s = lookup_branch(from);
        if (s) {
-               memcpy(sha1, s->sha1, 20);
+               hashcpy(sha1, s->sha1);
        } else if (*from == ':') {
-               unsigned long idnum = strtoul(from + 1, NULL, 10);
-               struct object_entry *oe = find_mark(idnum);
+               from_mark = strtoul(from + 1, NULL, 10);
+               struct object_entry *oe = find_mark(from_mark);
                if (oe->type != OBJ_COMMIT)
-                       die("Mark :%lu not a commit", idnum);
-               memcpy(sha1, oe->sha1, 20);
+                       die("Mark :%lu not a commit", from_mark);
+               hashcpy(sha1, oe->sha1);
        } else if (!get_sha1(from, sha1)) {
                unsigned long size;
                char *buf;
@@ -1426,10 +1683,49 @@ static void cmd_new_tag()
 
        store_object(OBJ_TAG, body, sp - body, NULL, t->sha1, 0);
        free(body);
+
+       if (branch_log) {
+               int need_dq = quote_c_style(t->name, NULL, NULL, 0);
+               fprintf(branch_log, "tag ");
+               if (need_dq) {
+                       fputc('"', branch_log);
+                       quote_c_style(t->name, NULL, branch_log, 0);
+                       fputc('"', branch_log);
+               } else
+                       fprintf(branch_log, "%s", t->name);
+               fprintf(branch_log," :%lu %s\n",from_mark,sha1_to_hex(t->sha1));
+       }
+}
+
+static void cmd_reset_branch()
+{
+       struct branch *b;
+       char *str_uq;
+       const char *endp;
+       char *sp;
+
+       /* Obtain the branch name from the rest of our command */
+       sp = strchr(command_buf.buf, ' ') + 1;
+       str_uq = unquote_c_style(sp, &endp);
+       if (str_uq) {
+               if (*endp)
+                       die("Garbage after ref in: %s", command_buf.buf);
+               sp = str_uq;
+       }
+       b = lookup_branch(sp);
+       if (b) {
+               b->last_commit = 0;
+               if (b->branch_tree.tree) {
+                       release_tree_content_recursive(b->branch_tree.tree);
+                       b->branch_tree.tree = NULL;
+               }
+       }
+       if (str_uq)
+               free(str_uq);
 }
 
 static const char fast_import_usage[] =
-"git-fast-import [--objects=n] [--depth=n] [--active-branches=n] temp.pack";
+"git-fast-import [--objects=n] [--depth=n] [--active-branches=n] [--export-marks=marks.file] [--branch-log=log] temp.pack";
 
 int main(int argc, const char **argv)
 {
@@ -1442,6 +1738,7 @@ int main(int argc, const char **argv)
 
        setup_ident();
        git_config(git_default_config);
+       page_size = getpagesize();
 
        for (i = 1; i < argc; i++) {
                const char *a = argv[i];
@@ -1454,6 +1751,13 @@ int main(int argc, const char **argv)
                        max_depth = strtoul(a + 8, NULL, 0);
                else if (!strncmp(a, "--active-branches=", 18))
                        max_active_branches = strtoul(a + 18, NULL, 0);
+               else if (!strncmp(a, "--export-marks=", 15))
+                       mark_file = a + 15;
+               else if (!strncmp(a, "--branch-log=", 13)) {
+                       branch_log = fopen(a + 13, "w");
+                       if (!branch_log)
+                               die("Can't create %s: %s", a + 13, strerror(errno));
+               }
                else
                        die("unknown option %s", a);
        }
@@ -1485,12 +1789,12 @@ int main(int argc, const char **argv)
                        break;
                else if (!strcmp("blob", command_buf.buf))
                        cmd_new_blob();
-               else if (!strncmp("branch ", command_buf.buf, 7))
-                       cmd_new_branch();
                else if (!strncmp("commit ", command_buf.buf, 7))
                        cmd_new_commit();
                else if (!strncmp("tag ", command_buf.buf, 4))
                        cmd_new_tag();
+               else if (!strncmp("reset ", command_buf.buf, 6))
+                       cmd_reset_branch();
                else
                        die("Unsupported command: %s", command_buf.buf);
        }
@@ -1500,21 +1804,25 @@ int main(int argc, const char **argv)
        write_index(idx_name);
        dump_branches();
        dump_tags();
+       dump_marks();
+       if (branch_log)
+               fclose(branch_log);
 
        fprintf(stderr, "%s statistics:\n", argv[0]);
        fprintf(stderr, "---------------------------------------------------\n");
        fprintf(stderr, "Alloc'd objects: %10lu (%10lu overflow  )\n", alloc_count, alloc_count - est_obj_cnt);
        fprintf(stderr, "Total objects:   %10lu (%10lu duplicates)\n", object_count, duplicate_count);
-       fprintf(stderr, "      blobs  :   %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB]);
-       fprintf(stderr, "      trees  :   %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE]);
-       fprintf(stderr, "      commits:   %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT]);
-       fprintf(stderr, "      tags   :   %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG]);
+       fprintf(stderr, "      blobs  :   %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB]);
+       fprintf(stderr, "      trees  :   %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE]);
+       fprintf(stderr, "      commits:   %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT]);
+       fprintf(stderr, "      tags   :   %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG]);
        fprintf(stderr, "Total branches:  %10lu (%10lu loads     )\n", branch_count, branch_load_count);
        fprintf(stderr, "      marks:     %10u (%10lu unique    )\n", (1 << marks->shift) * 1024, marks_set_count);
        fprintf(stderr, "      atoms:     %10u\n", atom_cnt);
        fprintf(stderr, "Memory total:    %10lu KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);
        fprintf(stderr, "       pools:    %10lu KiB\n", total_allocd/1024);
        fprintf(stderr, "     objects:    %10lu KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
+       fprintf(stderr, "Pack remaps:     %10lu\n", remap_count);
        fprintf(stderr, "---------------------------------------------------\n");
 
        stat(pack_name, &sb);