sha1_file: add the ability to parse objects in "pack file format"

The pack-file format is slightly different from the traditional git
object format, in that it has a much denser binary header encoding.
The traditional format uses an ASCII string with type and length
information, which is somewhat wasteful.

A new object format starts with uncompressed binary header
followed by compressed payload -- this will allow us later to
copy the payload straight to packfiles.

Obviously they cannot be read by older versions of git, so for
now new object files are created with the traditional format.
core.legacyheaders configuration item, when set to false makes
the code write in new format for people to experiment with.

Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/Documentation/config.txt b/Documentation/config.txt
index 0b434c1f1923fb10b2420a3295e59fd9d163f39b..9780c89bc34408c7127ddead848cf6fd6f6274a4 100644 (file)
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -97,6 +97,12 @@ core.compression::
        compression, and 1..9 are various speed/size tradeoffs, 9 being
        slowest.
 
        compression, and 1..9 are various speed/size tradeoffs, 9 being
        slowest.
 

+core.legacyheaders::
+       A boolean which enables the legacy object header format in case
+       you want to interoperate with old clients accessing the object
+       database directly (where the "http://" and "rsync://" protocols
+       count as direct access).
+

 alias.*::
        Command aliases for the gitlink:git[1] command wrapper - e.g.
        after defining "alias.last = cat-file commit HEAD", the invocation
 alias.*::
        Command aliases for the gitlink:git[1] command wrapper - e.g.
        after defining "alias.last = cat-file commit HEAD", the invocation

diff --git a/cache.h b/cache.h
index d433d46f23c956aa13b9f0d97344f792234e94ab..eee5fc9f8d75af20a1d03d569c702980f6c04469 100644 (file)
--- a/cache.h
+++ b/cache.h
@@ -176,6 +176,7 @@ extern int commit_lock_file(struct lock_file *);
 extern void rollback_lock_file(struct lock_file *);
 
 /* Environment bits from configuration mechanism */
 extern void rollback_lock_file(struct lock_file *);
 
 /* Environment bits from configuration mechanism */

+extern int use_legacy_headers;

 extern int trust_executable_bit;
 extern int assume_unchanged;
 extern int prefer_symlink_refs;
 extern int trust_executable_bit;
 extern int assume_unchanged;
 extern int prefer_symlink_refs;

diff --git a/config.c b/config.c
index 8445f7dcab3bdf96326b8d6a3e1b8c8a8161cb23..0ac6aebbbcbd666d5bd8201ce28e1868bd6d5a30 100644 (file)
--- a/config.c
+++ b/config.c
@@ -279,6 +279,11 @@ int git_default_config(const char *var, const char *value)
                return 0;
        }
 
                return 0;
        }
 

+       if (!strcmp(var, "core.legacyheaders")) {
+               use_legacy_headers = git_config_bool(var, value);
+               return 0;
+       }
+

        if (!strcmp(var, "core.compression")) {
                int level = git_config_int(var, value);
                if (level == -1)
        if (!strcmp(var, "core.compression")) {
                int level = git_config_int(var, value);
                if (level == -1)

diff --git a/environment.c b/environment.c
index 97d42b172be5abbada30da97893c77e67a0bcde0..42f39d657ef0056667bb8814622e0dde2733e373 100644 (file)
--- a/environment.c
+++ b/environment.c
@@ -11,6 +11,7 @@
 
 char git_default_email[MAX_GITNAME];
 char git_default_name[MAX_GITNAME];
 
 char git_default_email[MAX_GITNAME];
 char git_default_name[MAX_GITNAME];

+int use_legacy_headers = 1;

 int trust_executable_bit = 1;
 int assume_unchanged = 0;
 int prefer_symlink_refs = 0;
 int trust_executable_bit = 1;
 int assume_unchanged = 0;
 int prefer_symlink_refs = 0;

diff --git a/sha1_file.c b/sha1_file.c
index 8734d501fec50dc16b298e1473749c64ec25fe81..88a2579412336dae3ae456de05a20fb53b71c116 100644 (file)
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -684,26 +684,74 @@ static void *map_sha1_file_internal(const unsigned char *sha1,
        return map;
 }
 
        return map;
 }
 

-static int unpack_sha1_header(z_stream *stream, void *map, unsigned long mapsize, void *buffer, unsigned long size)
+static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)

 {
 {

+       unsigned char c;
+       unsigned int word, bits;
+       unsigned long size;
+       static const char *typename[8] = {
+               NULL,   /* OBJ_EXT */
+               "commit", "tree", "blob", "tag",
+               NULL, NULL, NULL
+       };
+       const char *type;
+

        /* Get the data stream */
        memset(stream, 0, sizeof(*stream));
        stream->next_in = map;
        stream->avail_in = mapsize;
        stream->next_out = buffer;
        /* Get the data stream */
        memset(stream, 0, sizeof(*stream));
        stream->next_in = map;
        stream->avail_in = mapsize;
        stream->next_out = buffer;

-       stream->avail_out = size;
+       stream->avail_out = bufsiz;
+
+       /*
+        * Is it a zlib-compressed buffer? If so, the first byte
+        * must be 0x78 (15-bit window size, deflated), and the
+        * first 16-bit word is evenly divisible by 31
+        */
+       word = (map[0] << 8) + map[1];
+       if (map[0] == 0x78 && !(word % 31)) {
+               inflateInit(stream);
+               return inflate(stream, 0);
+       }
+
+       c = *map++;
+       mapsize--;
+       type = typename[(c >> 4) & 7];
+       if (!type)
+               return -1;
+
+       bits = 4;
+       size = c & 0xf;
+       while ((c & 0x80)) {
+               if (bits >= 8*sizeof(long))
+                       return -1;
+               c = *map++;
+               size += (c & 0x7f) << bits;
+               bits += 7;
+               mapsize--;
+       }

 
 

+       /* Set up the stream for the rest.. */
+       stream->next_in = map;
+       stream->avail_in = mapsize;

        inflateInit(stream);
        inflateInit(stream);

-       return inflate(stream, 0);
+
+       /* And generate the fake traditional header */
+       stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu", type, size);
+       return 0;

 }
 
 static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size)
 {
        int bytes = strlen(buffer) + 1;
        unsigned char *buf = xmalloc(1+size);
 }
 
 static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size)
 {
        int bytes = strlen(buffer) + 1;
        unsigned char *buf = xmalloc(1+size);

+       unsigned long n;

 
 

-       memcpy(buf, (char *) buffer + bytes, stream->total_out - bytes);
-       bytes = stream->total_out - bytes;
+       n = stream->total_out - bytes;
+       if (n > size)
+               n = size;
+       memcpy(buf, (char *) buffer + bytes, n);
+       bytes = n;

        if (bytes < size) {
                stream->next_out = buf + bytes;
                stream->avail_out = size - bytes;
        if (bytes < size) {
                stream->next_out = buf + bytes;
                stream->avail_out = size - bytes;


@@ -1414,6 +1462,49 @@ static int write_buffer(int fd, const void *buf, size_t len)
        return 0;
 }
 
        return 0;
 }
 

+static int write_binary_header(unsigned char *hdr, enum object_type type, unsigned long len)
+{
+       int hdr_len;
+       unsigned char c;
+
+       c = (type << 4) | (len & 15);
+       len >>= 4;
+       hdr_len = 1;
+       while (len) {
+               *hdr++ = c | 0x80;
+               hdr_len++;
+               c = (len & 0x7f);
+               len >>= 7;
+       }
+       *hdr = c;
+       return hdr_len;
+}
+
+static void setup_object_header(z_stream *stream, const char *type, unsigned long len)
+{
+       int obj_type, hdr;
+
+       if (use_legacy_headers) {
+               while (deflate(stream, 0) == Z_OK)
+                       /* nothing */;
+               return;
+       }
+       if (!strcmp(type, blob_type))
+               obj_type = OBJ_BLOB;
+       else if (!strcmp(type, tree_type))
+               obj_type = OBJ_TREE;
+       else if (!strcmp(type, commit_type))
+               obj_type = OBJ_COMMIT;
+       else if (!strcmp(type, tag_type))
+               obj_type = OBJ_TAG;
+       else
+               die("trying to generate bogus object of type '%s'", type);
+       hdr = write_binary_header(stream->next_out, obj_type, len);
+       stream->total_out = hdr;
+       stream->next_out += hdr;
+       stream->avail_out -= hdr;
+}
+

 int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
 {
        int size;
 int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
 {
        int size;


@@ -1459,7 +1550,7 @@ int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned cha
        /* Set it up */
        memset(&stream, 0, sizeof(stream));
        deflateInit(&stream, zlib_compression_level);
        /* Set it up */
        memset(&stream, 0, sizeof(stream));
        deflateInit(&stream, zlib_compression_level);

-       size = deflateBound(&stream, len+hdrlen);
+       size = 8 + deflateBound(&stream, len+hdrlen);

        compressed = xmalloc(size);
 
        /* Compress it */
        compressed = xmalloc(size);
 
        /* Compress it */


@@ -1469,8 +1560,7 @@ int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned cha
        /* First header.. */
        stream.next_in = hdr;
        stream.avail_in = hdrlen;
        /* First header.. */
        stream.next_in = hdr;
        stream.avail_in = hdrlen;

-       while (deflate(&stream, 0) == Z_OK)
-               /* nothing */;
+       setup_object_header(&stream, type, len);

 
        /* Then the data itself.. */
        stream.next_in = buf;
 
        /* Then the data itself.. */
        stream.next_in = buf;