streaming_write_entry(): use streaming API in write_entry()

When the output to a path does not have to be converted, we can read from
the object database from the streaming API and write to the file in the
working tree, without having to hold everything in the memory.

The ident, auto- and safe- crlf conversions inherently require you to read
the whole thing before deciding what to do, so while it is technically
possible to support them by using a buffer of an unbound size or rewinding
and reading the stream twice, it is less practical than the traditional
"read the whole thing in core and convert" approach.

Adding streaming filters for the other conversions on top of this should
be doable by tweaking the can_bypass_conversion() function (it should be
renamed to can_filter_stream() when it happens). Then the streaming API
can be extended to wrap the git_istream streaming_write_entry() opens on
the underlying object in another git_istream that reads from it, filters
what is read, and let the streaming_write_entry() read the filtered
result. But that is outside the scope of this series.

Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/cache.h b/cache.h
index 7650d2e691e9d7e4a9ef7aca437b59ee2a198652..2a7a77f3ebd72c25eaf1d00c94f269ccb24dde55 100644 (file)
--- a/cache.h
+++ b/cache.h
@@ -1156,6 +1156,7 @@ extern int convert_to_git(const char *path, const char *src, size_t len,
                           struct strbuf *dst, enum safe_crlf checksafe);
 extern int convert_to_working_tree(const char *path, const char *src, size_t len, struct strbuf *dst);
 extern int renormalize_buffer(const char *path, const char *src, size_t len, struct strbuf *dst);
                           struct strbuf *dst, enum safe_crlf checksafe);
 extern int convert_to_working_tree(const char *path, const char *src, size_t len, struct strbuf *dst);
 extern int renormalize_buffer(const char *path, const char *src, size_t len, struct strbuf *dst);

+extern int can_bypass_conversion(const char *path);

 
 /* add */
 /*
 
 /* add */
 /*

diff --git a/convert.c b/convert.c
index efc7e07d475c66f7835dc6cbbd3bc358f01c41c3..d3c0041820b8734c56e17b01b51a695142ec2055 100644 (file)
--- a/convert.c
+++ b/convert.c
@@ -813,3 +813,26 @@ int renormalize_buffer(const char *path, const char *src, size_t len, struct str
        }
        return ret | convert_to_git(path, src, len, dst, 0);
 }
        }
        return ret | convert_to_git(path, src, len, dst, 0);
 }

+
+/*
+ * You would be crazy to set CRLF, smuge/clean or ident to
+ * a large binary blob you would want us not to slurp into
+ * the memory!
+ */
+int can_bypass_conversion(const char *path)
+{
+       struct conv_attrs ca;
+       enum crlf_action crlf_action;
+
+       convert_attrs(&ca, path);
+
+       if (ca.ident ||
+           (ca.drv && (ca.drv->smudge || ca.drv->clean)))
+               return 0;
+
+       crlf_action = input_crlf_action(ca.crlf_action, ca.eol_attr);
+       if ((crlf_action == CRLF_BINARY) ||
+           (crlf_action == CRLF_GUESS && auto_crlf == AUTO_CRLF_FALSE))
+               return 1;
+       return 0;
+}

diff --git a/entry.c b/entry.c
index cc6502a54868d07e73599c5041e42280f0dcff1f..da37d0151413b50dd3e4cc223f71ce55643f349b 100644 (file)
--- a/entry.c
+++ b/entry.c
@@ -1,6 +1,7 @@
 #include "cache.h"
 #include "blob.h"
 #include "dir.h"
 #include "cache.h"
 #include "blob.h"
 #include "dir.h"

+#include "streaming.h"

 
 static void create_directories(const char *path, int path_len,
                               const struct checkout *state)
 
 static void create_directories(const char *path, int path_len,
                               const struct checkout *state)


@@ -114,6 +115,50 @@ static int fstat_output(int fd, const struct checkout *state, struct stat *st)
        return 0;
 }
 
        return 0;
 }
 

+static int streaming_write_entry(struct cache_entry *ce, char *path,
+                                const struct checkout *state, int to_tempfile,
+                                int *fstat_done, struct stat *statbuf)
+{
+       struct git_istream *st;
+       enum object_type type;
+       unsigned long sz;
+       int result = -1;
+       int fd = -1;
+
+       st = open_istream(ce->sha1, &type, &sz);
+       if (!st)
+               return -1;
+       if (type != OBJ_BLOB)
+               goto close_and_exit;
+
+       fd = open_output_fd(path, ce, to_tempfile);
+       if (fd < 0)
+               goto close_and_exit;
+
+       for (;;) {
+               char buf[10240];
+               ssize_t wrote;
+               ssize_t readlen = read_istream(st, buf, sizeof(buf));
+
+               if (!readlen)
+                       break;
+
+               wrote = write_in_full(fd, buf, readlen);
+
+               if (wrote != readlen)
+                       goto close_and_exit;
+       }
+       *fstat_done = fstat_output(fd, state, statbuf);
+
+close_and_exit:
+       close_istream(st);
+       if (0 <= fd)
+               result = close(fd);
+       if (result && 0 <= fd)
+               unlink(path);
+       return result;
+}
+

 static int write_entry(struct cache_entry *ce, char *path, const struct checkout *state, int to_tempfile)
 {
        unsigned int ce_mode_s_ifmt = ce->ce_mode & S_IFMT;
 static int write_entry(struct cache_entry *ce, char *path, const struct checkout *state, int to_tempfile)
 {
        unsigned int ce_mode_s_ifmt = ce->ce_mode & S_IFMT;


@@ -124,6 +169,12 @@ static int write_entry(struct cache_entry *ce, char *path, const struct checkout
        size_t wrote, newsize = 0;
        struct stat st;
 
        size_t wrote, newsize = 0;
        struct stat st;
 

+       if ((ce_mode_s_ifmt == S_IFREG) &&
+           can_bypass_conversion(path) &&
+           !streaming_write_entry(ce, path, state, to_tempfile,
+                                  &fstat_done, &st))
+               goto finish;
+

        switch (ce_mode_s_ifmt) {
        case S_IFREG:
        case S_IFLNK:
        switch (ce_mode_s_ifmt) {
        case S_IFREG:
        case S_IFLNK:


@@ -176,6 +227,7 @@ static int write_entry(struct cache_entry *ce, char *path, const struct checkout
                return error("unknown file mode for %s in index", path);
        }
 
                return error("unknown file mode for %s in index", path);
        }
 

+finish:

        if (state->refresh_cache) {
                if (!fstat_done)
                        lstat(ce->name, &st);
        if (state->refresh_cache) {
                if (!fstat_done)
                        lstat(ce->name, &st);