commit: fix memory leak in `reduce_heads()`
[gitweb.git] / convert.c
index 8d652bf27c9444d3696c4b942dc85f062e2bf53e..387c1c5455be954d2be3d35f8be794bb5bd7f3e8 100644 (file)
--- a/convert.c
+++ b/convert.c
@@ -1,9 +1,12 @@
+#define NO_THE_INDEX_COMPATIBILITY_MACROS
 #include "cache.h"
+#include "config.h"
 #include "attr.h"
 #include "run-command.h"
 #include "quote.h"
 #include "sigchain.h"
 #include "pkt-line.h"
+#include "sub-process.h"
 
 /*
  * convert.c - convert a file when checking it out and checking it in.
@@ -133,11 +136,12 @@ static const char *gather_convert_stats_ascii(const char *data, unsigned long si
        }
 }
 
-const char *get_cached_convert_stats_ascii(const char *path)
+const char *get_cached_convert_stats_ascii(const struct index_state *istate,
+                                          const char *path)
 {
        const char *ret;
        unsigned long sz;
-       void *data = read_blob_data_from_cache(path, &sz);
+       void *data = read_blob_data_from_index(istate, path, &sz);
        ret = gather_convert_stats_ascii(data, sz);
        free(data);
        return ret;
@@ -216,13 +220,13 @@ static void check_safe_crlf(const char *path, enum crlf_action crlf_action,
        }
 }
 
-static int has_cr_in_index(const char *path)
+static int has_cr_in_index(const struct index_state *istate, const char *path)
 {
        unsigned long sz;
        void *data;
        int has_cr;
 
-       data = read_blob_data_from_cache(path, &sz);
+       data = read_blob_data_from_index(istate, path, &sz);
        if (!data)
                return 0;
        has_cr = memchr(data, '\r', sz) != NULL;
@@ -252,7 +256,8 @@ static int will_convert_lf_to_crlf(size_t len, struct text_stat *stats,
 
 }
 
-static int crlf_to_git(const char *path, const char *src, size_t len,
+static int crlf_to_git(const struct index_state *istate,
+                      const char *path, const char *src, size_t len,
                       struct strbuf *buf,
                       enum crlf_action crlf_action, enum safe_crlf checksafe)
 {
@@ -284,7 +289,8 @@ static int crlf_to_git(const char *path, const char *src, size_t len,
                 * unless we want to renormalize in a merge or
                 * cherry-pick.
                 */
-               if ((checksafe != SAFE_CRLF_RENORMALIZE) && has_cr_in_index(path))
+               if ((checksafe != SAFE_CRLF_RENORMALIZE) &&
+                   has_cr_in_index(istate, path))
                        convert_crlf_into_lf = 0;
        }
        if ((checksafe == SAFE_CRLF_WARN ||
@@ -495,216 +501,95 @@ static int apply_single_file_filter(const char *path, const char *src, size_t le
 
 #define CAP_CLEAN    (1u<<0)
 #define CAP_SMUDGE   (1u<<1)
+#define CAP_DELAY    (1u<<2)
 
 struct cmd2process {
-       struct hashmap_entry ent; /* must be the first member! */
+       struct subprocess_entry subprocess; /* must be the first member! */
        unsigned int supported_capabilities;
-       const char *cmd;
-       struct child_process process;
 };
 
-static int cmd_process_map_initialized;
-static struct hashmap cmd_process_map;
-
-static int cmd2process_cmp(const struct cmd2process *e1,
-                          const struct cmd2process *e2,
-                          const void *unused)
-{
-       return strcmp(e1->cmd, e2->cmd);
-}
-
-static struct cmd2process *find_multi_file_filter_entry(struct hashmap *hashmap, const char *cmd)
-{
-       struct cmd2process key;
-       hashmap_entry_init(&key, strhash(cmd));
-       key.cmd = cmd;
-       return hashmap_get(hashmap, &key, NULL);
-}
-
-static int packet_write_list(int fd, const char *line, ...)
-{
-       va_list args;
-       int err;
-       va_start(args, line);
-       for (;;) {
-               if (!line)
-                       break;
-               if (strlen(line) > LARGE_PACKET_DATA_MAX)
-                       return -1;
-               err = packet_write_fmt_gently(fd, "%s\n", line);
-               if (err)
-                       return err;
-               line = va_arg(args, const char*);
-       }
-       va_end(args);
-       return packet_flush_gently(fd);
-}
-
-static void read_multi_file_filter_status(int fd, struct strbuf *status)
-{
-       struct strbuf **pair;
-       char *line;
-       for (;;) {
-               line = packet_read_line(fd, NULL);
-               if (!line)
-                       break;
-               pair = strbuf_split_str(line, '=', 2);
-               if (pair[0] && pair[0]->len && pair[1]) {
-                       /* the last "status=<foo>" line wins */
-                       if (!strcmp(pair[0]->buf, "status=")) {
-                               strbuf_reset(status);
-                               strbuf_addbuf(status, pair[1]);
-                       }
-               }
-               strbuf_list_free(pair);
-       }
-}
-
-static void kill_multi_file_filter(struct hashmap *hashmap, struct cmd2process *entry)
-{
-       if (!entry)
-               return;
-
-       entry->process.clean_on_exit = 0;
-       kill(entry->process.pid, SIGTERM);
-       finish_command(&entry->process);
-
-       hashmap_remove(hashmap, entry, NULL);
-       free(entry);
-}
+static int subprocess_map_initialized;
+static struct hashmap subprocess_map;
 
-static void stop_multi_file_filter(struct child_process *process)
+static int start_multi_file_filter_fn(struct subprocess_entry *subprocess)
 {
-       sigchain_push(SIGPIPE, SIG_IGN);
-       /* Closing the pipe signals the filter to initiate a shutdown. */
-       close(process->in);
-       close(process->out);
-       sigchain_pop(SIGPIPE);
-       /* Finish command will wait until the shutdown is complete. */
-       finish_command(process);
+       static int versions[] = {2, 0};
+       static struct subprocess_capability capabilities[] = {
+               { "clean",  CAP_CLEAN  },
+               { "smudge", CAP_SMUDGE },
+               { "delay",  CAP_DELAY  },
+               { NULL, 0 }
+       };
+       struct cmd2process *entry = (struct cmd2process *)subprocess;
+       return subprocess_handshake(subprocess, "git-filter", versions, NULL,
+                                   capabilities,
+                                   &entry->supported_capabilities);
 }
 
-static struct cmd2process *start_multi_file_filter(struct hashmap *hashmap, const char *cmd)
-{
-       int err;
-       struct cmd2process *entry;
-       struct child_process *process;
-       const char *argv[] = { cmd, NULL };
-       struct string_list cap_list = STRING_LIST_INIT_NODUP;
-       char *cap_buf;
-       const char *cap_name;
-
-       entry = xmalloc(sizeof(*entry));
-       entry->cmd = cmd;
-       entry->supported_capabilities = 0;
-       process = &entry->process;
-
-       child_process_init(process);
-       process->argv = argv;
-       process->use_shell = 1;
-       process->in = -1;
-       process->out = -1;
-       process->clean_on_exit = 1;
-       process->clean_on_exit_handler = stop_multi_file_filter;
-
-       if (start_command(process)) {
-               error("cannot fork to run external filter '%s'", cmd);
-               return NULL;
-       }
-
-       hashmap_entry_init(entry, strhash(cmd));
-
-       sigchain_push(SIGPIPE, SIG_IGN);
-
-       err = packet_write_list(process->in, "git-filter-client", "version=2", NULL);
-       if (err)
-               goto done;
-
-       err = strcmp(packet_read_line(process->out, NULL), "git-filter-server");
-       if (err) {
-               error("external filter '%s' does not support filter protocol version 2", cmd);
-               goto done;
-       }
-       err = strcmp(packet_read_line(process->out, NULL), "version=2");
-       if (err)
-               goto done;
-       err = packet_read_line(process->out, NULL) != NULL;
-       if (err)
-               goto done;
-
-       err = packet_write_list(process->in, "capability=clean", "capability=smudge", NULL);
-
-       for (;;) {
-               cap_buf = packet_read_line(process->out, NULL);
-               if (!cap_buf)
-                       break;
-               string_list_split_in_place(&cap_list, cap_buf, '=', 1);
-
-               if (cap_list.nr != 2 || strcmp(cap_list.items[0].string, "capability"))
-                       continue;
-
-               cap_name = cap_list.items[1].string;
-               if (!strcmp(cap_name, "clean")) {
-                       entry->supported_capabilities |= CAP_CLEAN;
-               } else if (!strcmp(cap_name, "smudge")) {
-                       entry->supported_capabilities |= CAP_SMUDGE;
-               } else {
-                       warning(
-                               "external filter '%s' requested unsupported filter capability '%s'",
-                               cmd, cap_name
-                       );
-               }
-
-               string_list_clear(&cap_list, 0);
-       }
-
-done:
-       sigchain_pop(SIGPIPE);
-
-       if (err || errno == EPIPE) {
-               error("initialization for external filter '%s' failed", cmd);
-               kill_multi_file_filter(hashmap, entry);
-               return NULL;
+static void handle_filter_error(const struct strbuf *filter_status,
+                               struct cmd2process *entry,
+                               const unsigned int wanted_capability) {
+       if (!strcmp(filter_status->buf, "error"))
+               ; /* The filter signaled a problem with the file. */
+       else if (!strcmp(filter_status->buf, "abort") && wanted_capability) {
+               /*
+                * The filter signaled a permanent problem. Don't try to filter
+                * files with the same command for the lifetime of the current
+                * Git process.
+                */
+                entry->supported_capabilities &= ~wanted_capability;
+       } else {
+               /*
+                * Something went wrong with the protocol filter.
+                * Force shutdown and restart if another blob requires filtering.
+                */
+               error("external filter '%s' failed", entry->subprocess.cmd);
+               subprocess_stop(&subprocess_map, &entry->subprocess);
+               free(entry);
        }
-
-       hashmap_add(hashmap, entry);
-       return entry;
 }
 
 static int apply_multi_file_filter(const char *path, const char *src, size_t len,
                                   int fd, struct strbuf *dst, const char *cmd,
-                                  const unsigned int wanted_capability)
+                                  const unsigned int wanted_capability,
+                                  struct delayed_checkout *dco)
 {
        int err;
+       int can_delay = 0;
        struct cmd2process *entry;
        struct child_process *process;
        struct strbuf nbuf = STRBUF_INIT;
        struct strbuf filter_status = STRBUF_INIT;
        const char *filter_type;
 
-       if (!cmd_process_map_initialized) {
-               cmd_process_map_initialized = 1;
-               hashmap_init(&cmd_process_map, (hashmap_cmp_fn) cmd2process_cmp, 0);
+       if (!subprocess_map_initialized) {
+               subprocess_map_initialized = 1;
+               hashmap_init(&subprocess_map, (hashmap_cmp_fn) cmd2process_cmp,
+                            NULL, 0);
                entry = NULL;
        } else {
-               entry = find_multi_file_filter_entry(&cmd_process_map, cmd);
+               entry = (struct cmd2process *)subprocess_find_entry(&subprocess_map, cmd);
        }
 
        fflush(NULL);
 
        if (!entry) {
-               entry = start_multi_file_filter(&cmd_process_map, cmd);
-               if (!entry)
+               entry = xmalloc(sizeof(*entry));
+               entry->supported_capabilities = 0;
+
+               if (subprocess_start(&subprocess_map, &entry->subprocess, cmd, start_multi_file_filter_fn)) {
+                       free(entry);
                        return 0;
+               }
        }
-       process = &entry->process;
+       process = &entry->subprocess.process;
 
-       if (!(wanted_capability & entry->supported_capabilities))
+       if (!(entry->supported_capabilities & wanted_capability))
                return 0;
 
-       if (CAP_CLEAN & wanted_capability)
+       if (wanted_capability & CAP_CLEAN)
                filter_type = "clean";
-       else if (CAP_SMUDGE & wanted_capability)
+       else if (wanted_capability & CAP_SMUDGE)
                filter_type = "smudge";
        else
                die("unexpected filter type");
@@ -726,6 +611,14 @@ static int apply_multi_file_filter(const char *path, const char *src, size_t len
        if (err)
                goto done;
 
+       if ((entry->supported_capabilities & CAP_DELAY) &&
+           dco && dco->state == CE_CAN_DELAY) {
+               can_delay = 1;
+               err = packet_write_fmt_gently(process->in, "can-delay=1\n");
+               if (err)
+                       goto done;
+       }
+
        err = packet_flush_gently(process->in);
        if (err)
                goto done;
@@ -737,43 +630,88 @@ static int apply_multi_file_filter(const char *path, const char *src, size_t len
        if (err)
                goto done;
 
-       read_multi_file_filter_status(process->out, &filter_status);
-       err = strcmp(filter_status.buf, "success");
+       err = subprocess_read_status(process->out, &filter_status);
        if (err)
                goto done;
 
-       err = read_packetized_to_strbuf(process->out, &nbuf) < 0;
+       if (can_delay && !strcmp(filter_status.buf, "delayed")) {
+               string_list_insert(&dco->filters, cmd);
+               string_list_insert(&dco->paths, path);
+       } else {
+               /* The filter got the blob and wants to send us a response. */
+               err = strcmp(filter_status.buf, "success");
+               if (err)
+                       goto done;
+
+               err = read_packetized_to_strbuf(process->out, &nbuf) < 0;
+               if (err)
+                       goto done;
+
+               err = subprocess_read_status(process->out, &filter_status);
+               if (err)
+                       goto done;
+
+               err = strcmp(filter_status.buf, "success");
+       }
+
+done:
+       sigchain_pop(SIGPIPE);
+
+       if (err)
+               handle_filter_error(&filter_status, entry, wanted_capability);
+       else
+               strbuf_swap(dst, &nbuf);
+       strbuf_release(&nbuf);
+       return !err;
+}
+
+
+int async_query_available_blobs(const char *cmd, struct string_list *available_paths)
+{
+       int err;
+       char *line;
+       struct cmd2process *entry;
+       struct child_process *process;
+       struct strbuf filter_status = STRBUF_INIT;
+
+       assert(subprocess_map_initialized);
+       entry = (struct cmd2process *)subprocess_find_entry(&subprocess_map, cmd);
+       if (!entry) {
+               error("external filter '%s' is not available anymore although "
+                     "not all paths have been filtered", cmd);
+               return 0;
+       }
+       process = &entry->subprocess.process;
+       sigchain_push(SIGPIPE, SIG_IGN);
+
+       err = packet_write_fmt_gently(
+               process->in, "command=list_available_blobs\n");
+       if (err)
+               goto done;
+
+       err = packet_flush_gently(process->in);
+       if (err)
+               goto done;
+
+       while ((line = packet_read_line(process->out, NULL))) {
+               const char *path;
+               if (skip_prefix(line, "pathname=", &path))
+                       string_list_insert(available_paths, xstrdup(path));
+               else
+                       ; /* ignore unknown keys */
+       }
+
+       err = subprocess_read_status(process->out, &filter_status);
        if (err)
                goto done;
 
-       read_multi_file_filter_status(process->out, &filter_status);
        err = strcmp(filter_status.buf, "success");
 
 done:
        sigchain_pop(SIGPIPE);
 
-       if (err || errno == EPIPE) {
-               if (!strcmp(filter_status.buf, "error")) {
-                       /* The filter signaled a problem with the file. */
-               } else if (!strcmp(filter_status.buf, "abort")) {
-                       /*
-                        * The filter signaled a permanent problem. Don't try to filter
-                        * files with the same command for the lifetime of the current
-                        * Git process.
-                        */
-                        entry->supported_capabilities &= ~wanted_capability;
-               } else {
-                       /*
-                        * Something went wrong with the protocol filter.
-                        * Force shutdown and restart if another blob requires filtering.
-                        */
-                       error("external filter '%s' failed", cmd);
-                       kill_multi_file_filter(&cmd_process_map, entry);
-               }
-       } else {
-               strbuf_swap(dst, &nbuf);
-       }
-       strbuf_release(&nbuf);
+       if (err)
+               handle_filter_error(&filter_status, entry, 0);
        return !err;
 }
 
@@ -788,7 +726,8 @@ static struct convert_driver {
 
 static int apply_filter(const char *path, const char *src, size_t len,
                        int fd, struct strbuf *dst, struct convert_driver *drv,
-                       const unsigned int wanted_capability)
+                       const unsigned int wanted_capability,
+                       struct delayed_checkout *dco)
 {
        const char *cmd = NULL;
 
@@ -798,15 +737,16 @@ static int apply_filter(const char *path, const char *src, size_t len,
        if (!dst)
                return 1;
 
-       if ((CAP_CLEAN & wanted_capability) && !drv->process && drv->clean)
+       if ((wanted_capability & CAP_CLEAN) && !drv->process && drv->clean)
                cmd = drv->clean;
-       else if ((CAP_SMUDGE & wanted_capability) && !drv->process && drv->smudge)
+       else if ((wanted_capability & CAP_SMUDGE) && !drv->process && drv->smudge)
                cmd = drv->smudge;
 
        if (cmd && *cmd)
                return apply_single_file_filter(path, src, len, fd, dst, cmd);
        else if (drv->process && *drv->process)
-               return apply_multi_file_filter(path, src, len, fd, dst, drv->process, wanted_capability);
+               return apply_multi_file_filter(path, src, len, fd, dst,
+                       drv->process, wanted_capability, dco);
 
        return 0;
 }
@@ -1147,7 +1087,7 @@ int would_convert_to_git_filter_fd(const char *path)
        if (!ca.drv->required)
                return 0;
 
-       return apply_filter(path, NULL, 0, -1, NULL, ca.drv, CAP_CLEAN);
+       return apply_filter(path, NULL, 0, -1, NULL, ca.drv, CAP_CLEAN, NULL);
 }
 
 const char *get_convert_attr_ascii(const char *path)
@@ -1176,7 +1116,8 @@ const char *get_convert_attr_ascii(const char *path)
        return "";
 }
 
-int convert_to_git(const char *path, const char *src, size_t len,
+int convert_to_git(const struct index_state *istate,
+                  const char *path, const char *src, size_t len,
                    struct strbuf *dst, enum safe_crlf checksafe)
 {
        int ret = 0;
@@ -1184,7 +1125,7 @@ int convert_to_git(const char *path, const char *src, size_t len,
 
        convert_attrs(&ca, path);
 
-       ret |= apply_filter(path, src, len, -1, dst, ca.drv, CAP_CLEAN);
+       ret |= apply_filter(path, src, len, -1, dst, ca.drv, CAP_CLEAN, NULL);
        if (!ret && ca.drv && ca.drv->required)
                die("%s: clean filter '%s' failed", path, ca.drv->name);
 
@@ -1192,15 +1133,18 @@ int convert_to_git(const char *path, const char *src, size_t len,
                src = dst->buf;
                len = dst->len;
        }
-       ret |= crlf_to_git(path, src, len, dst, ca.crlf_action, checksafe);
-       if (ret && dst) {
-               src = dst->buf;
-               len = dst->len;
+       if (checksafe != SAFE_CRLF_KEEP_CRLF) {
+               ret |= crlf_to_git(istate, path, src, len, dst, ca.crlf_action, checksafe);
+               if (ret && dst) {
+                       src = dst->buf;
+                       len = dst->len;
+               }
        }
        return ret | ident_to_git(path, src, len, dst, ca.ident);
 }
 
-void convert_to_git_filter_fd(const char *path, int fd, struct strbuf *dst,
+void convert_to_git_filter_fd(const struct index_state *istate,
+                             const char *path, int fd, struct strbuf *dst,
                              enum safe_crlf checksafe)
 {
        struct conv_attrs ca;
@@ -1209,16 +1153,16 @@ void convert_to_git_filter_fd(const char *path, int fd, struct strbuf *dst,
        assert(ca.drv);
        assert(ca.drv->clean || ca.drv->process);
 
-       if (!apply_filter(path, NULL, 0, fd, dst, ca.drv, CAP_CLEAN))
+       if (!apply_filter(path, NULL, 0, fd, dst, ca.drv, CAP_CLEAN, NULL))
                die("%s: clean filter '%s' failed", path, ca.drv->name);
 
-       crlf_to_git(path, dst->buf, dst->len, dst, ca.crlf_action, checksafe);
+       crlf_to_git(istate, path, dst->buf, dst->len, dst, ca.crlf_action, checksafe);
        ident_to_git(path, dst->buf, dst->len, dst, ca.ident);
 }
 
 static int convert_to_working_tree_internal(const char *path, const char *src,
                                            size_t len, struct strbuf *dst,
-                                           int normalizing)
+                                           int normalizing, struct delayed_checkout *dco)
 {
        int ret = 0, ret_filter = 0;
        struct conv_attrs ca;
@@ -1243,26 +1187,35 @@ static int convert_to_working_tree_internal(const char *path, const char *src,
                }
        }
 
-       ret_filter = apply_filter(path, src, len, -1, dst, ca.drv, CAP_SMUDGE);
+       ret_filter = apply_filter(
+               path, src, len, -1, dst, ca.drv, CAP_SMUDGE, dco);
        if (!ret_filter && ca.drv && ca.drv->required)
                die("%s: smudge filter %s failed", path, ca.drv->name);
 
        return ret | ret_filter;
 }
 
+int async_convert_to_working_tree(const char *path, const char *src,
+                                 size_t len, struct strbuf *dst,
+                                 void *dco)
+{
+       return convert_to_working_tree_internal(path, src, len, dst, 0, dco);
+}
+
 int convert_to_working_tree(const char *path, const char *src, size_t len, struct strbuf *dst)
 {
-       return convert_to_working_tree_internal(path, src, len, dst, 0);
+       return convert_to_working_tree_internal(path, src, len, dst, 0, NULL);
 }
 
-int renormalize_buffer(const char *path, const char *src, size_t len, struct strbuf *dst)
+int renormalize_buffer(const struct index_state *istate, const char *path,
+                      const char *src, size_t len, struct strbuf *dst)
 {
-       int ret = convert_to_working_tree_internal(path, src, len, dst, 1);
+       int ret = convert_to_working_tree_internal(path, src, len, dst, 1, NULL);
        if (ret) {
                src = dst->buf;
                len = dst->len;
        }
-       return ret | convert_to_git(path, src, len, dst, SAFE_CRLF_RENORMALIZE);
+       return ret | convert_to_git(istate, path, src, len, dst, SAFE_CRLF_RENORMALIZE);
 }
 
 /*****************************************************************