Merge branch 'np/delta'
[gitweb.git] / diff.c
diff --git a/diff.c b/diff.c
index c73a40b9ab3abe3e4a1ddd51a23a19be36b07ea4..7a7b839e56ac1ba2ed0b770a047aaf07bce23722 100644 (file)
--- a/diff.c
+++ b/diff.c
@@ -8,8 +8,8 @@
 #include "quote.h"
 #include "diff.h"
 #include "diffcore.h"
-
-static const char *diff_opts = "-pu";
+#include "delta.h"
+#include "xdiff-interface.h"
 
 static int use_size_cache;
 
@@ -69,25 +69,10 @@ static const char *external_diff(void)
 {
        static const char *external_diff_cmd = NULL;
        static int done_preparing = 0;
-       const char *env_diff_opts;
 
        if (done_preparing)
                return external_diff_cmd;
-
-       /*
-        * Default values above are meant to match the
-        * Linux kernel development style.  Examples of
-        * alternative styles you can specify via environment
-        * variables are:
-        *
-        * GIT_DIFF_OPTS="-c";
-        */
        external_diff_cmd = getenv("GIT_EXTERNAL_DIFF");
-
-       /* In case external diff fails... */
-       env_diff_opts = getenv("GIT_DIFF_OPTS");
-       if (env_diff_opts) diff_opts = env_diff_opts;
-
        done_preparing = 1;
        return external_diff_cmd;
 }
@@ -101,13 +86,12 @@ static struct diff_tempfile {
        char tmp_path[TEMPFILE_PATH_LEN];
 } diff_temp[2];
 
-static int count_lines(const char *filename)
+static int count_lines(const char *data, int size)
 {
-       FILE *in;
        int count, ch, completely_empty = 1, nl_just_seen = 0;
-       in = fopen(filename, "r");
        count = 0;
-       while ((ch = fgetc(in)) != EOF)
+       while (0 < size--) {
+               ch = *data++;
                if (ch == '\n') {
                        count++;
                        nl_just_seen = 1;
@@ -117,7 +101,7 @@ static int count_lines(const char *filename)
                        nl_just_seen = 0;
                        completely_empty = 0;
                }
-       fclose(in);
+       }
        if (completely_empty)
                return 0;
        if (!nl_just_seen)
@@ -140,12 +124,11 @@ static void print_line_count(int count)
        }
 }
 
-static void copy_file(int prefix, const char *filename)
+static void copy_file(int prefix, const char *data, int size)
 {
-       FILE *in;
        int ch, nl_just_seen = 1;
-       in = fopen(filename, "r");
-       while ((ch = fgetc(in)) != EOF) {
+       while (0 < size--) {
+               ch = *data++;
                if (nl_just_seen)
                        putchar(prefix);
                putchar(ch);
@@ -154,92 +137,388 @@ static void copy_file(int prefix, const char *filename)
                else
                        nl_just_seen = 0;
        }
-       fclose(in);
        if (!nl_just_seen)
                printf("\n\\ No newline at end of file\n");
 }
 
 static void emit_rewrite_diff(const char *name_a,
                              const char *name_b,
-                             struct diff_tempfile *temp)
+                             struct diff_filespec *one,
+                             struct diff_filespec *two)
 {
-       /* Use temp[i].name as input, name_a and name_b as labels */
        int lc_a, lc_b;
-       lc_a = count_lines(temp[0].name);
-       lc_b = count_lines(temp[1].name);
+       diff_populate_filespec(one, 0);
+       diff_populate_filespec(two, 0);
+       lc_a = count_lines(one->data, one->size);
+       lc_b = count_lines(two->data, two->size);
        printf("--- %s\n+++ %s\n@@ -", name_a, name_b);
        print_line_count(lc_a);
        printf(" +");
        print_line_count(lc_b);
        printf(" @@\n");
        if (lc_a)
-               copy_file('-', temp[0].name);
+               copy_file('-', one->data, one->size);
        if (lc_b)
-               copy_file('+', temp[1].name);
+               copy_file('+', two->data, two->size);
 }
 
-static const char *builtin_diff(const char *name_a,
-                        const char *name_b,
-                        struct diff_tempfile *temp,
-                        const char *xfrm_msg,
-                        int complete_rewrite,
-                        const char **args)
+static int fill_mmfile(mmfile_t *mf, struct diff_filespec *one)
+{
+       if (!DIFF_FILE_VALID(one)) {
+               mf->ptr = ""; /* does not matter */
+               mf->size = 0;
+               return 0;
+       }
+       else if (diff_populate_filespec(one, 0))
+               return -1;
+       mf->ptr = one->data;
+       mf->size = one->size;
+       return 0;
+}
+
+struct emit_callback {
+       const char **label_path;
+};
+
+static int fn_out(void *priv, mmbuffer_t *mb, int nbuf)
 {
-       int i, next_at, cmd_size;
-       const char *const diff_cmd = "diff -L%s -L%s";
-       const char *const diff_arg  = "-- %s %s||:"; /* "||:" is to return 0 */
-       const char *input_name_sq[2];
-       const char *label_path[2];
-       char *cmd;
-
-       /* diff_cmd and diff_arg have 4 %s in total which makes
-        * the sum of these strings 8 bytes larger than required.
-        * we use 2 spaces around diff-opts, and we need to count
-        * terminating NUL; we used to subtract 5 here, but we do not
-        * care about small leaks in this subprocess that is about
-        * to exec "diff" anymore.
+       int i;
+       struct emit_callback *ecbdata = priv;
+
+       if (ecbdata->label_path[0]) {
+               printf("--- %s\n", ecbdata->label_path[0]);
+               printf("+++ %s\n", ecbdata->label_path[1]);
+               ecbdata->label_path[0] = ecbdata->label_path[1] = NULL;
+       }
+       for (i = 0; i < nbuf; i++)
+               if (!fwrite(mb[i].ptr, mb[i].size, 1, stdout))
+                       return -1;
+       return 0;
+}
+
+static char *pprint_rename(const char *a, const char *b)
+{
+       const char *old = a;
+       const char *new = b;
+       char *name = NULL;
+       int pfx_length, sfx_length;
+       int len_a = strlen(a);
+       int len_b = strlen(b);
+
+       /* Find common prefix */
+       pfx_length = 0;
+       while (*old && *new && *old == *new) {
+               if (*old == '/')
+                       pfx_length = old - a + 1;
+               old++;
+               new++;
+       }
+
+       /* Find common suffix */
+       old = a + len_a;
+       new = b + len_b;
+       sfx_length = 0;
+       while (a <= old && b <= new && *old == *new) {
+               if (*old == '/')
+                       sfx_length = len_a - (old - a);
+               old--;
+               new--;
+       }
+
+       /*
+        * pfx{mid-a => mid-b}sfx
+        * {pfx-a => pfx-b}sfx
+        * pfx{sfx-a => sfx-b}
+        * name-a => name-b
         */
-       cmd_size = (strlen(diff_cmd) + strlen(diff_opts) + strlen(diff_arg)
-                   + 128);
-
-       for (i = 0; i < 2; i++) {
-               input_name_sq[i] = sq_quote(temp[i].name);
-               if (!strcmp(temp[i].name, "/dev/null"))
-                       label_path[i] = "/dev/null";
-               else if (!i)
-                       label_path[i] = sq_quote(quote_two("a/", name_a));
-               else
-                       label_path[i] = sq_quote(quote_two("b/", name_b));
-               cmd_size += (strlen(label_path[i]) + strlen(input_name_sq[i]));
+       if (pfx_length + sfx_length) {
+               name = xmalloc(len_a + len_b - pfx_length - sfx_length + 7);
+               sprintf(name, "%.*s{%.*s => %.*s}%s",
+                       pfx_length, a,
+                       len_a - pfx_length - sfx_length, a + pfx_length,
+                       len_b - pfx_length - sfx_length, b + pfx_length,
+                       a + len_a - sfx_length);
+       }
+       else {
+               name = xmalloc(len_a + len_b + 5);
+               sprintf(name, "%s => %s", a, b);
+       }
+       return name;
+}
+
+struct diffstat_t {
+       struct xdiff_emit_state xm;
+
+       int nr;
+       int alloc;
+       struct diffstat_file {
+               char *name;
+               unsigned is_unmerged:1;
+               unsigned is_binary:1;
+               unsigned is_renamed:1;
+               unsigned int added, deleted;
+       } **files;
+};
+
+static struct diffstat_file *diffstat_add(struct diffstat_t *diffstat,
+                                         const char *name_a,
+                                         const char *name_b)
+{
+       struct diffstat_file *x;
+       x = xcalloc(sizeof (*x), 1);
+       if (diffstat->nr == diffstat->alloc) {
+               diffstat->alloc = alloc_nr(diffstat->alloc);
+               diffstat->files = xrealloc(diffstat->files,
+                               diffstat->alloc * sizeof(x));
+       }
+       diffstat->files[diffstat->nr++] = x;
+       if (name_b) {
+               x->name = pprint_rename(name_a, name_b);
+               x->is_renamed = 1;
+       }
+       else
+               x->name = strdup(name_a);
+       return x;
+}
+
+static void diffstat_consume(void *priv, char *line, unsigned long len)
+{
+       struct diffstat_t *diffstat = priv;
+       struct diffstat_file *x = diffstat->files[diffstat->nr - 1];
+
+       if (line[0] == '+')
+               x->added++;
+       else if (line[0] == '-')
+               x->deleted++;
+}
+
+static const char pluses[] = "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++";
+static const char minuses[]= "----------------------------------------------------------------------";
+
+static void show_stats(struct diffstat_t* data)
+{
+       int i, len, add, del, total, adds = 0, dels = 0;
+       int max, max_change = 0, max_len = 0;
+       int total_files = data->nr;
+
+       if (data->nr == 0)
+               return;
+
+       for (i = 0; i < data->nr; i++) {
+               struct diffstat_file *file = data->files[i];
+
+               len = strlen(file->name);
+               if (max_len < len)
+                       max_len = len;
+
+               if (file->is_binary || file->is_unmerged)
+                       continue;
+               if (max_change < file->added + file->deleted)
+                       max_change = file->added + file->deleted;
        }
 
-       cmd = xmalloc(cmd_size);
+       for (i = 0; i < data->nr; i++) {
+               char *prefix = "";
+               char *name = data->files[i]->name;
+               int added = data->files[i]->added;
+               int deleted = data->files[i]->deleted;
+
+               if (0 < (len = quote_c_style(name, NULL, NULL, 0))) {
+                       char *qname = xmalloc(len + 1);
+                       quote_c_style(name, qname, NULL, 0);
+                       free(name);
+                       data->files[i]->name = name = qname;
+               }
+
+               /*
+                * "scale" the filename
+                */
+               len = strlen(name);
+               max = max_len;
+               if (max > 50)
+                       max = 50;
+               if (len > max) {
+                       char *slash;
+                       prefix = "...";
+                       max -= 3;
+                       name += len - max;
+                       slash = strchr(name, '/');
+                       if (slash)
+                               name = slash;
+               }
+               len = max;
+
+               /*
+                * scale the add/delete
+                */
+               max = max_change;
+               if (max + len > 70)
+                       max = 70 - len;
+
+               if (data->files[i]->is_binary) {
+                       printf(" %s%-*s |  Bin\n", prefix, len, name);
+                       goto free_diffstat_file;
+               }
+               else if (data->files[i]->is_unmerged) {
+                       printf(" %s%-*s |  Unmerged\n", prefix, len, name);
+                       goto free_diffstat_file;
+               }
+               else if (!data->files[i]->is_renamed &&
+                        (added + deleted == 0)) {
+                       total_files--;
+                       goto free_diffstat_file;
+               }
 
-       next_at = 0;
-       next_at += snprintf(cmd+next_at, cmd_size-next_at,
-                           diff_cmd, label_path[0], label_path[1]);
-       next_at += snprintf(cmd+next_at, cmd_size-next_at,
-                           " %s ", diff_opts);
-       next_at += snprintf(cmd+next_at, cmd_size-next_at,
-                           diff_arg, input_name_sq[0], input_name_sq[1]);
+               add = added;
+               del = deleted;
+               total = add + del;
+               adds += add;
+               dels += del;
 
-       printf("diff --git %s %s\n",
-              quote_two("a/", name_a), quote_two("b/", name_b));
-       if (label_path[0][0] == '/') {
-               /* dev/null */
-               printf("new file mode %s\n", temp[1].mode);
+               if (max_change > 0) {
+                       total = (total * max + max_change / 2) / max_change;
+                       add = (add * max + max_change / 2) / max_change;
+                       del = total - add;
+               }
+               printf(" %s%-*s |%5d %.*s%.*s\n", prefix,
+                               len, name, added + deleted,
+                               add, pluses, del, minuses);
+       free_diffstat_file:
+               free(data->files[i]->name);
+               free(data->files[i]);
+       }
+       free(data->files);
+       printf(" %d files changed, %d insertions(+), %d deletions(-)\n",
+                       total_files, adds, dels);
+}
+
+static unsigned char *deflate_it(char *data,
+                                unsigned long size,
+                                unsigned long *result_size)
+{
+       int bound;
+       unsigned char *deflated;
+       z_stream stream;
+
+       memset(&stream, 0, sizeof(stream));
+       deflateInit(&stream, Z_BEST_COMPRESSION);
+       bound = deflateBound(&stream, size);
+       deflated = xmalloc(bound);
+       stream.next_out = deflated;
+       stream.avail_out = bound;
+
+       stream.next_in = (unsigned char *)data;
+       stream.avail_in = size;
+       while (deflate(&stream, Z_FINISH) == Z_OK)
+               ; /* nothing */
+       deflateEnd(&stream);
+       *result_size = stream.total_out;
+       return deflated;
+}
+
+static void emit_binary_diff(mmfile_t *one, mmfile_t *two)
+{
+       void *cp;
+       void *delta;
+       void *deflated;
+       void *data;
+       unsigned long orig_size;
+       unsigned long delta_size;
+       unsigned long deflate_size;
+       unsigned long data_size;
+
+       printf("GIT binary patch\n");
+       /* We could do deflated delta, or we could do just deflated two,
+        * whichever is smaller.
+        */
+       delta = NULL;
+       deflated = deflate_it(two->ptr, two->size, &deflate_size);
+       if (one->size && two->size) {
+               delta = diff_delta(one->ptr, one->size,
+                                  two->ptr, two->size,
+                                  &delta_size, deflate_size);
+               if (delta) {
+                       void *to_free = delta;
+                       orig_size = delta_size;
+                       delta = deflate_it(delta, delta_size, &delta_size);
+                       free(to_free);
+               }
+       }
+
+       if (delta && delta_size < deflate_size) {
+               printf("delta %lu\n", orig_size);
+               free(deflated);
+               data = delta;
+               data_size = delta_size;
+       }
+       else {
+               printf("literal %lu\n", two->size);
+               free(delta);
+               data = deflated;
+               data_size = deflate_size;
+       }
+
+       /* emit data encoded in base85 */
+       cp = data;
+       while (data_size) {
+               int bytes = (52 < data_size) ? 52 : data_size;
+               char line[70];
+               data_size -= bytes;
+               if (bytes <= 26)
+                       line[0] = bytes + 'A' - 1;
+               else
+                       line[0] = bytes - 26 + 'a' - 1;
+               encode_85(line + 1, cp, bytes);
+               cp += bytes;
+               puts(line);
+       }
+       printf("\n");
+       free(data);
+}
+
+#define FIRST_FEW_BYTES 8000
+static int mmfile_is_binary(mmfile_t *mf)
+{
+       long sz = mf->size;
+       if (FIRST_FEW_BYTES < sz)
+               sz = FIRST_FEW_BYTES;
+       if (memchr(mf->ptr, 0, sz))
+               return 1;
+       return 0;
+}
+
+static void builtin_diff(const char *name_a,
+                        const char *name_b,
+                        struct diff_filespec *one,
+                        struct diff_filespec *two,
+                        const char *xfrm_msg,
+                        struct diff_options *o,
+                        int complete_rewrite)
+{
+       mmfile_t mf1, mf2;
+       const char *lbl[2];
+       char *a_one, *b_two;
+
+       a_one = quote_two("a/", name_a);
+       b_two = quote_two("b/", name_b);
+       lbl[0] = DIFF_FILE_VALID(one) ? a_one : "/dev/null";
+       lbl[1] = DIFF_FILE_VALID(two) ? b_two : "/dev/null";
+       printf("diff --git %s %s\n", a_one, b_two);
+       if (lbl[0][0] == '/') {
+               /* /dev/null */
+               printf("new file mode %06o\n", two->mode);
                if (xfrm_msg && xfrm_msg[0])
                        puts(xfrm_msg);
        }
-       else if (label_path[1][0] == '/') {
-               printf("deleted file mode %s\n", temp[0].mode);
+       else if (lbl[1][0] == '/') {
+               printf("deleted file mode %06o\n", one->mode);
                if (xfrm_msg && xfrm_msg[0])
                        puts(xfrm_msg);
        }
        else {
-               if (strcmp(temp[0].mode, temp[1].mode)) {
-                       printf("old mode %s\n", temp[0].mode);
-                       printf("new mode %s\n", temp[1].mode);
+               if (one->mode != two->mode) {
+                       printf("old mode %06o\n", one->mode);
+                       printf("new mode %06o\n", two->mode);
                }
                if (xfrm_msg && xfrm_msg[0])
                        puts(xfrm_msg);
@@ -247,20 +526,97 @@ static const char *builtin_diff(const char *name_a,
                 * we do not run diff between different kind
                 * of objects.
                 */
-               if (strncmp(temp[0].mode, temp[1].mode, 3))
-                       return NULL;
+               if ((one->mode ^ two->mode) & S_IFMT)
+                       goto free_ab_and_return;
                if (complete_rewrite) {
-                       emit_rewrite_diff(name_a, name_b, temp);
-                       return NULL;
+                       emit_rewrite_diff(name_a, name_b, one, two);
+                       goto free_ab_and_return;
                }
        }
 
-       /* This is disgusting */
-       *args++ = "sh";
-       *args++ = "-c";
-       *args++ = cmd;
-       *args = NULL;
-       return "/bin/sh";
+       if (fill_mmfile(&mf1, one) < 0 || fill_mmfile(&mf2, two) < 0)
+               die("unable to read files to diff");
+
+       if (mmfile_is_binary(&mf1) || mmfile_is_binary(&mf2)) {
+               /* Quite common confusing case */
+               if (mf1.size == mf2.size &&
+                   !memcmp(mf1.ptr, mf2.ptr, mf1.size))
+                       goto free_ab_and_return;
+               if (o->binary)
+                       emit_binary_diff(&mf1, &mf2);
+               else
+                       printf("Binary files %s and %s differ\n",
+                              lbl[0], lbl[1]);
+       }
+       else {
+               /* Crazy xdl interfaces.. */
+               const char *diffopts = getenv("GIT_DIFF_OPTS");
+               xpparam_t xpp;
+               xdemitconf_t xecfg;
+               xdemitcb_t ecb;
+               struct emit_callback ecbdata;
+
+               ecbdata.label_path = lbl;
+               xpp.flags = XDF_NEED_MINIMAL;
+               xecfg.ctxlen = 3;
+               xecfg.flags = XDL_EMIT_FUNCNAMES;
+               if (!diffopts)
+                       ;
+               else if (!strncmp(diffopts, "--unified=", 10))
+                       xecfg.ctxlen = strtoul(diffopts + 10, NULL, 10);
+               else if (!strncmp(diffopts, "-u", 2))
+                       xecfg.ctxlen = strtoul(diffopts + 2, NULL, 10);
+               ecb.outf = fn_out;
+               ecb.priv = &ecbdata;
+               xdl_diff(&mf1, &mf2, &xpp, &xecfg, &ecb);
+       }
+
+ free_ab_and_return:
+       free(a_one);
+       free(b_two);
+       return;
+}
+
+static void builtin_diffstat(const char *name_a, const char *name_b,
+                            struct diff_filespec *one,
+                            struct diff_filespec *two,
+                            struct diffstat_t *diffstat,
+                            int complete_rewrite)
+{
+       mmfile_t mf1, mf2;
+       struct diffstat_file *data;
+
+       data = diffstat_add(diffstat, name_a, name_b);
+
+       if (!one || !two) {
+               data->is_unmerged = 1;
+               return;
+       }
+       if (complete_rewrite) {
+               diff_populate_filespec(one, 0);
+               diff_populate_filespec(two, 0);
+               data->deleted = count_lines(one->data, one->size);
+               data->added = count_lines(two->data, two->size);
+               return;
+       }
+       if (fill_mmfile(&mf1, one) < 0 || fill_mmfile(&mf2, two) < 0)
+               die("unable to read files to diff");
+
+       if (mmfile_is_binary(&mf1) || mmfile_is_binary(&mf2))
+               data->is_binary = 1;
+       else {
+               /* Crazy xdl interfaces.. */
+               xpparam_t xpp;
+               xdemitconf_t xecfg;
+               xdemitcb_t ecb;
+
+               xpp.flags = XDF_NEED_MINIMAL;
+               xecfg.ctxlen = 0;
+               xecfg.flags = 0;
+               ecb.outf = xdiff_outf;
+               ecb.priv = diffstat;
+               xdl_diff(&mf1, &mf2, &xpp, &xecfg, &ecb);
+       }
 }
 
 struct diff_filespec *alloc_filespec(const char *path)
@@ -278,7 +634,7 @@ void fill_filespec(struct diff_filespec *spec, const unsigned char *sha1,
                   unsigned short mode)
 {
        if (mode) {
-               spec->mode = DIFF_FILE_CANON_MODE(mode);
+               spec->mode = canon_mode(mode);
                memcpy(spec->sha1, sha1, 20);
                spec->sha1_valid = !!memcmp(sha1, null_sha1, 20);
        }
@@ -620,6 +976,7 @@ static void run_external_diff(const char *pgm,
        int retval;
        static int atexit_asked = 0;
        const char *othername;
+       const char **arg = &spawn_arg[0];
 
        othername = (other? other : name);
        if (one && two) {
@@ -634,36 +991,25 @@ static void run_external_diff(const char *pgm,
                signal(SIGINT, remove_tempfile_on_signal);
        }
 
-       if (pgm) {
-               const char **arg = &spawn_arg[0];
-               if (one && two) {
-                       *arg++ = pgm;
-                       *arg++ = name;
-                       *arg++ = temp[0].name;
-                       *arg++ = temp[0].hex;
-                       *arg++ = temp[0].mode;
-                       *arg++ = temp[1].name;
-                       *arg++ = temp[1].hex;
-                       *arg++ = temp[1].mode;
-                       if (other) {
-                               *arg++ = other;
-                               *arg++ = xfrm_msg;
-                       }
-               } else {
-                       *arg++ = pgm;
-                       *arg++ = name;
+       if (one && two) {
+               *arg++ = pgm;
+               *arg++ = name;
+               *arg++ = temp[0].name;
+               *arg++ = temp[0].hex;
+               *arg++ = temp[0].mode;
+               *arg++ = temp[1].name;
+               *arg++ = temp[1].hex;
+               *arg++ = temp[1].mode;
+               if (other) {
+                       *arg++ = other;
+                       *arg++ = xfrm_msg;
                }
-               *arg = NULL;
        } else {
-               if (one && two) {
-                       pgm = builtin_diff(name, othername, temp, xfrm_msg, complete_rewrite, spawn_arg);
-               } else
-                       printf("* Unmerged path %s\n", name);
+               *arg++ = pgm;
+               *arg++ = name;
        }
-
-       retval = 0;
-       if (pgm)
-               retval = spawn_prog(pgm, spawn_arg);
+       *arg = NULL;
+       retval = spawn_prog(pgm, spawn_arg);
        remove_tempfile();
        if (retval) {
                fprintf(stderr, "external diff died, stopping at %s.\n", name);
@@ -671,6 +1017,27 @@ static void run_external_diff(const char *pgm,
        }
 }
 
+static void run_diff_cmd(const char *pgm,
+                        const char *name,
+                        const char *other,
+                        struct diff_filespec *one,
+                        struct diff_filespec *two,
+                        const char *xfrm_msg,
+                        struct diff_options *o,
+                        int complete_rewrite)
+{
+       if (pgm) {
+               run_external_diff(pgm, name, other, one, two, xfrm_msg,
+                                 complete_rewrite);
+               return;
+       }
+       if (one && two)
+               builtin_diff(name, other ? other : name,
+                            one, two, xfrm_msg, o, complete_rewrite);
+       else
+               printf("* Unmerged path %s\n", name);
+}
+
 static void diff_fill_sha1_info(struct diff_filespec *one)
 {
        if (DIFF_FILE_VALID(one)) {
@@ -700,8 +1067,7 @@ static void run_diff(struct diff_filepair *p, struct diff_options *o)
 
        if (DIFF_PAIR_UNMERGED(p)) {
                /* unmerged */
-               run_external_diff(pgm, p->one->path, NULL, NULL, NULL, NULL,
-                                 0);
+               run_diff_cmd(pgm, p->one->path, NULL, NULL, NULL, NULL, o, 0);
                return;
        }
 
@@ -748,14 +1114,12 @@ static void run_diff(struct diff_filepair *p, struct diff_options *o)
        }
 
        if (memcmp(one->sha1, two->sha1, 20)) {
-               char one_sha1[41];
                int abbrev = o->full_index ? 40 : DEFAULT_ABBREV;
-               memcpy(one_sha1, sha1_to_hex(one->sha1), 41);
 
                len += snprintf(msg + len, sizeof(msg) - len,
                                "index %.*s..%.*s",
-                               abbrev, one_sha1, abbrev,
-                               sha1_to_hex(two->sha1));
+                               abbrev, sha1_to_hex(one->sha1),
+                               abbrev, sha1_to_hex(two->sha1));
                if (one->mode == two->mode)
                        len += snprintf(msg + len, sizeof(msg) - len,
                                        " %06o", one->mode);
@@ -773,20 +1137,44 @@ static void run_diff(struct diff_filepair *p, struct diff_options *o)
                 * needs to be split into deletion and creation.
                 */
                struct diff_filespec *null = alloc_filespec(two->path);
-               run_external_diff(NULL, name, other, one, null, xfrm_msg, 0);
+               run_diff_cmd(NULL, name, other, one, null, xfrm_msg, o, 0);
                free(null);
                null = alloc_filespec(one->path);
-               run_external_diff(NULL, name, other, null, two, xfrm_msg, 0);
+               run_diff_cmd(NULL, name, other, null, two, xfrm_msg, o, 0);
                free(null);
        }
        else
-               run_external_diff(pgm, name, other, one, two, xfrm_msg,
-                                 complete_rewrite);
+               run_diff_cmd(pgm, name, other, one, two, xfrm_msg, o,
+                            complete_rewrite);
 
        free(name_munged);
        free(other_munged);
 }
 
+static void run_diffstat(struct diff_filepair *p, struct diff_options *o,
+                        struct diffstat_t *diffstat)
+{
+       const char *name;
+       const char *other;
+       int complete_rewrite = 0;
+
+       if (DIFF_PAIR_UNMERGED(p)) {
+               /* unmerged */
+               builtin_diffstat(p->one->path, NULL, NULL, NULL, diffstat, 0);
+               return;
+       }
+
+       name = p->one->path;
+       other = (strcmp(name, p->two->path) ? p->two->path : NULL);
+
+       diff_fill_sha1_info(p->one);
+       diff_fill_sha1_info(p->two);
+
+       if (p->status == DIFF_STATUS_MODIFIED && p->score)
+               complete_rewrite = 1;
+       builtin_diffstat(name, other, p->one, p->two, diffstat, complete_rewrite);
+}
+
 void diff_setup(struct diff_options *options)
 {
        memset(options, 0, sizeof(*options));
@@ -805,6 +1193,15 @@ int diff_setup_done(struct diff_options *options)
             options->detect_rename != DIFF_DETECT_COPY) ||
            (0 <= options->rename_limit && !options->detect_rename))
                return -1;
+
+       /*
+        * These cases always need recursive; we do not drop caller-supplied
+        * recursive bits for other formats here.
+        */
+       if ((options->output_format == DIFF_FORMAT_PATCH) ||
+           (options->output_format == DIFF_FORMAT_DIFFSTAT))
+               options->recursive = 1;
+
        if (options->detect_rename && options->rename_limit < 0)
                options->rename_limit = diff_rename_limit_default;
        if (options->setup & DIFF_SETUP_USE_CACHE) {
@@ -830,12 +1227,26 @@ int diff_opt_parse(struct diff_options *options, const char **av, int ac)
        const char *arg = av[0];
        if (!strcmp(arg, "-p") || !strcmp(arg, "-u"))
                options->output_format = DIFF_FORMAT_PATCH;
+       else if (!strcmp(arg, "--patch-with-raw")) {
+               options->output_format = DIFF_FORMAT_PATCH;
+               options->with_raw = 1;
+       }
+       else if (!strcmp(arg, "--stat"))
+               options->output_format = DIFF_FORMAT_DIFFSTAT;
+       else if (!strcmp(arg, "--patch-with-stat")) {
+               options->output_format = DIFF_FORMAT_PATCH;
+               options->with_stat = 1;
+       }
        else if (!strcmp(arg, "-z"))
                options->line_termination = 0;
        else if (!strncmp(arg, "-l", 2))
                options->rename_limit = strtoul(arg+2, NULL, 10);
        else if (!strcmp(arg, "--full-index"))
                options->full_index = 1;
+       else if (!strcmp(arg, "--binary")) {
+               options->output_format = DIFF_FORMAT_PATCH;
+               options->full_index = options->binary = 1;
+       }
        else if (!strcmp(arg, "--name-only"))
                options->output_format = DIFF_FORMAT_NAME;
        else if (!strcmp(arg, "--name-status"))
@@ -852,6 +1263,8 @@ int diff_opt_parse(struct diff_options *options, const char **av, int ac)
                options->filter = arg + 14;
        else if (!strcmp(arg, "--pickaxe-all"))
                options->pickaxe_opts = DIFF_PICKAXE_ALL;
+       else if (!strcmp(arg, "--pickaxe-regex"))
+               options->pickaxe_opts = DIFF_PICKAXE_REGEX;
        else if (!strncmp(arg, "-B", 2)) {
                if ((options->break_opt =
                     diff_scoreopt_parse(arg)) == -1)
@@ -1014,13 +1427,13 @@ const char *diff_unique_abbrev(const unsigned char *sha1, int len)
 static void diff_flush_raw(struct diff_filepair *p,
                           int line_termination,
                           int inter_name_termination,
-                          struct diff_options *options)
+                          struct diff_options *options,
+                          int output_format)
 {
        int two_paths;
        char status[10];
        int abbrev = options->abbrev;
        const char *path_one, *path_two;
-       int output_format = options->output_format;
 
        path_one = p->one->path;
        path_two = p->two->path;
@@ -1122,11 +1535,24 @@ static void diff_flush_patch(struct diff_filepair *p, struct diff_options *o)
 
        if ((DIFF_FILE_VALID(p->one) && S_ISDIR(p->one->mode)) ||
            (DIFF_FILE_VALID(p->two) && S_ISDIR(p->two->mode)))
-               return; /* no tree diffs in patch format */ 
+               return; /* no tree diffs in patch format */
 
        run_diff(p, o);
 }
 
+static void diff_flush_stat(struct diff_filepair *p, struct diff_options *o,
+                           struct diffstat_t *diffstat)
+{
+       if (diff_unmodified_pair(p))
+               return;
+
+       if ((DIFF_FILE_VALID(p->one) && S_ISDIR(p->one->mode)) ||
+           (DIFF_FILE_VALID(p->two) && S_ISDIR(p->two->mode)))
+               return; /* no tree diffs in patch format */
+
+       run_diffstat(p, o, diffstat);
+}
+
 int diff_queue_is_empty(void)
 {
        struct diff_queue_struct *q = &diff_queued_diff;
@@ -1236,25 +1662,27 @@ static void diff_resolve_rename_copy(void)
        diff_debug_queue("resolve-rename-copy done", q);
 }
 
-void diff_flush(struct diff_options *options)
+static void flush_one_pair(struct diff_filepair *p,
+                          int diff_output_format,
+                          struct diff_options *options,
+                          struct diffstat_t *diffstat)
 {
-       struct diff_queue_struct *q = &diff_queued_diff;
-       int i;
        int inter_name_termination = '\t';
-       int diff_output_format = options->output_format;
        int line_termination = options->line_termination;
-
        if (!line_termination)
                inter_name_termination = 0;
 
-       for (i = 0; i < q->nr; i++) {
-               struct diff_filepair *p = q->queue[i];
-               if ((diff_output_format == DIFF_FORMAT_NO_OUTPUT) ||
-                   (p->status == DIFF_STATUS_UNKNOWN))
-                       continue;
-               if (p->status == 0)
-                       die("internal error in diff-resolve-rename-copy");
+       switch (p->status) {
+       case DIFF_STATUS_UNKNOWN:
+               break;
+       case 0:
+               die("internal error in diff-resolve-rename-copy");
+               break;
+       default:
                switch (diff_output_format) {
+               case DIFF_FORMAT_DIFFSTAT:
+                       diff_flush_stat(p, options, diffstat);
+                       break;
                case DIFF_FORMAT_PATCH:
                        diff_flush_patch(p, options);
                        break;
@@ -1262,16 +1690,60 @@ void diff_flush(struct diff_options *options)
                case DIFF_FORMAT_NAME_STATUS:
                        diff_flush_raw(p, line_termination,
                                       inter_name_termination,
-                                      options);
+                                      options, diff_output_format);
                        break;
                case DIFF_FORMAT_NAME:
                        diff_flush_name(p,
                                        inter_name_termination,
                                        line_termination);
                        break;
+               case DIFF_FORMAT_NO_OUTPUT:
+                       break;
                }
-               diff_free_filepair(q->queue[i]);
        }
+}
+
+void diff_flush(struct diff_options *options)
+{
+       struct diff_queue_struct *q = &diff_queued_diff;
+       int i;
+       int diff_output_format = options->output_format;
+       struct diffstat_t *diffstat = NULL;
+
+       if (diff_output_format == DIFF_FORMAT_DIFFSTAT || options->with_stat) {
+               diffstat = xcalloc(sizeof (struct diffstat_t), 1);
+               diffstat->xm.consume = diffstat_consume;
+       }
+
+       if (options->with_raw) {
+               for (i = 0; i < q->nr; i++) {
+                       struct diff_filepair *p = q->queue[i];
+                       flush_one_pair(p, DIFF_FORMAT_RAW, options, NULL);
+               }
+               putchar(options->line_termination);
+       }
+       if (options->with_stat) {
+               for (i = 0; i < q->nr; i++) {
+                       struct diff_filepair *p = q->queue[i];
+                       flush_one_pair(p, DIFF_FORMAT_DIFFSTAT, options,
+                                      diffstat);
+               }
+               show_stats(diffstat);
+               free(diffstat);
+               diffstat = NULL;
+               putchar(options->line_termination);
+       }
+       for (i = 0; i < q->nr; i++) {
+               struct diff_filepair *p = q->queue[i];
+               flush_one_pair(p, diff_output_format, options, diffstat);
+               diff_free_filepair(p);
+       }
+
+       if (diffstat) {
+               show_stats(diffstat);
+               free(diffstat);
+       }
+
        free(q->queue);
        q->queue = NULL;
        q->nr = q->alloc = 0;
@@ -1335,8 +1807,6 @@ static void diffcore_apply_filter(const char *filter)
 
 void diffcore_std(struct diff_options *options)
 {
-       if (options->paths && options->paths[0])
-               diffcore_pathspec(options->paths);
        if (options->break_opt != -1)
                diffcore_break(options->break_opt);
        if (options->detect_rename)