upload-pack/fetch-pack: support side-band communication

This implements a protocol extension between fetch-pack and
upload-pack to allow stderr stream from upload-pack (primarily
used for the progress bar display) to be passed back.

Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/cache.h b/cache.h
index eaa5c0c356645df722f05786860b362730e77074..efeafea70f61a8496cfd6fd6405793459d3df7a9 100644 (file)
--- a/cache.h
+++ b/cache.h
@@ -374,8 +374,8 @@ extern char git_commit_encoding[MAX_ENCODING_LENGTH];
 extern int copy_fd(int ifd, int ofd);
 
 /* Finish off pack transfer receiving end */
 extern int copy_fd(int ifd, int ofd);
 
 /* Finish off pack transfer receiving end */

-extern int receive_unpack_pack(int fd[2], const char *me, int quiet);
-extern int receive_keep_pack(int fd[2], const char *me, int quiet);
+extern int receive_unpack_pack(int fd[2], const char *me, int quiet, int);
+extern int receive_keep_pack(int fd[2], const char *me, int quiet, int);

 
 /* pager.c */
 extern void setup_pager(void);
 
 /* pager.c */
 extern void setup_pager(void);

diff --git a/fetch-clone.c b/fetch-clone.c
index da1b3ffbaa13ee4dfc8d080759b3f936850b7647..c16b0c481bb4b7dd3810e7dfbcb21243572f6844 100644 (file)
--- a/fetch-clone.c
+++ b/fetch-clone.c
@@ -1,5 +1,6 @@
 #include "cache.h"
 #include "exec_cmd.h"
 #include "cache.h"
 #include "exec_cmd.h"

+#include "pkt-line.h"

 #include <sys/wait.h>
 #include <sys/time.h>
 
 #include <sys/wait.h>
 #include <sys/time.h>
 


@@ -23,7 +24,7 @@ static int finish_pack(const char *pack_tmp_name, const char *me)
 
        pid = fork();
        if (pid < 0)
 
        pid = fork();
        if (pid < 0)

-               die("git-clone-pack: unable to fork off git-index-pack");
+               die("%s: unable to fork off git-index-pack", me);

        if (!pid) {
                close(0);
                dup2(pipe_fd[1], 1);
        if (!pid) {
                close(0);
                dup2(pipe_fd[1], 1);


@@ -94,11 +95,69 @@ static int finish_pack(const char *pack_tmp_name, const char *me)
        exit(1);
 }
 
        exit(1);
 }
 

-int receive_unpack_pack(int fd[2], const char *me, int quiet)
+static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2])
+{
+       pid_t side_pid;
+
+       if (!sideband) {
+               fd[0] = xd[0];
+               fd[1] = xd[1];
+               return 0;
+       }
+       /* xd[] is talking with upload-pack; subprocess reads from
+        * xd[0], spits out band#2 to stderr, and feeds us band#1
+        * through our fd[0].
+        */
+       if (pipe(fd) < 0)
+               die("%s: unable to set up pipe", me);
+       side_pid = fork();
+       if (side_pid < 0)
+               die("%s: unable to fork off sideband demultiplexer", me);
+       if (!side_pid) {
+               /* subprocess */
+               close(fd[0]);
+               if (xd[0] != xd[1])
+                       close(xd[1]);
+               while (1) {
+                       char buf[1024];
+                       int len = packet_read_line(xd[0], buf, sizeof(buf));
+                       if (len == 0)
+                               break;
+                       if (len < 1)
+                               die("%s: protocol error: no band designator",
+                                   me);
+                       len--;
+                       switch (buf[0] & 0xFF) {
+                       case 3:
+                               safe_write(2, buf+1, len);
+                               fprintf(stderr, "\n");
+                               exit(1);
+                       case 2:
+                               safe_write(2, buf+1, len);
+                               continue;
+                       case 1:
+                               safe_write(fd[1], buf+1, len);
+                               continue;
+                       default:
+                               die("%s: protocol error: bad band #%d",
+                                   me, (buf[0] & 0xFF));
+                       }
+               }
+               exit(0);
+       }
+       close(xd[0]);
+       close(fd[1]);
+       fd[1] = xd[1];
+       return side_pid;
+}
+
+int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband)

 {
        int status;
 {
        int status;

-       pid_t pid;
+       pid_t pid, side_pid;
+       int fd[2];

 
 

+       side_pid = setup_sideband(sideband, me, fd, xd);

        pid = fork();
        if (pid < 0)
                die("%s: unable to fork off git-unpack-objects", me);
        pid = fork();
        if (pid < 0)
                die("%s: unable to fork off git-unpack-objects", me);


@@ -147,10 +206,10 @@ int receive_unpack_pack(int fd[2], const char *me, int quiet)
  */
 #define usec_to_binarymsec(x) ((int)(x) / (1000512 >> 10))
 
  */
 #define usec_to_binarymsec(x) ((int)(x) / (1000512 >> 10))
 

-int receive_keep_pack(int fd[2], const char *me, int quiet)
+int receive_keep_pack(int xd[2], const char *me, int quiet, int sideband)

 {
        char tmpfile[PATH_MAX];
 {
        char tmpfile[PATH_MAX];

-       int ofd, ifd;
+       int ofd, ifd, fd[2];

        unsigned long total;
        static struct timeval prev_tv;
        struct average {
        unsigned long total;
        static struct timeval prev_tv;
        struct average {


@@ -160,6 +219,8 @@ int receive_keep_pack(int fd[2], const char *me, int quiet)
        unsigned long avg_bytes, avg_time;
        int idx = 0;
 
        unsigned long avg_bytes, avg_time;
        int idx = 0;
 

+       setup_sideband(sideband, me, fd, xd);
+

        ifd = fd[0];
        snprintf(tmpfile, sizeof(tmpfile),
                 "%s/pack/tmp-XXXXXX", get_object_directory());
        ifd = fd[0];
        snprintf(tmpfile, sizeof(tmpfile),
                 "%s/pack/tmp-XXXXXX", get_object_directory());

diff --git a/fetch-pack.c b/fetch-pack.c
index 7d23a8071a9c2923b1f9bcf1d897e590e04f432b..f2c51ebe4b2b524e5d5bce6af8aa3105df19f177 100644 (file)
--- a/fetch-pack.c
+++ b/fetch-pack.c
@@ -25,7 +25,7 @@ static const char *exec = "git-upload-pack";
 #define MAX_IN_VAIN 256
 
 static struct commit_list *rev_list = NULL;
 #define MAX_IN_VAIN 256
 
 static struct commit_list *rev_list = NULL;

-static int non_common_revs = 0, multi_ack = 0, use_thin_pack = 0;
+static int non_common_revs = 0, multi_ack = 0, use_thin_pack = 0, use_sideband;

 
 static void rev_list_push(struct commit *commit, int mark)
 {
 
 static void rev_list_push(struct commit *commit, int mark)
 {


@@ -165,9 +165,14 @@ static int find_common(int fd[2], unsigned char *result_sha1,
                        continue;
                }
 
                        continue;
                }
 

-               packet_write(fd[1], "want %s%s%s\n", sha1_to_hex(remote),
-                            (multi_ack ? " multi_ack" : ""),
-                            (use_thin_pack ? " thin-pack" : ""));
+               if (!fetching)
+                       packet_write(fd[1], "want %s%s%s%s\n",
+                                    sha1_to_hex(remote),
+                                    (multi_ack ? " multi_ack" : ""),
+                                    (use_sideband ? " side-band" : ""),
+                                    (use_thin_pack ? " thin-pack" : ""));
+               else
+                       packet_write(fd[1], "want %s\n", sha1_to_hex(remote));

                fetching++;
        }
        packet_flush(fd[1]);
                fetching++;
        }
        packet_flush(fd[1]);


@@ -421,6 +426,11 @@ static int fetch_pack(int fd[2], int nr_match, char **match)
                        fprintf(stderr, "Server supports multi_ack\n");
                multi_ack = 1;
        }
                        fprintf(stderr, "Server supports multi_ack\n");
                multi_ack = 1;
        }

+       if (server_supports("side-band")) {
+               if (verbose)
+                       fprintf(stderr, "Server supports side-band\n");
+               use_sideband = 1;
+       }

        if (!ref) {
                packet_flush(fd[1]);
                die("no matching remote head");
        if (!ref) {
                packet_flush(fd[1]);
                die("no matching remote head");


@@ -437,9 +447,9 @@ static int fetch_pack(int fd[2], int nr_match, char **match)
                        fprintf(stderr, "warning: no common commits\n");
 
        if (keep_pack)
                        fprintf(stderr, "warning: no common commits\n");
 
        if (keep_pack)

-               status = receive_keep_pack(fd, "git-fetch-pack", quiet);
+               status = receive_keep_pack(fd, "git-fetch-pack", quiet, use_sideband);

        else
        else

-               status = receive_unpack_pack(fd, "git-fetch-pack", quiet);
+               status = receive_unpack_pack(fd, "git-fetch-pack", quiet, use_sideband);

 
        if (status)
                die("git-fetch-pack: fetch failed.");
 
        if (status)
                die("git-fetch-pack: fetch failed.");

diff --git a/pkt-line.c b/pkt-line.c
index bb3bab05cd203e114b24b778cb5f55218abdec95..3d724acf23fab5e8a8e3dd5b1dce957ea1d4a9d3 100644 (file)
--- a/pkt-line.c
+++ b/pkt-line.c
@@ -16,8 +16,9 @@
  * The writing side could use stdio, but since the reading
  * side can't, we stay with pure read/write interfaces.
  */
  * The writing side could use stdio, but since the reading
  * side can't, we stay with pure read/write interfaces.
  */

-static void safe_write(int fd, const void *buf, unsigned n)
+ssize_t safe_write(int fd, const void *buf, ssize_t n)

 {
 {

+       ssize_t nn = n;

        while (n) {
                int ret = xwrite(fd, buf, n);
                if (ret > 0) {
        while (n) {
                int ret = xwrite(fd, buf, n);
                if (ret > 0) {


@@ -29,6 +30,7 @@ static void safe_write(int fd, const void *buf, unsigned n)
                        die("write error (disk full?)");
                die("write error (%s)", strerror(errno));
        }
                        die("write error (disk full?)");
                die("write error (%s)", strerror(errno));
        }

+       return nn;

 }
 
 /*
 }
 
 /*

diff --git a/pkt-line.h b/pkt-line.h
index 51d0cbe219f2f45bf3920441f09dfcc2744adb06..9abef24de36119a56a9bee48e49c2c8fccae32fc 100644 (file)
--- a/pkt-line.h
+++ b/pkt-line.h
@@ -8,5 +8,6 @@ void packet_flush(int fd);
 void packet_write(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3)));
 
 int packet_read_line(int fd, char *buffer, unsigned size);
 void packet_write(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3)));
 
 int packet_read_line(int fd, char *buffer, unsigned size);

+ssize_t safe_write(int, const void *, ssize_t);

 
 #endif
 
 #endif

diff --git a/upload-pack.c b/upload-pack.c
index 13eaa22780ad0afbabd3927e52fb4735318bf234..7b86f6965b5306f2162a9076f6ab3bbc1cc32a4b 100644 (file)
--- a/upload-pack.c
+++ b/upload-pack.c
@@ -21,6 +21,7 @@ static int use_thin_pack = 0;
 static unsigned char has_sha1[MAX_HAS][20];
 static unsigned char needs_sha1[MAX_NEEDS][20];
 static unsigned int timeout = 0;
 static unsigned char has_sha1[MAX_HAS][20];
 static unsigned char needs_sha1[MAX_NEEDS][20];
 static unsigned int timeout = 0;

+static int use_sideband = 0;

 
 static void reset_timeout(void)
 {
 
 static void reset_timeout(void)
 {


@@ -34,6 +35,43 @@ static int strip(char *line, int len)
        return len;
 }
 
        return len;
 }
 

+#define PACKET_MAX 1000
+static ssize_t send_client_data(int fd, const char *data, ssize_t sz)
+{
+       ssize_t ssz;
+       const char *p;
+
+       if (!data) {
+               if (!use_sideband)
+                       return 0;
+               packet_flush(1);
+       }
+
+       if (!use_sideband) {
+               if (fd == 3)
+                       /* emergency quit */
+                       fd = 2;
+               return safe_write(fd, data, sz);
+       }
+       p = data;
+       ssz = sz;
+       while (sz) {
+               unsigned n;
+               char hdr[5];
+
+               n = sz;
+               if (PACKET_MAX - 5 < n)
+                       n = PACKET_MAX - 5;
+               sprintf(hdr, "%04x", n + 5);
+               hdr[4] = fd;
+               safe_write(1, hdr, 5);
+               safe_write(1, p, n);
+               p += n;
+               sz -= n;
+       }
+       return ssz;
+}
+

 static void create_pack_file(void)
 {
        /* Pipes between rev-list to pack-objects, pack-objects to us
 static void create_pack_file(void)
 {
        /* Pipes between rev-list to pack-objects, pack-objects to us


@@ -43,6 +81,8 @@ static void create_pack_file(void)
        pid_t pid_rev_list, pid_pack_objects;
        int create_full_pack = (nr_our_refs == nr_needs && !nr_has);
        char data[8193], progress[128];
        pid_t pid_rev_list, pid_pack_objects;
        int create_full_pack = (nr_our_refs == nr_needs && !nr_has);
        char data[8193], progress[128];

+       char abort_msg[] = "aborting due to possible repository "
+               "corruption on the remote side.";

        int buffered = -1;
 
        if (pipe(lp_pipe) < 0)
        int buffered = -1;
 
        if (pipe(lp_pipe) < 0)


@@ -132,7 +172,6 @@ static void create_pack_file(void)
 
        while (1) {
                const char *who;
 
        while (1) {
                const char *who;

-               char *cp;

                struct pollfd pfd[2];
                pid_t pid;
                int status;
                struct pollfd pfd[2];
                pid_t pid;
                int status;


@@ -199,19 +238,18 @@ static void create_pack_file(void)
                                }
                                else
                                        buffered = -1;
                                }
                                else
                                        buffered = -1;

-                               sz = xwrite(1, data, sz);
+                               sz = send_client_data(1, data, sz);

                                if (sz < 0)
                                        goto fail;
                        }
                        if (0 <= pe && (pfd[pe].revents & (POLLIN|POLLHUP))) {
                                if (sz < 0)
                                        goto fail;
                        }
                        if (0 <= pe && (pfd[pe].revents & (POLLIN|POLLHUP))) {

-                               /* Status ready; we do not use it for now,
-                                * but later we will add side-band to send it
-                                * to the other side.
+                               /* Status ready; we ship that in the side-band
+                                * or dump to the standard error.

                                 */
                                sz = read(pe_pipe[0], progress,
                                          sizeof(progress));
                                if (0 < sz)
                                 */
                                sz = read(pe_pipe[0], progress,
                                          sizeof(progress));
                                if (0 < sz)

-                                       write(2, progress, sz);
+                                       send_client_data(2, progress, sz);

                                else if (sz == 0) {
                                        close(pe_pipe[0]);
                                        pe_pipe[0] = -1;
                                else if (sz == 0) {
                                        close(pe_pipe[0]);
                                        pe_pipe[0] = -1;


@@ -259,11 +297,12 @@ static void create_pack_file(void)
                /* flush the data */
                if (0 <= buffered) {
                        data[0] = buffered;
                /* flush the data */
                if (0 <= buffered) {
                        data[0] = buffered;

-                       sz = xwrite(1, data, 1);
+                       sz = send_client_data(1, data, 1);

                        if (sz < 0)
                                goto fail;
                        fprintf(stderr, "flushed.\n");
                }
                        if (sz < 0)
                                goto fail;
                        fprintf(stderr, "flushed.\n");
                }

+               send_client_data(1, NULL, 0);

                return;
        }
  fail:
                return;
        }
  fail:


@@ -271,7 +310,8 @@ static void create_pack_file(void)
                kill(pid_pack_objects, SIGKILL);
        if (pid_rev_list)
                kill(pid_rev_list, SIGKILL);
                kill(pid_pack_objects, SIGKILL);
        if (pid_rev_list)
                kill(pid_rev_list, SIGKILL);

-       die("git-upload-pack: aborting due to possible repository corruption on the remote side.");
+       send_client_data(3, abort_msg, sizeof(abort_msg));
+       die("git-upload-pack: %s", abort_msg);

 }
 
 static int got_sha1(char *hex, unsigned char *sha1)
 }
 
 static int got_sha1(char *hex, unsigned char *sha1)


@@ -378,6 +418,8 @@ static int receive_needs(void)
                        multi_ack = 1;
                if (strstr(line+45, "thin-pack"))
                        use_thin_pack = 1;
                        multi_ack = 1;
                if (strstr(line+45, "thin-pack"))
                        use_thin_pack = 1;

+               if (strstr(line+45, "side-band"))
+                       use_sideband = 1;

 
                /* We have sent all our refs already, and the other end
                 * should have chosen out of them; otherwise they are
 
                /* We have sent all our refs already, and the other end
                 * should have chosen out of them; otherwise they are


@@ -399,7 +441,7 @@ static int receive_needs(void)
 
 static int send_ref(const char *refname, const unsigned char *sha1)
 {
 
 static int send_ref(const char *refname, const unsigned char *sha1)
 {

-       static char *capabilities = "multi_ack thin-pack";
+       static char *capabilities = "multi_ack thin-pack side-band";

        struct object *o = parse_object(sha1);
 
        if (!o)
        struct object *o = parse_object(sha1);
 
        if (!o)