Merge branch 'jk/push-progress'
authorJunio C Hamano <gitster@pobox.com>
Wed, 3 Aug 2016 22:10:27 +0000 (15:10 -0700)
committerJunio C Hamano <gitster@pobox.com>
Wed, 3 Aug 2016 22:10:28 +0000 (15:10 -0700)
"git push" and "git clone" learned to give better progress meters
to the end user who is waiting on the terminal.

* jk/push-progress:
receive-pack: send keepalives during quiet periods
receive-pack: turn on connectivity progress
receive-pack: relay connectivity errors to sideband
receive-pack: turn on index-pack resolving progress
index-pack: add flag for showing delta-resolution progress
clone: use a real progress meter for connectivity check
check_connected: add progress flag
check_connected: relay errors to alternate descriptor
check_everything_connected: use a struct with named options
check_everything_connected: convert to argv_array
rev-list: add optional progress reporting
check_everything_connected: always pass --quiet to rev-list

1  2 
Documentation/config.txt
builtin/fetch.c
builtin/index-pack.c
builtin/receive-pack.c
diff --combined Documentation/config.txt
index 4ae564b48764795e1bfe22d6a75d37c5347d76ca,25c8e3659bbb2956c2a31047be82867f8b483c85..bc1c433c4e0584a095f79196997426780521b377
@@@ -412,11 -412,13 +412,11 @@@ file with mixed line endings would be r
  mechanism.
  
  core.autocrlf::
 -      Setting this variable to "true" is almost the same as setting
 -      the `text` attribute to "auto" on all files except that text
 -      files are not guaranteed to be normalized: files that contain
 -      `CRLF` in the repository will not be touched.  Use this
 -      setting if you want to have `CRLF` line endings in your
 -      working directory even though the repository does not have
 -      normalized line endings.  This variable can be set to 'input',
 +      Setting this variable to "true" is the same as setting
 +      the `text` attribute to "auto" on all files and core.eol to "crlf".
 +      Set to true if you want to have `CRLF` line endings in your
 +      working directory and the repository has LF line endings.
 +      This variable can be set to 'input',
        in which case no output conversion is performed.
  
  core.symlinks::
@@@ -2427,13 -2429,8 +2427,13 @@@ rebase.instructionForma
  
  receive.advertiseAtomic::
        By default, git-receive-pack will advertise the atomic push
 -      capability to its clients. If you don't want to this capability
 -      to be advertised, set this variable to false.
 +      capability to its clients. If you don't want to advertise this
 +      capability, set this variable to false.
 +
 +receive.advertisePushOptions::
 +      By default, git-receive-pack will advertise the push options
 +      capability to its clients. If you don't want to advertise this
 +      capability, set this variable to false.
  
  receive.autogc::
        By default, git-receive-pack will run "git-gc --auto" after
@@@ -2488,6 -2485,15 +2488,15 @@@ receive.fsck.skipList:
        can be safely ignored such as invalid committer email addresses.
        Note: corrupt objects cannot be skipped with this setting.
  
+ receive.keepAlive::
+       After receiving the pack from the client, `receive-pack` may
+       produce no output (if `--quiet` was specified) while processing
+       the pack, causing some networks to drop the TCP connection.
+       With this option set, if `receive-pack` does not transmit
+       any data in this phase for `receive.keepAlive` seconds, it will
+       send a short keepalive packet.  The default is 5 seconds; set
+       to 0 to disable keepalives entirely.
  receive.unpackLimit::
        If the number of objects received in a push is below this
        limit then the objects will be unpacked into loose object
diff --combined builtin/fetch.c
index acd0cf1755eb5afec9dc179b8dc1a93050579023,75ee4800999a33889406ca15cb90d41e0e6efd0a..164623bb6f2eb3ffad3eac59b8ec440b9cf60d9c
@@@ -729,7 -729,7 +729,7 @@@ static int store_updated_refs(const cha
                url = xstrdup("foreign");
  
        rm = ref_map;
-       if (check_everything_connected(iterate_ref_map, 0, &rm)) {
+       if (check_connected(iterate_ref_map, &rm, NULL)) {
                rc = error(_("%s did not send all necessary objects\n"), url);
                goto abort;
        }
  static int quickfetch(struct ref *ref_map)
  {
        struct ref *rm = ref_map;
+       struct check_connected_options opt = CHECK_CONNECTED_INIT;
  
        /*
         * If we are deepening a shallow clone we already have these
         */
        if (depth)
                return -1;
-       return check_everything_connected(iterate_ref_map, 1, &rm);
+       opt.quiet = 1;
+       return check_connected(iterate_ref_map, &rm, &opt);
  }
  
  static int fetch_refs(struct transport *transport, struct ref *ref_map)
@@@ -921,7 -923,7 +923,7 @@@ static int prune_refs(struct refspec *r
                for (ref = stale_refs; ref; ref = ref->next)
                        string_list_append(&refnames, ref->name);
  
 -              result = delete_refs(&refnames);
 +              result = delete_refs(&refnames, 0);
                string_list_clear(&refnames, 0);
        }
  
diff --combined builtin/index-pack.c
index 1008d7f63cab33b162da43f5a8b13cd6e1aea88b,54f2cfbd6e4ebb58beeaae19cc3ef56e0882a0e9..1d2ea583a45507935ec2007d6c44f5e968a1aed5
@@@ -77,6 -77,7 +77,7 @@@ static int strict
  static int do_fsck_object;
  static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
  static int verbose;
+ static int show_resolving_progress;
  static int show_stat;
  static int check_self_contained_and_connected;
  
@@@ -338,10 -339,10 +339,10 @@@ static void parse_pack_header(void
        use(sizeof(struct pack_header));
  }
  
 -static NORETURN void bad_object(unsigned long offset, const char *format,
 +static NORETURN void bad_object(off_t offset, const char *format,
                       ...) __attribute__((format (printf, 2, 3)));
  
 -static NORETURN void bad_object(unsigned long offset, const char *format, ...)
 +static NORETURN void bad_object(off_t offset, const char *format, ...)
  {
        va_list params;
        char buf[1024];
        va_start(params, format);
        vsnprintf(buf, sizeof(buf), format, params);
        va_end(params);
 -      die(_("pack has bad object at offset %lu: %s"), offset, buf);
 +      die(_("pack has bad object at offset %"PRIuMAX": %s"),
 +          (uintmax_t)offset, buf);
  }
  
  static inline struct thread_local *get_thread_data(void)
@@@ -430,7 -430,7 +431,7 @@@ static int is_delta_type(enum object_ty
        return (type == OBJ_REF_DELTA || type == OBJ_OFS_DELTA);
  }
  
 -static void *unpack_entry_data(unsigned long offset, unsigned long size,
 +static void *unpack_entry_data(off_t offset, unsigned long size,
                               enum object_type type, unsigned char *sha1)
  {
        static char fixed_buf[8192];
@@@ -550,13 -550,13 +551,13 @@@ static void *unpack_data(struct object_
                         void *cb_data)
  {
        off_t from = obj[0].idx.offset + obj[0].hdr_size;
 -      unsigned long len = obj[1].idx.offset - from;
 +      off_t len = obj[1].idx.offset - from;
        unsigned char *data, *inbuf;
        git_zstream stream;
        int status;
  
        data = xmallocz(consume ? 64*1024 : obj->size);
 -      inbuf = xmalloc((len < 64*1024) ? len : 64*1024);
 +      inbuf = xmalloc((len < 64*1024) ? (int)len : 64*1024);
  
        memset(&stream, 0, sizeof(stream));
        git_inflate_init(&stream);
        stream.avail_out = consume ? 64*1024 : obj->size;
  
        do {
 -              ssize_t n = (len < 64*1024) ? len : 64*1024;
 +              ssize_t n = (len < 64*1024) ? (ssize_t)len : 64*1024;
                n = xpread(get_thread_data()->pack_fd, inbuf, n, from);
                if (n < 0)
                        die_errno(_("cannot pread pack file"));
                if (!n)
 -                      die(Q_("premature end of pack file, %lu byte missing",
 -                             "premature end of pack file, %lu bytes missing",
 -                             len),
 -                          len);
 +                      die(Q_("premature end of pack file, %"PRIuMAX" byte missing",
 +                             "premature end of pack file, %"PRIuMAX" bytes missing",
 +                             (unsigned int)len),
 +                          (uintmax_t)len);
                from += n;
                len -= n;
                stream.next_in = inbuf;
@@@ -1191,7 -1191,7 +1192,7 @@@ static void resolve_deltas(void
        qsort(ref_deltas, nr_ref_deltas, sizeof(struct ref_delta_entry),
              compare_ref_delta_entry);
  
-       if (verbose)
+       if (verbose || show_resolving_progress)
                progress = start_progress(_("Resolving deltas"),
                                          nr_ref_deltas + nr_ofs_deltas);
  
@@@ -1626,6 -1626,7 +1627,7 @@@ int cmd_index_pack(int argc, const cha
        struct pack_idx_option opts;
        unsigned char pack_sha1[20];
        unsigned foreign_nr = 1;        /* zero is a "good" value, assume bad */
+       int report_end_of_input = 0;
  
        if (argc == 2 && !strcmp(argv[1], "-h"))
                usage(index_pack_usage);
                                input_len = sizeof(*hdr);
                        } else if (!strcmp(arg, "-v")) {
                                verbose = 1;
+                       } else if (!strcmp(arg, "--show-resolving-progress")) {
+                               show_resolving_progress = 1;
+                       } else if (!strcmp(arg, "--report-end-of-input")) {
+                               report_end_of_input = 1;
                        } else if (!strcmp(arg, "-o")) {
                                if (index_name || (i+1) >= argc)
                                        usage(index_pack_usage);
                obj_stat = xcalloc(st_add(nr_objects, 1), sizeof(struct object_stat));
        ofs_deltas = xcalloc(nr_objects, sizeof(struct ofs_delta_entry));
        parse_pack_objects(pack_sha1);
+       if (report_end_of_input)
+               write_in_full(2, "\0", 1);
        resolve_deltas();
        conclude_pack(fix_thin_pack, curr_pack, pack_sha1);
        free(ofs_deltas);
diff --combined builtin/receive-pack.c
index 3c9360aa6dfee96d3101a58b3c8bd899e51b4db3,e41f55f4f53e57dd294135d7799768d3aa235b18..92e1213ecc05969f4601c7d3c48f6d96dcf72a0a
@@@ -44,12 -44,10 +44,12 @@@ static struct strbuf fsck_msg_types = S
  static int receive_unpack_limit = -1;
  static int transfer_unpack_limit = -1;
  static int advertise_atomic_push = 1;
 +static int advertise_push_options;
  static int unpack_limit = 100;
  static int report_status;
  static int use_sideband;
  static int use_atomic;
 +static int use_push_options;
  static int quiet;
  static int prefer_ofs_delta = 1;
  static int auto_update_server_info;
@@@ -78,6 -76,13 +78,13 @@@ static long nonce_stamp_slop
  static unsigned long nonce_stamp_slop_limit;
  static struct ref_transaction *transaction;
  
+ static enum {
+       KEEPALIVE_NEVER = 0,
+       KEEPALIVE_AFTER_NUL,
+       KEEPALIVE_ALWAYS
+ } use_keepalive;
+ static int keepalive_in_sec = 5;
  static enum deny_action parse_deny_action(const char *var, const char *value)
  {
        if (value) {
@@@ -195,11 -200,11 +202,16 @@@ static int receive_pack_config(const ch
                return 0;
        }
  
 +      if (strcmp(var, "receive.advertisepushoptions") == 0) {
 +              advertise_push_options = git_config_bool(var, value);
 +              return 0;
 +      }
 +
+       if (strcmp(var, "receive.keepalive") == 0) {
+               keepalive_in_sec = git_config_int(var, value);
+               return 0;
+       }
        return git_default_config(var, value, cb);
  }
  
@@@ -218,8 -223,6 +230,8 @@@ static void show_ref(const char *path, 
                        strbuf_addstr(&cap, " ofs-delta");
                if (push_cert_nonce)
                        strbuf_addf(&cap, " push-cert=%s", push_cert_nonce);
 +              if (advertise_push_options)
 +                      strbuf_addstr(&cap, " push-options");
                strbuf_addf(&cap, " agent=%s", git_user_agent_sanitized());
                packet_write(1, "%s %s%c%s\n",
                             sha1_to_hex(sha1), path, 0, cap.buf);
@@@ -328,10 -331,60 +340,60 @@@ static void rp_error(const char *err, .
  static int copy_to_sideband(int in, int out, void *arg)
  {
        char data[128];
+       int keepalive_active = 0;
+       if (keepalive_in_sec <= 0)
+               use_keepalive = KEEPALIVE_NEVER;
+       if (use_keepalive == KEEPALIVE_ALWAYS)
+               keepalive_active = 1;
        while (1) {
-               ssize_t sz = xread(in, data, sizeof(data));
+               ssize_t sz;
+               if (keepalive_active) {
+                       struct pollfd pfd;
+                       int ret;
+                       pfd.fd = in;
+                       pfd.events = POLLIN;
+                       ret = poll(&pfd, 1, 1000 * keepalive_in_sec);
+                       if (ret < 0) {
+                               if (errno == EINTR)
+                                       continue;
+                               else
+                                       break;
+                       } else if (ret == 0) {
+                               /* no data; send a keepalive packet */
+                               static const char buf[] = "0005\1";
+                               write_or_die(1, buf, sizeof(buf) - 1);
+                               continue;
+                       } /* else there is actual data to read */
+               }
+               sz = xread(in, data, sizeof(data));
                if (sz <= 0)
                        break;
+               if (use_keepalive == KEEPALIVE_AFTER_NUL && !keepalive_active) {
+                       const char *p = memchr(data, '\0', sz);
+                       if (p) {
+                               /*
+                                * The NUL tells us to start sending keepalives. Make
+                                * sure we send any other data we read along
+                                * with it.
+                                */
+                               keepalive_active = 1;
+                               send_sideband(1, 2, data, p - data, use_sideband);
+                               send_sideband(1, 2, p + 1, sz - (p - data + 1), use_sideband);
+                               continue;
+                       }
+               }
+               /*
+                * Either we're not looking for a NUL signal, or we didn't see
+                * it yet; just pass along the data.
+                */
                send_sideband(1, 2, data, sz, use_sideband);
        }
        close(in);
@@@ -559,16 -612,8 +621,16 @@@ static void prepare_push_cert_sha1(stru
        }
  }
  
 +struct receive_hook_feed_state {
 +      struct command *cmd;
 +      int skip_broken;
 +      struct strbuf buf;
 +      const struct string_list *push_options;
 +};
 +
  typedef int (*feed_fn)(void *, const char **, size_t *);
 -static int run_and_feed_hook(const char *hook_name, feed_fn feed, void *feed_state)
 +static int run_and_feed_hook(const char *hook_name, feed_fn feed,
 +                           struct receive_hook_feed_state *feed_state)
  {
        struct child_process proc = CHILD_PROCESS_INIT;
        struct async muxer;
        proc.argv = argv;
        proc.in = -1;
        proc.stdout_to_stderr = 1;
 +      if (feed_state->push_options) {
 +              int i;
 +              for (i = 0; i < feed_state->push_options->nr; i++)
 +                      argv_array_pushf(&proc.env_array,
 +                              "GIT_PUSH_OPTION_%d=%s", i,
 +                              feed_state->push_options->items[i].string);
 +              argv_array_pushf(&proc.env_array, "GIT_PUSH_OPTION_COUNT=%d",
 +                               feed_state->push_options->nr);
 +      } else
 +              argv_array_pushf(&proc.env_array, "GIT_PUSH_OPTION_COUNT");
  
        if (use_sideband) {
                memset(&muxer, 0, sizeof(muxer));
        return finish_command(&proc);
  }
  
 -struct receive_hook_feed_state {
 -      struct command *cmd;
 -      int skip_broken;
 -      struct strbuf buf;
 -};
 -
  static int feed_receive_hook(void *state_, const char **bufp, size_t *sizep)
  {
        struct receive_hook_feed_state *state = state_;
        return 0;
  }
  
 -static int run_receive_hook(struct command *commands, const char *hook_name,
 -                          int skip_broken)
 +static int run_receive_hook(struct command *commands,
 +                          const char *hook_name,
 +                          int skip_broken,
 +                          const struct string_list *push_options)
  {
        struct receive_hook_feed_state state;
        int status;
        if (feed_receive_hook(&state, NULL, NULL))
                return 0;
        state.cmd = commands;
 +      state.push_options = push_options;
        status = run_and_feed_hook(hook_name, feed_receive_hook, &state);
        strbuf_release(&state.buf);
        return status;
@@@ -761,7 -799,7 +823,7 @@@ static int update_shallow_ref(struct co
  {
        static struct lock_file shallow_lock;
        struct sha1_array extra = SHA1_ARRAY_INIT;
-       const char *alt_file;
+       struct check_connected_options opt = CHECK_CONNECTED_INIT;
        uint32_t mask = 1 << (cmd->index % 32);
        int i;
  
                    !delayed_reachability_test(si, i))
                        sha1_array_append(&extra, si->shallow->sha1[i]);
  
-       setup_alternate_shallow(&shallow_lock, &alt_file, &extra);
-       if (check_shallow_connected(command_singleton_iterator,
-                                   0, cmd, alt_file)) {
+       setup_alternate_shallow(&shallow_lock, &opt.shallow_file, &extra);
+       if (check_connected(command_singleton_iterator, cmd, &opt)) {
                rollback_lock_file(&shallow_lock);
                sha1_array_clear(&extra);
                return -1;
@@@ -1184,8 -1221,8 +1245,8 @@@ static void set_connectivity_errors(str
                if (shallow_update && si->shallow_ref[cmd->index])
                        /* to be checked in update_shallow_ref() */
                        continue;
-               if (!check_everything_connected(command_singleton_iterator,
-                                               0, &singleton))
+               if (!check_connected(command_singleton_iterator, &singleton,
+                                    NULL))
                        continue;
                cmd->error_string = "missing necessary objects";
        }
@@@ -1340,12 -1377,14 +1401,15 @@@ cleanup
  
  static void execute_commands(struct command *commands,
                             const char *unpacker_error,
 -                           struct shallow_info *si)
 +                           struct shallow_info *si,
 +                           const struct string_list *push_options)
  {
+       struct check_connected_options opt = CHECK_CONNECTED_INIT;
        struct command *cmd;
        unsigned char sha1[20];
        struct iterate_data data;
+       struct async muxer;
+       int err_fd = 0;
  
        if (unpacker_error) {
                for (cmd = commands; cmd; cmd = cmd->next)
                return;
        }
  
+       if (use_sideband) {
+               memset(&muxer, 0, sizeof(muxer));
+               muxer.proc = copy_to_sideband;
+               muxer.in = -1;
+               if (!start_async(&muxer))
+                       err_fd = muxer.in;
+               /* ...else, continue without relaying sideband */
+       }
        data.cmds = commands;
        data.si = si;
-       if (check_everything_connected(iterate_receive_command_list, 0, &data))
+       opt.err_fd = err_fd;
+       opt.progress = err_fd && !quiet;
+       if (check_connected(iterate_receive_command_list, &data, &opt))
                set_connectivity_errors(commands, si);
  
+       if (use_sideband)
+               finish_async(&muxer);
        reject_updates_to_hidden(commands);
  
 -      if (run_receive_hook(commands, "pre-receive", 0)) {
 +      if (run_receive_hook(commands, "pre-receive", 0, push_options)) {
                for (cmd = commands; cmd; cmd = cmd->next) {
                        if (!cmd->error_string)
                                cmd->error_string = "pre-receive hook declined";
@@@ -1464,9 -1517,6 +1542,9 @@@ static struct command *read_head_info(s
                        if (advertise_atomic_push
                            && parse_feature_request(feature_list, "atomic"))
                                use_atomic = 1;
 +                      if (advertise_push_options
 +                          && parse_feature_request(feature_list, "push-options"))
 +                              use_push_options = 1;
                }
  
                if (!strcmp(line, "push-cert")) {
        return commands;
  }
  
 +static void read_push_options(struct string_list *options)
 +{
 +      while (1) {
 +              char *line;
 +              int len;
 +
 +              line = packet_read_line(0, &len);
 +
 +              if (!line)
 +                      break;
 +
 +              string_list_append(options, line);
 +      }
 +}
 +
  static const char *parse_pack_header(struct pack_header *hdr)
  {
        switch (read_pack_header(0, hdr)) {
@@@ -1591,6 -1626,10 +1669,10 @@@ static const char *unpack(int err_fd, s
                                 (uintmax_t)getpid(),
                                 hostname);
  
+               if (!quiet && err_fd)
+                       argv_array_push(&child.args, "--show-resolving-progress");
+               if (use_sideband)
+                       argv_array_push(&child.args, "--report-end-of-input");
                if (fsck_objects)
                        argv_array_pushf(&child.args, "--strict%s",
                                fsck_msg_types.buf);
@@@ -1620,6 -1659,7 +1702,7 @@@ static const char *unpack_with_sideband
        if (!use_sideband)
                return unpack(0, si);
  
+       use_keepalive = KEEPALIVE_AFTER_NUL;
        memset(&muxer, 0, sizeof(muxer));
        muxer.proc = copy_to_sideband;
        muxer.in = -1;
@@@ -1799,10 -1839,6 +1882,10 @@@ int cmd_receive_pack(int argc, const ch
  
        if ((commands = read_head_info(&shallow)) != NULL) {
                const char *unpack_status = NULL;
 +              struct string_list push_options = STRING_LIST_INIT_DUP;
 +
 +              if (use_push_options)
 +                      read_push_options(&push_options);
  
                prepare_shallow_info(&si, &shallow);
                if (!si.nr_ours && !si.nr_theirs)
                        unpack_status = unpack_with_sideband(&si);
                        update_shallow_info(commands, &si, &ref);
                }
 -              execute_commands(commands, unpack_status, &si);
+               use_keepalive = KEEPALIVE_ALWAYS;
 +              execute_commands(commands, unpack_status, &si,
 +                               &push_options);
                if (pack_lockfile)
                        unlink_or_warn(pack_lockfile);
                if (report_status)
                        report(commands, unpack_status);
 -              run_receive_hook(commands, "post-receive", 1);
 +              run_receive_hook(commands, "post-receive", 1,
 +                               &push_options);
                run_update_post_hook(commands);
 +              if (push_options.nr)
 +                      string_list_clear(&push_options, 0);
                if (auto_gc) {
                        const char *argv_gc_auto[] = {
                                "gc", "--auto", "--quiet", NULL,