From: Junio C Hamano Date: Wed, 3 Aug 2016 22:10:27 +0000 (-0700) Subject: Merge branch 'jk/push-progress' X-Git-Tag: v2.10.0-rc0~60 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/a58a8e3f7156dadd5d5e9643168545ff057c111a?ds=inline;hp=-c Merge branch 'jk/push-progress' "git push" and "git clone" learned to give better progress meters to the end user who is waiting on the terminal. * jk/push-progress: receive-pack: send keepalives during quiet periods receive-pack: turn on connectivity progress receive-pack: relay connectivity errors to sideband receive-pack: turn on index-pack resolving progress index-pack: add flag for showing delta-resolution progress clone: use a real progress meter for connectivity check check_connected: add progress flag check_connected: relay errors to alternate descriptor check_everything_connected: use a struct with named options check_everything_connected: convert to argv_array rev-list: add optional progress reporting check_everything_connected: always pass --quiet to rev-list --- a58a8e3f7156dadd5d5e9643168545ff057c111a diff --combined Documentation/config.txt index 4ae564b487,25c8e3659b..bc1c433c4e --- a/Documentation/config.txt +++ b/Documentation/config.txt @@@ -412,11 -412,13 +412,11 @@@ file with mixed line endings would be r mechanism. core.autocrlf:: - Setting this variable to "true" is almost the same as setting - the `text` attribute to "auto" on all files except that text - files are not guaranteed to be normalized: files that contain - `CRLF` in the repository will not be touched. Use this - setting if you want to have `CRLF` line endings in your - working directory even though the repository does not have - normalized line endings. This variable can be set to 'input', + Setting this variable to "true" is the same as setting + the `text` attribute to "auto" on all files and core.eol to "crlf". + Set to true if you want to have `CRLF` line endings in your + working directory and the repository has LF line endings. + This variable can be set to 'input', in which case no output conversion is performed. core.symlinks:: @@@ -2427,13 -2429,8 +2427,13 @@@ rebase.instructionForma receive.advertiseAtomic:: By default, git-receive-pack will advertise the atomic push - capability to its clients. If you don't want to this capability - to be advertised, set this variable to false. + capability to its clients. If you don't want to advertise this + capability, set this variable to false. + +receive.advertisePushOptions:: + By default, git-receive-pack will advertise the push options + capability to its clients. If you don't want to advertise this + capability, set this variable to false. receive.autogc:: By default, git-receive-pack will run "git-gc --auto" after @@@ -2488,6 -2485,15 +2488,15 @@@ receive.fsck.skipList: can be safely ignored such as invalid committer email addresses. Note: corrupt objects cannot be skipped with this setting. + receive.keepAlive:: + After receiving the pack from the client, `receive-pack` may + produce no output (if `--quiet` was specified) while processing + the pack, causing some networks to drop the TCP connection. + With this option set, if `receive-pack` does not transmit + any data in this phase for `receive.keepAlive` seconds, it will + send a short keepalive packet. The default is 5 seconds; set + to 0 to disable keepalives entirely. + receive.unpackLimit:: If the number of objects received in a push is below this limit then the objects will be unpacked into loose object diff --combined builtin/fetch.c index acd0cf1755,75ee480099..164623bb6f --- a/builtin/fetch.c +++ b/builtin/fetch.c @@@ -729,7 -729,7 +729,7 @@@ static int store_updated_refs(const cha url = xstrdup("foreign"); rm = ref_map; - if (check_everything_connected(iterate_ref_map, 0, &rm)) { + if (check_connected(iterate_ref_map, &rm, NULL)) { rc = error(_("%s did not send all necessary objects\n"), url); goto abort; } @@@ -866,6 -866,7 +866,7 @@@ static int quickfetch(struct ref *ref_map) { struct ref *rm = ref_map; + struct check_connected_options opt = CHECK_CONNECTED_INIT; /* * If we are deepening a shallow clone we already have these @@@ -876,7 -877,8 +877,8 @@@ */ if (depth) return -1; - return check_everything_connected(iterate_ref_map, 1, &rm); + opt.quiet = 1; + return check_connected(iterate_ref_map, &rm, &opt); } static int fetch_refs(struct transport *transport, struct ref *ref_map) @@@ -921,7 -923,7 +923,7 @@@ static int prune_refs(struct refspec *r for (ref = stale_refs; ref; ref = ref->next) string_list_append(&refnames, ref->name); - result = delete_refs(&refnames); + result = delete_refs(&refnames, 0); string_list_clear(&refnames, 0); } diff --combined builtin/index-pack.c index 1008d7f63c,54f2cfbd6e..1d2ea583a4 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@@ -77,6 -77,7 +77,7 @@@ static int strict static int do_fsck_object; static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT; static int verbose; + static int show_resolving_progress; static int show_stat; static int check_self_contained_and_connected; @@@ -338,10 -339,10 +339,10 @@@ static void parse_pack_header(void use(sizeof(struct pack_header)); } -static NORETURN void bad_object(unsigned long offset, const char *format, +static NORETURN void bad_object(off_t offset, const char *format, ...) __attribute__((format (printf, 2, 3))); -static NORETURN void bad_object(unsigned long offset, const char *format, ...) +static NORETURN void bad_object(off_t offset, const char *format, ...) { va_list params; char buf[1024]; @@@ -349,8 -350,7 +350,8 @@@ va_start(params, format); vsnprintf(buf, sizeof(buf), format, params); va_end(params); - die(_("pack has bad object at offset %lu: %s"), offset, buf); + die(_("pack has bad object at offset %"PRIuMAX": %s"), + (uintmax_t)offset, buf); } static inline struct thread_local *get_thread_data(void) @@@ -430,7 -430,7 +431,7 @@@ static int is_delta_type(enum object_ty return (type == OBJ_REF_DELTA || type == OBJ_OFS_DELTA); } -static void *unpack_entry_data(unsigned long offset, unsigned long size, +static void *unpack_entry_data(off_t offset, unsigned long size, enum object_type type, unsigned char *sha1) { static char fixed_buf[8192]; @@@ -550,13 -550,13 +551,13 @@@ static void *unpack_data(struct object_ void *cb_data) { off_t from = obj[0].idx.offset + obj[0].hdr_size; - unsigned long len = obj[1].idx.offset - from; + off_t len = obj[1].idx.offset - from; unsigned char *data, *inbuf; git_zstream stream; int status; data = xmallocz(consume ? 64*1024 : obj->size); - inbuf = xmalloc((len < 64*1024) ? len : 64*1024); + inbuf = xmalloc((len < 64*1024) ? (int)len : 64*1024); memset(&stream, 0, sizeof(stream)); git_inflate_init(&stream); @@@ -564,15 -564,15 +565,15 @@@ stream.avail_out = consume ? 64*1024 : obj->size; do { - ssize_t n = (len < 64*1024) ? len : 64*1024; + ssize_t n = (len < 64*1024) ? (ssize_t)len : 64*1024; n = xpread(get_thread_data()->pack_fd, inbuf, n, from); if (n < 0) die_errno(_("cannot pread pack file")); if (!n) - die(Q_("premature end of pack file, %lu byte missing", - "premature end of pack file, %lu bytes missing", - len), - len); + die(Q_("premature end of pack file, %"PRIuMAX" byte missing", + "premature end of pack file, %"PRIuMAX" bytes missing", + (unsigned int)len), + (uintmax_t)len); from += n; len -= n; stream.next_in = inbuf; @@@ -1191,7 -1191,7 +1192,7 @@@ static void resolve_deltas(void qsort(ref_deltas, nr_ref_deltas, sizeof(struct ref_delta_entry), compare_ref_delta_entry); - if (verbose) + if (verbose || show_resolving_progress) progress = start_progress(_("Resolving deltas"), nr_ref_deltas + nr_ofs_deltas); @@@ -1626,6 -1626,7 +1627,7 @@@ int cmd_index_pack(int argc, const cha struct pack_idx_option opts; unsigned char pack_sha1[20]; unsigned foreign_nr = 1; /* zero is a "good" value, assume bad */ + int report_end_of_input = 0; if (argc == 2 && !strcmp(argv[1], "-h")) usage(index_pack_usage); @@@ -1695,6 -1696,10 +1697,10 @@@ input_len = sizeof(*hdr); } else if (!strcmp(arg, "-v")) { verbose = 1; + } else if (!strcmp(arg, "--show-resolving-progress")) { + show_resolving_progress = 1; + } else if (!strcmp(arg, "--report-end-of-input")) { + report_end_of_input = 1; } else if (!strcmp(arg, "-o")) { if (index_name || (i+1) >= argc) usage(index_pack_usage); @@@ -1752,6 -1757,8 +1758,8 @@@ obj_stat = xcalloc(st_add(nr_objects, 1), sizeof(struct object_stat)); ofs_deltas = xcalloc(nr_objects, sizeof(struct ofs_delta_entry)); parse_pack_objects(pack_sha1); + if (report_end_of_input) + write_in_full(2, "\0", 1); resolve_deltas(); conclude_pack(fix_thin_pack, curr_pack, pack_sha1); free(ofs_deltas); diff --combined builtin/receive-pack.c index 3c9360aa6d,e41f55f4f5..92e1213ecc --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@@ -44,12 -44,10 +44,12 @@@ static struct strbuf fsck_msg_types = S static int receive_unpack_limit = -1; static int transfer_unpack_limit = -1; static int advertise_atomic_push = 1; +static int advertise_push_options; static int unpack_limit = 100; static int report_status; static int use_sideband; static int use_atomic; +static int use_push_options; static int quiet; static int prefer_ofs_delta = 1; static int auto_update_server_info; @@@ -78,6 -76,13 +78,13 @@@ static long nonce_stamp_slop static unsigned long nonce_stamp_slop_limit; static struct ref_transaction *transaction; + static enum { + KEEPALIVE_NEVER = 0, + KEEPALIVE_AFTER_NUL, + KEEPALIVE_ALWAYS + } use_keepalive; + static int keepalive_in_sec = 5; + static enum deny_action parse_deny_action(const char *var, const char *value) { if (value) { @@@ -195,11 -200,11 +202,16 @@@ static int receive_pack_config(const ch return 0; } + if (strcmp(var, "receive.advertisepushoptions") == 0) { + advertise_push_options = git_config_bool(var, value); + return 0; + } + + if (strcmp(var, "receive.keepalive") == 0) { + keepalive_in_sec = git_config_int(var, value); + return 0; + } + return git_default_config(var, value, cb); } @@@ -218,8 -223,6 +230,8 @@@ static void show_ref(const char *path, strbuf_addstr(&cap, " ofs-delta"); if (push_cert_nonce) strbuf_addf(&cap, " push-cert=%s", push_cert_nonce); + if (advertise_push_options) + strbuf_addstr(&cap, " push-options"); strbuf_addf(&cap, " agent=%s", git_user_agent_sanitized()); packet_write(1, "%s %s%c%s\n", sha1_to_hex(sha1), path, 0, cap.buf); @@@ -328,10 -331,60 +340,60 @@@ static void rp_error(const char *err, . static int copy_to_sideband(int in, int out, void *arg) { char data[128]; + int keepalive_active = 0; + + if (keepalive_in_sec <= 0) + use_keepalive = KEEPALIVE_NEVER; + if (use_keepalive == KEEPALIVE_ALWAYS) + keepalive_active = 1; + while (1) { - ssize_t sz = xread(in, data, sizeof(data)); + ssize_t sz; + + if (keepalive_active) { + struct pollfd pfd; + int ret; + + pfd.fd = in; + pfd.events = POLLIN; + ret = poll(&pfd, 1, 1000 * keepalive_in_sec); + + if (ret < 0) { + if (errno == EINTR) + continue; + else + break; + } else if (ret == 0) { + /* no data; send a keepalive packet */ + static const char buf[] = "0005\1"; + write_or_die(1, buf, sizeof(buf) - 1); + continue; + } /* else there is actual data to read */ + } + + sz = xread(in, data, sizeof(data)); if (sz <= 0) break; + + if (use_keepalive == KEEPALIVE_AFTER_NUL && !keepalive_active) { + const char *p = memchr(data, '\0', sz); + if (p) { + /* + * The NUL tells us to start sending keepalives. Make + * sure we send any other data we read along + * with it. + */ + keepalive_active = 1; + send_sideband(1, 2, data, p - data, use_sideband); + send_sideband(1, 2, p + 1, sz - (p - data + 1), use_sideband); + continue; + } + } + + /* + * Either we're not looking for a NUL signal, or we didn't see + * it yet; just pass along the data. + */ send_sideband(1, 2, data, sz, use_sideband); } close(in); @@@ -559,16 -612,8 +621,16 @@@ static void prepare_push_cert_sha1(stru } } +struct receive_hook_feed_state { + struct command *cmd; + int skip_broken; + struct strbuf buf; + const struct string_list *push_options; +}; + typedef int (*feed_fn)(void *, const char **, size_t *); -static int run_and_feed_hook(const char *hook_name, feed_fn feed, void *feed_state) +static int run_and_feed_hook(const char *hook_name, feed_fn feed, + struct receive_hook_feed_state *feed_state) { struct child_process proc = CHILD_PROCESS_INIT; struct async muxer; @@@ -584,16 -629,6 +646,16 @@@ proc.argv = argv; proc.in = -1; proc.stdout_to_stderr = 1; + if (feed_state->push_options) { + int i; + for (i = 0; i < feed_state->push_options->nr; i++) + argv_array_pushf(&proc.env_array, + "GIT_PUSH_OPTION_%d=%s", i, + feed_state->push_options->items[i].string); + argv_array_pushf(&proc.env_array, "GIT_PUSH_OPTION_COUNT=%d", + feed_state->push_options->nr); + } else + argv_array_pushf(&proc.env_array, "GIT_PUSH_OPTION_COUNT"); if (use_sideband) { memset(&muxer, 0, sizeof(muxer)); @@@ -633,6 -668,12 +695,6 @@@ return finish_command(&proc); } -struct receive_hook_feed_state { - struct command *cmd; - int skip_broken; - struct strbuf buf; -}; - static int feed_receive_hook(void *state_, const char **bufp, size_t *sizep) { struct receive_hook_feed_state *state = state_; @@@ -655,10 -696,8 +717,10 @@@ return 0; } -static int run_receive_hook(struct command *commands, const char *hook_name, - int skip_broken) +static int run_receive_hook(struct command *commands, + const char *hook_name, + int skip_broken, + const struct string_list *push_options) { struct receive_hook_feed_state state; int status; @@@ -669,7 -708,6 +731,7 @@@ if (feed_receive_hook(&state, NULL, NULL)) return 0; state.cmd = commands; + state.push_options = push_options; status = run_and_feed_hook(hook_name, feed_receive_hook, &state); strbuf_release(&state.buf); return status; @@@ -761,7 -799,7 +823,7 @@@ static int update_shallow_ref(struct co { static struct lock_file shallow_lock; struct sha1_array extra = SHA1_ARRAY_INIT; - const char *alt_file; + struct check_connected_options opt = CHECK_CONNECTED_INIT; uint32_t mask = 1 << (cmd->index % 32); int i; @@@ -773,9 -811,8 +835,8 @@@ !delayed_reachability_test(si, i)) sha1_array_append(&extra, si->shallow->sha1[i]); - setup_alternate_shallow(&shallow_lock, &alt_file, &extra); - if (check_shallow_connected(command_singleton_iterator, - 0, cmd, alt_file)) { + setup_alternate_shallow(&shallow_lock, &opt.shallow_file, &extra); + if (check_connected(command_singleton_iterator, cmd, &opt)) { rollback_lock_file(&shallow_lock); sha1_array_clear(&extra); return -1; @@@ -1184,8 -1221,8 +1245,8 @@@ static void set_connectivity_errors(str if (shallow_update && si->shallow_ref[cmd->index]) /* to be checked in update_shallow_ref() */ continue; - if (!check_everything_connected(command_singleton_iterator, - 0, &singleton)) + if (!check_connected(command_singleton_iterator, &singleton, + NULL)) continue; cmd->error_string = "missing necessary objects"; } @@@ -1340,12 -1377,14 +1401,15 @@@ cleanup static void execute_commands(struct command *commands, const char *unpacker_error, - struct shallow_info *si) + struct shallow_info *si, + const struct string_list *push_options) { + struct check_connected_options opt = CHECK_CONNECTED_INIT; struct command *cmd; unsigned char sha1[20]; struct iterate_data data; + struct async muxer; + int err_fd = 0; if (unpacker_error) { for (cmd = commands; cmd; cmd = cmd->next) @@@ -1353,14 -1392,28 +1417,28 @@@ return; } + if (use_sideband) { + memset(&muxer, 0, sizeof(muxer)); + muxer.proc = copy_to_sideband; + muxer.in = -1; + if (!start_async(&muxer)) + err_fd = muxer.in; + /* ...else, continue without relaying sideband */ + } + data.cmds = commands; data.si = si; - if (check_everything_connected(iterate_receive_command_list, 0, &data)) + opt.err_fd = err_fd; + opt.progress = err_fd && !quiet; + if (check_connected(iterate_receive_command_list, &data, &opt)) set_connectivity_errors(commands, si); + if (use_sideband) + finish_async(&muxer); + reject_updates_to_hidden(commands); - if (run_receive_hook(commands, "pre-receive", 0)) { + if (run_receive_hook(commands, "pre-receive", 0, push_options)) { for (cmd = commands; cmd; cmd = cmd->next) { if (!cmd->error_string) cmd->error_string = "pre-receive hook declined"; @@@ -1464,9 -1517,6 +1542,9 @@@ static struct command *read_head_info(s if (advertise_atomic_push && parse_feature_request(feature_list, "atomic")) use_atomic = 1; + if (advertise_push_options + && parse_feature_request(feature_list, "push-options")) + use_push_options = 1; } if (!strcmp(line, "push-cert")) { @@@ -1499,21 -1549,6 +1577,21 @@@ return commands; } +static void read_push_options(struct string_list *options) +{ + while (1) { + char *line; + int len; + + line = packet_read_line(0, &len); + + if (!line) + break; + + string_list_append(options, line); + } +} + static const char *parse_pack_header(struct pack_header *hdr) { switch (read_pack_header(0, hdr)) { @@@ -1591,6 -1626,10 +1669,10 @@@ static const char *unpack(int err_fd, s (uintmax_t)getpid(), hostname); + if (!quiet && err_fd) + argv_array_push(&child.args, "--show-resolving-progress"); + if (use_sideband) + argv_array_push(&child.args, "--report-end-of-input"); if (fsck_objects) argv_array_pushf(&child.args, "--strict%s", fsck_msg_types.buf); @@@ -1620,6 -1659,7 +1702,7 @@@ static const char *unpack_with_sideband if (!use_sideband) return unpack(0, si); + use_keepalive = KEEPALIVE_AFTER_NUL; memset(&muxer, 0, sizeof(muxer)); muxer.proc = copy_to_sideband; muxer.in = -1; @@@ -1799,10 -1839,6 +1882,10 @@@ int cmd_receive_pack(int argc, const ch if ((commands = read_head_info(&shallow)) != NULL) { const char *unpack_status = NULL; + struct string_list push_options = STRING_LIST_INIT_DUP; + + if (use_push_options) + read_push_options(&push_options); prepare_shallow_info(&si, &shallow); if (!si.nr_ours && !si.nr_theirs) @@@ -1811,17 -1847,14 +1894,18 @@@ unpack_status = unpack_with_sideband(&si); update_shallow_info(commands, &si, &ref); } + use_keepalive = KEEPALIVE_ALWAYS; - execute_commands(commands, unpack_status, &si); + execute_commands(commands, unpack_status, &si, + &push_options); if (pack_lockfile) unlink_or_warn(pack_lockfile); if (report_status) report(commands, unpack_status); - run_receive_hook(commands, "post-receive", 1); + run_receive_hook(commands, "post-receive", 1, + &push_options); run_update_post_hook(commands); + if (push_options.nr) + string_list_clear(&push_options, 0); if (auto_gc) { const char *argv_gc_auto[] = { "gc", "--auto", "--quiet", NULL,