From: Junio C Hamano Date: Thu, 2 Aug 2018 22:30:43 +0000 (-0700) Subject: Merge branch 'jt/fetch-nego-tip' X-Git-Tag: v2.19.0-rc0~102 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/30bf8d9f4f06d8e35793612c8611cb03c53a73db?hp=-c Merge branch 'jt/fetch-nego-tip' "git fetch" learned a new option "--negotiation-tip" to limit the set of commits it tells the other end as "have", to reduce wasted bandwidth and cycles, which would be helpful when the receiving repository has a lot of refs that have little to do with the history at the remote it is fetching from. * jt/fetch-nego-tip: fetch-pack: support negotiation tip whitelist --- 30bf8d9f4f06d8e35793612c8611cb03c53a73db diff --combined builtin/fetch.c index c3c58a5453,49ab6ac064..34d2bd123b --- a/builtin/fetch.c +++ b/builtin/fetch.c @@@ -6,7 -6,6 +6,7 @@@ #include "repository.h" #include "refs.h" #include "refspec.h" +#include "object-store.h" #include "commit.h" #include "builtin.h" #include "string-list.h" @@@ -64,6 -63,7 +64,7 @@@ static int shown_url = 0 static struct refspec refmap = REFSPEC_INIT_FETCH; static struct list_objects_filter_options filter_options; static struct string_list server_options = STRING_LIST_INIT_DUP; + static struct string_list negotiation_tip = STRING_LIST_INIT_NODUP; static int git_fetch_config(const char *k, const char *v, void *cb) { @@@ -94,6 -94,19 +95,6 @@@ return git_default_config(k, v, cb); } -static int gitmodules_fetch_config(const char *var, const char *value, void *cb) -{ - if (!strcmp(var, "submodule.fetchjobs")) { - max_children = parse_submodule_fetchjobs(var, value); - return 0; - } else if (!strcmp(var, "fetch.recursesubmodules")) { - recurse_submodules = parse_fetch_recurse_submodules_arg(var, value); - return 0; - } - - return 0; -} - static int parse_refmap_arg(const struct option *opt, const char *arg, int unset) { /* @@@ -162,6 -175,8 +163,8 @@@ static struct option builtin_fetch_opti TRANSPORT_FAMILY_IPV4), OPT_SET_INT('6', "ipv6", &family, N_("use IPv6 addresses only"), TRANSPORT_FAMILY_IPV6), + OPT_STRING_LIST(0, "negotiation-tip", &negotiation_tip, N_("revision"), + N_("report that we have only objects reachable from this object")), OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options), OPT_END() }; @@@ -242,9 -257,9 +245,9 @@@ static int will_fetch(struct ref **head return 0; } -static void find_non_local_tags(struct transport *transport, - struct ref **head, - struct ref ***tail) +static void find_non_local_tags(const struct ref *refs, + struct ref **head, + struct ref ***tail) { struct string_list existing_refs = STRING_LIST_INIT_DUP; struct string_list remote_refs = STRING_LIST_INIT_NODUP; @@@ -252,7 -267,7 +255,7 @@@ struct string_list_item *item = NULL; for_each_ref(add_existing, &existing_refs); - for (ref = transport_get_remote_refs(transport, NULL); ref; ref = ref->next) { + for (ref = refs; ref; ref = ref->next) { if (!starts_with(ref->name, "refs/tags/")) continue; @@@ -326,8 -341,7 +329,8 @@@ string_list_clear(&remote_refs, 0); } -static struct ref *get_ref_map(struct transport *transport, +static struct ref *get_ref_map(struct remote *remote, + const struct ref *remote_refs, struct refspec *rs, int tags, int *autotags) { @@@ -335,11 -349,26 +338,11 @@@ struct ref *rm; struct ref *ref_map = NULL; struct ref **tail = &ref_map; - struct argv_array ref_prefixes = ARGV_ARRAY_INIT; /* opportunistically-updated references: */ struct ref *orefs = NULL, **oref_tail = &orefs; - const struct ref *remote_refs; - - if (rs->nr) - refspec_ref_prefixes(rs, &ref_prefixes); - else if (transport->remote && transport->remote->fetch.nr) - refspec_ref_prefixes(&transport->remote->fetch, &ref_prefixes); - - if (ref_prefixes.argc && - (tags == TAGS_SET || (tags == TAGS_DEFAULT && !rs->nr))) { - argv_array_push(&ref_prefixes, "refs/tags/"); - } - - remote_refs = transport_get_remote_refs(transport, &ref_prefixes); - - argv_array_clear(&ref_prefixes); + struct string_list existing_refs = STRING_LIST_INIT_DUP; if (rs->nr) { struct refspec *fetch_refspec; @@@ -376,7 -405,7 +379,7 @@@ if (refmap.nr) fetch_refspec = &refmap; else - fetch_refspec = &transport->remote->fetch; + fetch_refspec = &remote->fetch; for (i = 0; i < fetch_refspec->nr; i++) get_fetch_map(ref_map, &fetch_refspec->items[i], &oref_tail, 1); @@@ -384,6 -413,7 +387,6 @@@ die("--refmap option is only meaningful with command-line refspec(s)."); } else { /* Use the defaults */ - struct remote *remote = transport->remote; struct branch *branch = branch_get(NULL); int has_merge = branch_has_merge_config(branch); if (remote && @@@ -422,7 -452,7 +425,7 @@@ /* also fetch all tags */ get_fetch_map(remote_refs, tag_refspec, &tail, 0); else if (tags == TAGS_DEFAULT && *autotags) - find_non_local_tags(transport, &ref_map, &tail); + find_non_local_tags(remote_refs, &ref_map, &tail); /* Now append any refs to be updated opportunistically: */ *tail = orefs; @@@ -431,23 -461,7 +434,23 @@@ tail = &rm->next; } - return ref_remove_duplicates(ref_map); + ref_map = ref_remove_duplicates(ref_map); + + for_each_ref(add_existing, &existing_refs); + for (rm = ref_map; rm; rm = rm->next) { + if (rm->peer_ref) { + struct string_list_item *peer_item = + string_list_lookup(&existing_refs, + rm->peer_ref->name); + if (peer_item) { + struct object_id *old_oid = peer_item->util; + oidcpy(&rm->peer_ref->old_oid, old_oid); + } + } + } + string_list_clear(&existing_refs, 1); + + return ref_map; } #define STORE_REF_ERROR_OTHER 1 @@@ -672,10 -686,8 +675,10 @@@ static int update_local_ref(struct ref return r; } - current = lookup_commit_reference_gently(&ref->old_oid, 1); - updated = lookup_commit_reference_gently(&ref->new_oid, 1); + current = lookup_commit_reference_gently(the_repository, + &ref->old_oid, 1); + updated = lookup_commit_reference_gently(the_repository, + &ref->new_oid, 1); if (!current || !updated) { const char *msg; const char *what; @@@ -759,7 -771,7 +762,7 @@@ static int iterate_ref_map(void *cb_dat } static int store_updated_refs(const char *raw_url, const char *remote_name, - struct ref *ref_map) + int connectivity_checked, struct ref *ref_map) { FILE *fp; struct commit *commit; @@@ -768,7 -780,7 +771,7 @@@ const char *what, *kind; struct ref *rm; char *url; - const char *filename = dry_run ? "/dev/null" : git_path_fetch_head(); + const char *filename = dry_run ? "/dev/null" : git_path_fetch_head(the_repository); int want_status; int summary_width = transport_summary_width(ref_map); @@@ -781,12 -793,10 +784,12 @@@ else url = xstrdup("foreign"); - rm = ref_map; - if (check_connected(iterate_ref_map, &rm, NULL)) { - rc = error(_("%s did not send all necessary objects\n"), url); - goto abort; + if (!connectivity_checked) { + rm = ref_map; + if (check_connected(iterate_ref_map, &rm, NULL)) { + rc = error(_("%s did not send all necessary objects\n"), url); + goto abort; + } } prepare_format_display(ref_map); @@@ -810,8 -820,7 +813,8 @@@ continue; } - commit = lookup_commit_reference_gently(&rm->old_oid, + commit = lookup_commit_reference_gently(the_repository, + &rm->old_oid, 1); if (!commit) rm->fetch_head_status = FETCH_HEAD_NOT_FOR_MERGE; @@@ -939,32 -948,15 +942,32 @@@ static int quickfetch(struct ref *ref_m return check_connected(iterate_ref_map, &rm, &opt); } -static int fetch_refs(struct transport *transport, struct ref *ref_map) +static int fetch_refs(struct transport *transport, struct ref *ref_map, + struct ref **updated_remote_refs) { int ret = quickfetch(ref_map); if (ret) - ret = transport_fetch_refs(transport, ref_map); + ret = transport_fetch_refs(transport, ref_map, + updated_remote_refs); if (!ret) - ret |= store_updated_refs(transport->url, - transport->remote->name, - ref_map); + /* + * Keep the new pack's ".keep" file around to allow the caller + * time to update refs to reference the new objects. + */ + return 0; + transport_unlock_pack(transport); + return ret; +} + +/* Update local refs based on the ref values fetched from a remote */ +static int consume_refs(struct transport *transport, struct ref *ref_map) +{ + int connectivity_checked = transport->smart_options + ? transport->smart_options->connectivity_checked : 0; + int ret = store_updated_refs(transport->url, + transport->remote->name, + connectivity_checked, + ref_map); transport_unlock_pack(transport); return ret; } @@@ -1040,7 -1032,7 +1043,7 @@@ static void check_not_current_branch(st static int truncate_fetch_head(void) { - const char *filename = git_path_fetch_head(); + const char *filename = git_path_fetch_head(the_repository); FILE *fp = fopen_for_writing(filename); if (!fp) @@@ -1060,6 -1052,40 +1063,40 @@@ static void set_option(struct transpor name, transport->url); } + + static int add_oid(const char *refname, const struct object_id *oid, int flags, + void *cb_data) + { + struct oid_array *oids = cb_data; + + oid_array_append(oids, oid); + return 0; + } + + static void add_negotiation_tips(struct git_transport_options *smart_options) + { + struct oid_array *oids = xcalloc(1, sizeof(*oids)); + int i; + + for (i = 0; i < negotiation_tip.nr; i++) { + const char *s = negotiation_tip.items[i].string; + int old_nr; + if (!has_glob_specials(s)) { + struct object_id oid; + if (get_oid(s, &oid)) + die("%s is not a valid object", s); + oid_array_append(oids, &oid); + continue; + } + old_nr = oids->nr; + for_each_glob_ref(add_oid, s, oids); + if (old_nr == oids->nr) + warning("Ignoring --negotiation-tip=%s because it does not match any refs", + s); + } + smart_options->negotiation_tips = oids; + } + static struct transport *prepare_transport(struct remote *remote, int deepen) { struct transport *transport; @@@ -1086,6 -1112,12 +1123,12 @@@ filter_options.filter_spec); set_option(transport, TRANS_OPT_FROM_PROMISOR, "1"); } + if (negotiation_tip.nr) { + if (transport->smart_options) + add_negotiation_tips(transport->smart_options); + else + warning("Ignoring --negotiation-tip because the protocol does not support it."); + } return transport; } @@@ -1110,8 -1142,7 +1153,8 @@@ static void backfill_tags(struct transp transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, NULL); transport_set_option(transport, TRANS_OPT_DEPTH, "0"); transport_set_option(transport, TRANS_OPT_DEEPEN_RELATIVE, NULL); - fetch_refs(transport, ref_map); + if (!fetch_refs(transport, ref_map, NULL)) + consume_refs(transport, ref_map); if (gsecondary) { transport_disconnect(gsecondary); @@@ -1122,12 -1153,13 +1165,12 @@@ static int do_fetch(struct transport *transport, struct refspec *rs) { - struct string_list existing_refs = STRING_LIST_INIT_DUP; struct ref *ref_map; - struct ref *rm; int autotags = (transport->remote->fetch_tags == 1); int retcode = 0; - - for_each_ref(add_existing, &existing_refs); + const struct ref *remote_refs; + struct ref *updated_remote_refs = NULL; + struct argv_array ref_prefixes = ARGV_ARRAY_INIT; if (tags == TAGS_DEFAULT) { if (transport->remote->fetch_tags == 2) @@@ -1143,24 -1175,22 +1186,24 @@@ goto cleanup; } - ref_map = get_ref_map(transport, rs, tags, &autotags); - if (!update_head_ok) - check_not_current_branch(ref_map); + if (rs->nr) + refspec_ref_prefixes(rs, &ref_prefixes); + else if (transport->remote && transport->remote->fetch.nr) + refspec_ref_prefixes(&transport->remote->fetch, &ref_prefixes); - for (rm = ref_map; rm; rm = rm->next) { - if (rm->peer_ref) { - struct string_list_item *peer_item = - string_list_lookup(&existing_refs, - rm->peer_ref->name); - if (peer_item) { - struct object_id *old_oid = peer_item->util; - oidcpy(&rm->peer_ref->old_oid, old_oid); - } - } + if (ref_prefixes.argc && + (tags == TAGS_SET || (tags == TAGS_DEFAULT && !rs->nr))) { + argv_array_push(&ref_prefixes, "refs/tags/"); } + remote_refs = transport_get_remote_refs(transport, &ref_prefixes); + argv_array_clear(&ref_prefixes); + + ref_map = get_ref_map(transport->remote, remote_refs, rs, + tags, &autotags); + if (!update_head_ok) + check_not_current_branch(ref_map); + if (tags == TAGS_DEFAULT && autotags) transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, "1"); if (prune) { @@@ -1177,24 -1207,7 +1220,24 @@@ transport->url); } } - if (fetch_refs(transport, ref_map)) { + + if (fetch_refs(transport, ref_map, &updated_remote_refs)) { + free_refs(ref_map); + retcode = 1; + goto cleanup; + } + if (updated_remote_refs) { + /* + * Regenerate ref_map using the updated remote refs. This is + * to account for additional information which may be provided + * by the transport (e.g. shallow info). + */ + free_refs(ref_map); + ref_map = get_ref_map(transport->remote, updated_remote_refs, rs, + tags, &autotags); + free_refs(updated_remote_refs); + } + if (consume_refs(transport, ref_map)) { free_refs(ref_map); retcode = 1; goto cleanup; @@@ -1206,13 -1219,14 +1249,13 @@@ if (tags == TAGS_DEFAULT && autotags) { struct ref **tail = &ref_map; ref_map = NULL; - find_non_local_tags(transport, &ref_map, &tail); + find_non_local_tags(remote_refs, &ref_map, &tail); if (ref_map) backfill_tags(transport, ref_map); free_refs(ref_map); } cleanup: - string_list_clear(&existing_refs, 1); return retcode; } @@@ -1462,7 -1476,7 +1505,7 @@@ int cmd_fetch(int argc, const char **ar for (i = 1; i < argc; i++) strbuf_addf(&default_rla, " %s", argv[i]); - config_from_gitmodules(gitmodules_fetch_config, NULL); + fetch_config_from_gitmodules(&max_children, &recurse_submodules); git_config(git_fetch_config, NULL); argc = parse_options(argc, argv, prefix, @@@ -1478,7 -1492,7 +1521,7 @@@ if (unshallow) { if (depth) die(_("--depth and --unshallow cannot be used together")); - else if (!is_repository_shallow()) + else if (!is_repository_shallow(the_repository)) die(_("--unshallow on a complete repository does not make sense")); else depth = xstrfmt("%d", INFINITE_DEPTH); diff --combined fetch-pack.c index 5056e22e6d,1e50d90082..0a9d184adb --- a/fetch-pack.c +++ b/fetch-pack.c @@@ -18,8 -18,6 +18,8 @@@ #include "sha1-array.h" #include "oidset.h" #include "packfile.h" +#include "object-store.h" +#include "connected.h" #include "fetch-negotiator.h" static int transfer_unpack_limit = -1; @@@ -78,7 -76,7 +78,7 @@@ static void cache_one_alternate(const c void *vcache) { struct alternate_object_cache *cache = vcache; - struct object *obj = parse_object(oid); + struct object *obj = parse_object(the_repository, oid); if (!obj || (obj->flags & ALTERNATE)) return; @@@ -109,9 -107,7 +109,9 @@@ static int rev_list_insert_ref(struct f const char *refname, const struct object_id *oid) { - struct object *o = deref_tag(parse_object(oid), refname, 0); + struct object *o = deref_tag(the_repository, + parse_object(the_repository, oid), + refname, 0); if (o && o->type == OBJ_COMMIT) negotiator->add_tip(negotiator, (struct commit *)o); @@@ -217,6 -213,22 +217,22 @@@ static int next_flush(int stateless_rpc return count; } + static void mark_tips(struct fetch_negotiator *negotiator, + const struct oid_array *negotiation_tips) + { + int i; + + if (!negotiation_tips) { + for_each_ref(rev_list_insert_ref_oid, negotiator); + return; + } + + for (i = 0; i < negotiation_tips->nr; i++) + rev_list_insert_ref(negotiator, NULL, + &negotiation_tips->oid[i]); + return; + } + static int find_common(struct fetch_negotiator *negotiator, struct fetch_pack_args *args, int fd[2], struct object_id *result_oid, @@@ -234,7 -246,7 +250,7 @@@ if (args->stateless_rpc && multi_ack == 1) die(_("--stateless-rpc requires multi_ack_detailed")); - for_each_ref(rev_list_insert_ref_oid, negotiator); + mark_tips(negotiator, args->negotiation_tips); for_each_cached_alternate(negotiator, insert_one_alternate_object); fetching = 0; @@@ -253,7 -265,7 +269,7 @@@ * interested in the case we *know* the object is * reachable and we have already scanned it. */ - if (((o = lookup_object(remote->hash)) != NULL) && + if (((o = lookup_object(the_repository, remote->hash)) != NULL) && (o->flags & COMPLETE)) { continue; } @@@ -290,7 -302,7 +306,7 @@@ return 1; } - if (is_repository_shallow()) + if (is_repository_shallow(the_repository)) write_shallow_commits(&req_buf, 1, NULL); if (args->depth > 0) packet_buf_write(&req_buf, "deepen %d", args->depth); @@@ -321,16 -333,16 +337,16 @@@ if (skip_prefix(line, "shallow ", &arg)) { if (get_oid_hex(arg, &oid)) die(_("invalid shallow line: %s"), line); - register_shallow(&oid); + register_shallow(the_repository, &oid); continue; } if (skip_prefix(line, "unshallow ", &arg)) { if (get_oid_hex(arg, &oid)) die(_("invalid unshallow line: %s"), line); - if (!lookup_object(oid.hash)) + if (!lookup_object(the_repository, oid.hash)) die(_("object not found: %s"), line); /* make sure that it is parsed as shallow */ - if (!parse_object(&oid)) + if (!parse_object(the_repository, &oid)) die(_("error in object: %s"), line); if (unregister_shallow(&oid)) die(_("no shallow found: %s"), line); @@@ -389,10 -401,8 +405,10 @@@ case ACK_ready: case ACK_continue: { struct commit *commit = - lookup_commit(result_oid); + lookup_commit(the_repository, + result_oid); int was_common; + if (!commit) die(_("invalid commit %s"), oid_to_hex(result_oid)); was_common = negotiator->ack(negotiator, commit); @@@ -466,14 -476,14 +482,14 @@@ static struct commit_list *complete static int mark_complete(const struct object_id *oid) { - struct object *o = parse_object(oid); + struct object *o = parse_object(the_repository, oid); while (o && o->type == OBJ_TAG) { struct tag *t = (struct tag *) o; if (!t->tagged) break; /* broken repository */ o->flags |= COMPLETE; - o = parse_object(&t->tagged->oid); + o = parse_object(the_repository, &t->tagged->oid); } if (o && o->type == OBJ_COMMIT) { struct commit *commit = (struct commit *)o; @@@ -554,11 -564,11 +570,11 @@@ static void filter_refs(struct fetch_pa } i++; } - } - if (!keep && args->fetch_all && - (!args->deepen || !starts_with(ref->name, "refs/tags/"))) - keep = 1; + if (!keep && args->fetch_all && + (!args->deepen || !starts_with(ref->name, "refs/tags/"))) + keep = 1; + } if (keep) { *newtail = ref; @@@ -674,7 -684,7 +690,7 @@@ static void mark_complete_and_common_re if (!has_object_file_with_flags(&ref->old_oid, flags)) continue; - o = parse_object(&ref->old_oid); + o = parse_object(the_repository, &ref->old_oid); if (!o) continue; @@@ -705,9 -715,7 +721,9 @@@ * Don't mark them common yet; the server has to be told so first. */ for (ref = *refs; ref; ref = ref->next) { - struct object *o = deref_tag(lookup_object(ref->old_oid.hash), + struct object *o = deref_tag(the_repository, + lookup_object(the_repository, + ref->old_oid.hash), NULL, 0); if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE)) @@@ -735,7 -743,7 +751,7 @@@ static int everything_local(struct fetc const struct object_id *remote = &ref->old_oid; struct object *o; - o = lookup_object(remote->hash); + o = lookup_object(the_repository, remote->hash); if (!o || !(o->flags & COMPLETE)) { retval = 0; print_verbose(args, "want %s (%s)", oid_to_hex(remote), @@@ -902,7 -910,7 +918,7 @@@ static struct ref *do_fetch_pack(struc sort_ref_list(&ref, ref_compare_name); QSORT(sought, nr_sought, cmp_ref_by_name); - if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow")) + if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow")) die(_("Server does not support shallow clients")); if (args->depth > 0 || args->deepen_since || args->deepen_not) args->deepen = 1; @@@ -1003,7 -1011,7 +1019,7 @@@ static void add_shallow_requests(struct strbuf *req_buf, const struct fetch_pack_args *args) { - if (is_repository_shallow()) + if (is_repository_shallow(the_repository)) write_shallow_commits(req_buf, 1, NULL); if (args->depth > 0) packet_buf_write(req_buf, "deepen %d", args->depth); @@@ -1022,10 -1030,9 +1038,10 @@@ static void add_wants(const struct ref *wants, struct strbuf *req_buf) { + int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0); + for ( ; wants ; wants = wants->next) { const struct object_id *remote = &wants->old_oid; - const char *remote_hex; struct object *o; /* @@@ -1038,15 -1045,13 +1054,15 @@@ * interested in the case we *know* the object is * reachable and we have already scanned it. */ - if (((o = lookup_object(remote->hash)) != NULL) && + if (((o = lookup_object(the_repository, remote->hash)) != NULL) && (o->flags & COMPLETE)) { continue; } - remote_hex = oid_to_hex(remote); - packet_buf_write(req_buf, "want %s\n", remote_hex); + if (!use_ref_in_want || wants->exact_oid) + packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote)); + else + packet_buf_write(req_buf, "want-ref %s\n", wants->name); } } @@@ -1121,7 -1126,7 +1137,7 @@@ static int send_fetch_request(struct fe /* Add shallow-info and deepen request */ if (server_supports_feature("fetch", "shallow", 0)) add_shallow_requests(&req_buf, args); - else if (is_repository_shallow() || args->deepen) + else if (is_repository_shallow(the_repository) || args->deepen) die(_("Server does not support shallow requests")); /* Add filter */ @@@ -1202,7 -1207,7 +1218,7 @@@ static int process_acks(struct fetch_ne if (!get_oid_hex(arg, &oid)) { struct commit *commit; oidset_insert(common, &oid); - commit = lookup_commit(&oid); + commit = lookup_commit(the_repository, &oid); negotiator->ack(negotiator, commit); } continue; @@@ -1235,16 -1240,16 +1251,16 @@@ static void receive_shallow_info(struc if (skip_prefix(reader->line, "shallow ", &arg)) { if (get_oid_hex(arg, &oid)) die(_("invalid shallow line: %s"), reader->line); - register_shallow(&oid); + register_shallow(the_repository, &oid); continue; } if (skip_prefix(reader->line, "unshallow ", &arg)) { if (get_oid_hex(arg, &oid)) die(_("invalid unshallow line: %s"), reader->line); - if (!lookup_object(oid.hash)) + if (!lookup_object(the_repository, oid.hash)) die(_("object not found: %s"), reader->line); /* make sure that it is parsed as shallow */ - if (!parse_object(&oid)) + if (!parse_object(the_repository, &oid)) die(_("error in object: %s"), reader->line); if (unregister_shallow(&oid)) die(_("no shallow found: %s"), reader->line); @@@ -1261,32 -1266,6 +1277,32 @@@ args->deepen = 1; } +static void receive_wanted_refs(struct packet_reader *reader, struct ref *refs) +{ + process_section_header(reader, "wanted-refs", 0); + while (packet_reader_read(reader) == PACKET_READ_NORMAL) { + struct object_id oid; + const char *end; + struct ref *r = NULL; + + if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ') + die("expected wanted-ref, got '%s'", reader->line); + + for (r = refs; r; r = r->next) { + if (!strcmp(end, r->name)) { + oidcpy(&r->old_oid, &oid); + break; + } + } + + if (!r) + die("unexpected wanted-ref: '%s'", reader->line); + } + + if (reader->status != PACKET_READ_DELIM) + die("error processing wanted refs: %d", reader->status); +} + enum fetch_state { FETCH_CHECK_LOCAL = 0, FETCH_SEND_REQUEST, @@@ -1332,7 -1311,7 +1348,7 @@@ static struct ref *do_fetch_pack_v2(str else state = FETCH_SEND_REQUEST; - for_each_ref(rev_list_insert_ref_oid, &negotiator); + mark_tips(&negotiator, args->negotiation_tips); for_each_cached_alternate(&negotiator, insert_one_alternate_object); break; @@@ -1363,9 -1342,6 +1379,9 @@@ if (process_section_header(&reader, "shallow-info", 1)) receive_shallow_info(args, &reader); + if (process_section_header(&reader, "wanted-refs", 1)) + receive_wanted_refs(&reader, ref); + /* get the pack */ process_section_header(&reader, "packfile", 0); if (get_pack(args, fd, pack_lockfile)) @@@ -1429,17 -1405,16 +1445,17 @@@ static int remove_duplicates_in_refs(st } static void update_shallow(struct fetch_pack_args *args, - struct ref **sought, int nr_sought, + struct ref *refs, struct shallow_info *si) { struct oid_array ref = OID_ARRAY_INIT; int *status; int i; + struct ref *r; if (args->deepen && alternate_shallow_file) { if (*alternate_shallow_file == '\0') { /* --unshallow */ - unlink_or_warn(git_path_shallow()); + unlink_or_warn(git_path_shallow(the_repository)); rollback_lock_file(&shallow_lock); } else commit_lock_file(&shallow_lock); @@@ -1477,8 -1452,8 +1493,8 @@@ remove_nonexistent_theirs_shallow(si); if (!si->nr_ours && !si->nr_theirs) return; - for (i = 0; i < nr_sought; i++) - oid_array_append(&ref, &sought[i]->old_oid); + for (r = refs; r; r = r->next) + oid_array_append(&ref, &r->old_oid); si->ref = &ref; if (args->update_shallow) { @@@ -1512,29 -1487,17 +1528,29 @@@ * remote is also shallow, check what ref is safe to update * without updating .git/shallow */ - status = xcalloc(nr_sought, sizeof(*status)); + status = xcalloc(ref.nr, sizeof(*status)); assign_shallow_commits_to_refs(si, NULL, status); if (si->nr_ours || si->nr_theirs) { - for (i = 0; i < nr_sought; i++) + for (r = refs, i = 0; r; r = r->next, i++) if (status[i]) - sought[i]->status = REF_STATUS_REJECT_SHALLOW; + r->status = REF_STATUS_REJECT_SHALLOW; } free(status); oid_array_clear(&ref); } +static int iterate_ref_map(void *cb_data, struct object_id *oid) +{ + struct ref **rm = cb_data; + struct ref *ref = *rm; + + if (!ref) + return -1; /* end of the list */ + *rm = ref->next; + oidcpy(oid, &ref->old_oid); + return 0; +} + struct ref *fetch_pack(struct fetch_pack_args *args, int fd[], struct child_process *conn, const struct ref *ref, @@@ -1563,25 -1526,7 +1579,25 @@@ ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, &si, pack_lockfile); reprepare_packed_git(the_repository); - update_shallow(args, sought, nr_sought, &si); + + if (!args->cloning && args->deepen) { + struct check_connected_options opt = CHECK_CONNECTED_INIT; + struct ref *iterator = ref_cpy; + opt.shallow_file = alternate_shallow_file; + if (args->deepen) + opt.is_deepening_fetch = 1; + if (check_connected(iterate_ref_map, &iterator, &opt)) { + error(_("remote did not send all necessary objects")); + free_refs(ref_cpy); + ref_cpy = NULL; + rollback_lock_file(&shallow_lock); + goto cleanup; + } + args->connectivity_checked = 1; + } + + update_shallow(args, ref_cpy, &si); +cleanup: clear_shallow_info(&si); return ref_cpy; } diff --combined fetch-pack.h index 2160be9164,1859ee9275..5b6e868802 --- a/fetch-pack.h +++ b/fetch-pack.h @@@ -16,6 -16,13 +16,13 @@@ struct fetch_pack_args const struct string_list *deepen_not; struct list_objects_filter_options filter_options; const struct string_list *server_options; + + /* + * If not NULL, during packfile negotiation, fetch-pack will send "have" + * lines only with these tips and their ancestors. + */ + const struct oid_array *negotiation_tips; + unsigned deepen_relative:1; unsigned quiet:1; unsigned keep_pack:1; @@@ -41,21 -48,6 +48,21 @@@ * regardless of which object flags it uses (if any). */ unsigned no_dependents:1; + + /* + * Because fetch_pack() overwrites the shallow file upon a + * successful deepening non-clone fetch, if this struct + * specifies such a fetch, fetch_pack() needs to perform a + * connectivity check before deciding if a fetch is successful + * (and overwriting the shallow file). fetch_pack() sets this + * field to 1 if such a connectivity check was performed. + * + * This is different from check_self_contained_and_connected + * in that the former allows existing objects in the + * repository to satisfy connectivity needs, whereas the + * latter doesn't. + */ + unsigned connectivity_checked:1; }; /* diff --combined transport-helper.c index 8b5abca29f,ad8f7c7726..eab7f47565 --- a/transport-helper.c +++ b/transport-helper.c @@@ -651,16 -651,14 +651,16 @@@ static int connect_helper(struct transp } static int fetch(struct transport *transport, - int nr_heads, struct ref **to_fetch) + int nr_heads, struct ref **to_fetch, + struct ref **fetched_refs) { struct helper_data *data = transport->data; int i, count; if (process_connect(transport, 0)) { do_take_over(transport); - return transport->vtable->fetch(transport, nr_heads, to_fetch); + return transport->vtable->fetch(transport, nr_heads, to_fetch, + fetched_refs); } count = 0; @@@ -686,6 -684,9 +686,9 @@@ transport, "filter", data->transport_options.filter_options.filter_spec); + if (data->transport_options.negotiation_tips) + warning("Ignoring --negotiation-tip because the protocol does not support it."); + if (data->fetch) return fetch_with_fetch(transport, nr_heads, to_fetch); diff --combined transport.c index fdd813f684,9f10f8ad9f..b64b7bcb86 --- a/transport.c +++ b/transport.c @@@ -151,8 -151,7 +151,8 @@@ static struct ref *get_refs_from_bundle } static int fetch_refs_from_bundle(struct transport *transport, - int nr_heads, struct ref **to_fetch) + int nr_heads, struct ref **to_fetch, + struct ref **fetched_refs) { struct bundle_transport_data *data = transport->data; return unbundle(&data->header, data->fd, @@@ -288,8 -287,7 +288,8 @@@ static struct ref *get_refs_via_connect } static int fetch_refs_via_pack(struct transport *transport, - int nr_heads, struct ref **to_fetch) + int nr_heads, struct ref **to_fetch, + struct ref **fetched_refs) { int ret = 0; struct git_transport_data *data = transport->data; @@@ -320,6 -318,7 +320,7 @@@ args.filter_options = data->options.filter_options; args.stateless_rpc = transport->stateless_rpc; args.server_options = transport->server_options; + args.negotiation_tips = data->options.negotiation_tips; if (!data->got_remote_heads) refs_tmp = get_refs_via_connect(transport, 0, NULL); @@@ -350,19 -349,14 +351,19 @@@ data->got_remote_heads = 0; data->options.self_contained_and_connected = args.self_contained_and_connected; + data->options.connectivity_checked = args.connectivity_checked; if (refs == NULL) ret = -1; if (report_unmatched_refs(to_fetch, nr_heads)) ret = -1; + if (fetched_refs) + *fetched_refs = refs; + else + free_refs(refs); + free_refs(refs_tmp); - free_refs(refs); free(dest); return ret; } @@@ -1222,31 -1216,19 +1223,31 @@@ const struct ref *transport_get_remote_ return transport->remote_refs; } -int transport_fetch_refs(struct transport *transport, struct ref *refs) +int transport_fetch_refs(struct transport *transport, struct ref *refs, + struct ref **fetched_refs) { int rc; int nr_heads = 0, nr_alloc = 0, nr_refs = 0; struct ref **heads = NULL; + struct ref *nop_head = NULL, **nop_tail = &nop_head; struct ref *rm; for (rm = refs; rm; rm = rm->next) { nr_refs++; if (rm->peer_ref && !is_null_oid(&rm->old_oid) && - !oidcmp(&rm->peer_ref->old_oid, &rm->old_oid)) + !oidcmp(&rm->peer_ref->old_oid, &rm->old_oid)) { + /* + * These need to be reported as fetched, but we don't + * actually need to fetch them. + */ + if (fetched_refs) { + struct ref *nop_ref = copy_ref(rm); + *nop_tail = nop_ref; + nop_tail = &nop_ref->next; + } continue; + } ALLOC_GROW(heads, nr_heads + 1, nr_alloc); heads[nr_heads++] = rm; } @@@ -1264,11 -1246,7 +1265,11 @@@ heads[nr_heads++] = rm; } - rc = transport->vtable->fetch(transport, nr_heads, heads); + rc = transport->vtable->fetch(transport, nr_heads, heads, fetched_refs); + if (fetched_refs && nop_head) { + *nop_tail = *fetched_refs; + *fetched_refs = nop_head; + } free(heads); return rc; diff --combined transport.h index 7a9a7fcaf3,d31be5be63..113530ea54 --- a/transport.h +++ b/transport.h @@@ -18,17 -18,6 +18,17 @@@ struct git_transport_options unsigned deepen_relative : 1; unsigned from_promisor : 1; unsigned no_dependents : 1; + + /* + * If this transport supports connect or stateless-connect, + * the corresponding field in struct fetch_pack_args is copied + * here after fetching. + * + * See the definition of connectivity_checked in struct + * fetch_pack_args for more information. + */ + unsigned connectivity_checked:1; + int depth; const char *deepen_since; const struct string_list *deepen_not; @@@ -36,6 -25,16 +36,16 @@@ const char *receivepack; struct push_cas_option *cas; struct list_objects_filter_options filter_options; + + /* + * This is only used during fetch. See the documentation of + * negotiation_tips in struct fetch_pack_args. + * + * This field is only supported by transports that support connect or + * stateless_connect. Set this field directly instead of using + * transport_set_option(). + */ + struct oid_array *negotiation_tips; }; enum transport_family { @@@ -229,8 -228,7 +239,8 @@@ int transport_push(struct transport *co const struct ref *transport_get_remote_refs(struct transport *transport, const struct argv_array *ref_prefixes); -int transport_fetch_refs(struct transport *transport, struct ref *refs); +int transport_fetch_refs(struct transport *transport, struct ref *refs, + struct ref **fetched_refs); void transport_unlock_pack(struct transport *transport); int transport_disconnect(struct transport *transport); char *transport_anonymize_url(const char *url);