From: Junio C Hamano Date: Thu, 2 Aug 2018 22:30:41 +0000 (-0700) Subject: Merge branch 'jt/fetch-pack-negotiator' X-Git-Tag: v2.19.0-rc0~106 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/af8ac73801eebf0f3690c3875751eb9e108ceda8?hp=-c Merge branch 'jt/fetch-pack-negotiator' Code restructuring and a small fix to transport protocol v2 during fetching. * jt/fetch-pack-negotiator: fetch-pack: introduce negotiator API fetch-pack: move common check and marking together fetch-pack: make negotiation-related vars local fetch-pack: use ref adv. to prune "have" sent fetch-pack: directly end negotiation if ACK ready fetch-pack: clear marks before re-marking fetch-pack: split up everything_local() --- af8ac73801eebf0f3690c3875751eb9e108ceda8 diff --combined Makefile index 08e5c54549,96f84d1dcf..ab5f603798 --- a/Makefile +++ b/Makefile @@@ -620,7 -620,6 +620,7 @@@ SCRIPT_LIB += git-mergetool--li SCRIPT_LIB += git-parse-remote SCRIPT_LIB += git-rebase--am SCRIPT_LIB += git-rebase--interactive +SCRIPT_LIB += git-rebase--preserve-merges SCRIPT_LIB += git-rebase--merge SCRIPT_LIB += git-sh-setup SCRIPT_LIB += git-sh-i18n @@@ -690,6 -689,7 +690,6 @@@ PROGRAM_OBJS += http-backend. PROGRAM_OBJS += imap-send.o PROGRAM_OBJS += sh-i18n--envsubst.o PROGRAM_OBJS += shell.o -PROGRAM_OBJS += show-index.o PROGRAM_OBJS += remote-testsvn.o # Binary suffix, set to .exe for Windows builds @@@ -859,6 -859,7 +859,7 @@@ LIB_OBJS += ewah/ewah_bitmap. LIB_OBJS += ewah/ewah_io.o LIB_OBJS += ewah/ewah_rlw.o LIB_OBJS += exec-cmd.o + LIB_OBJS += fetch-negotiator.o LIB_OBJS += fetch-object.o LIB_OBJS += fetch-pack.o LIB_OBJS += fsck.o @@@ -891,6 -892,7 +892,7 @@@ LIB_OBJS += merge-blobs. LIB_OBJS += merge-recursive.o LIB_OBJS += mergesort.o LIB_OBJS += name-hash.o + LIB_OBJS += negotiator/default.o LIB_OBJS += notes.o LIB_OBJS += notes-cache.o LIB_OBJS += notes-merge.o @@@ -1077,7 -1079,6 +1079,7 @@@ BUILTIN_OBJS += builtin/send-pack. BUILTIN_OBJS += builtin/serve.o BUILTIN_OBJS += builtin/shortlog.o BUILTIN_OBJS += builtin/show-branch.o +BUILTIN_OBJS += builtin/show-index.o BUILTIN_OBJS += builtin/show-ref.o BUILTIN_OBJS += builtin/stripspace.o BUILTIN_OBJS += builtin/submodule--helper.o @@@ -1352,19 -1353,17 +1354,19 @@@ ifdef APPLE_COMMON_CRYPT LIB_4_CRYPTO += -framework Security -framework CoreFoundation endif endif -ifdef NEEDS_LIBICONV - ifdef ICONVDIR - BASIC_CFLAGS += -I$(ICONVDIR)/include - ICONV_LINK = -L$(ICONVDIR)/$(lib) $(CC_LD_DYNPATH)$(ICONVDIR)/$(lib) - else - ICONV_LINK = - endif - ifdef NEEDS_LIBINTL_BEFORE_LIBICONV - ICONV_LINK += -lintl +ifndef NO_ICONV + ifdef NEEDS_LIBICONV + ifdef ICONVDIR + BASIC_CFLAGS += -I$(ICONVDIR)/include + ICONV_LINK = -L$(ICONVDIR)/$(lib) $(CC_LD_DYNPATH)$(ICONVDIR)/$(lib) + else + ICONV_LINK = + endif + ifdef NEEDS_LIBINTL_BEFORE_LIBICONV + ICONV_LINK += -lintl + endif + EXTLIBS += $(ICONV_LINK) -liconv endif - EXTLIBS += $(ICONV_LINK) -liconv endif ifdef NEEDS_LIBGEN EXTLIBS += -lgen @@@ -2021,9 -2020,8 +2023,9 @@@ version.sp version.s version.o: GIT-VER version.sp version.s version.o: EXTRA_CPPFLAGS = \ '-DGIT_VERSION="$(GIT_VERSION)"' \ '-DGIT_USER_AGENT=$(GIT_USER_AGENT_CQ_SQ)' \ - '-DGIT_BUILT_FROM_COMMIT="$(shell GIT_CEILING_DIRECTORIES=\"$(CURDIR)/..\" \ - git rev-parse -q --verify HEAD || :)"' + '-DGIT_BUILT_FROM_COMMIT="$(shell \ + GIT_CEILING_DIRECTORIES="$(CURDIR)/.." \ + git rev-parse -q --verify HEAD 2>/dev/null)"' $(BUILT_INS): git$X $(QUIET_BUILT_IN)$(RM) $@ && \ @@@ -2111,7 -2109,7 +2113,7 @@@ $(SCRIPT_PERL_GEN): % : %.perl GIT-PERL $(QUIET_GEN)$(RM) $@ $@+ && \ sed -e '1{' \ -e ' s|#!.*perl|#!$(PERL_PATH_SQ)|' \ - -e ' rGIT-PERL-HEADER' \ + -e ' r GIT-PERL-HEADER' \ -e ' G' \ -e '}' \ -e 's/@@GIT_VERSION@@/$(GIT_VERSION)/g' \ @@@ -2399,7 -2397,6 +2401,7 @@@ LOCALIZED_C = $(C_OBJ:o=c) $(LIB_H) $(G LOCALIZED_SH = $(SCRIPT_SH) LOCALIZED_SH += git-parse-remote.sh LOCALIZED_SH += git-rebase--interactive.sh +LOCALIZED_SH += git-rebase--preserve-merges.sh LOCALIZED_SH += git-sh-setup.sh LOCALIZED_PERL = $(SCRIPT_PERL) diff --combined fetch-pack.c index 7ccb9c0d45,ba12085c4a..8fb67b0e31 --- a/fetch-pack.c +++ b/fetch-pack.c @@@ -15,12 -15,10 +15,12 @@@ #include "connect.h" #include "transport.h" #include "version.h" - #include "prio-queue.h" #include "sha1-array.h" #include "oidset.h" #include "packfile.h" +#include "object-store.h" +#include "connected.h" + #include "fetch-negotiator.h" static int transfer_unpack_limit = -1; static int fetch_unpack_limit = -1; @@@ -38,13 -36,7 +38,7 @@@ static const char *alternate_shallow_fi /* Remember to update object flag allocation in object.h */ #define COMPLETE (1U << 0) - #define COMMON (1U << 1) - #define COMMON_REF (1U << 2) - #define SEEN (1U << 3) - #define POPPED (1U << 4) - #define ALTERNATE (1U << 5) - - static int marked; + #define ALTERNATE (1U << 1) /* * After sending this many "have"s if we do not get any new ACK , we @@@ -52,8 -44,7 +46,7 @@@ */ #define MAX_IN_VAIN 256 - static struct prio_queue rev_list = { compare_commits_by_commit_date }; - static int non_common_revs, multi_ack, use_sideband; + static int multi_ack, use_sideband; /* Allow specifying sha1 if it is a ref tip. */ #define ALLOW_TIP_SHA1 01 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */ @@@ -95,7 -86,9 +88,9 @@@ static void cache_one_alternate(const c cache->items[cache->nr++] = obj; } - static void for_each_cached_alternate(void (*cb)(struct object *)) + static void for_each_cached_alternate(struct fetch_negotiator *negotiator, + void (*cb)(struct fetch_negotiator *, + struct object *)) { static int initialized; static struct alternate_object_cache cache; @@@ -107,30 -100,17 +102,17 @@@ } for (i = 0; i < cache.nr; i++) - cb(cache.items[i]); - } - - static void rev_list_push(struct commit *commit, int mark) - { - if (!(commit->object.flags & mark)) { - commit->object.flags |= mark; - - if (parse_commit(commit)) - return; - - prio_queue_put(&rev_list, commit); - - if (!(commit->object.flags & COMMON)) - non_common_revs++; - } + cb(negotiator, cache.items[i]); } - static int rev_list_insert_ref(const char *refname, const struct object_id *oid) + static int rev_list_insert_ref(struct fetch_negotiator *negotiator, + const char *refname, + const struct object_id *oid) { struct object *o = deref_tag(parse_object(oid), refname, 0); if (o && o->type == OBJ_COMMIT) - rev_list_push((struct commit *)o, SEEN); + negotiator->add_tip(negotiator, (struct commit *)o); return 0; } @@@ -138,98 -118,7 +120,7 @@@ static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid, int flag, void *cb_data) { - return rev_list_insert_ref(refname, oid); - } - - static int clear_marks(const char *refname, const struct object_id *oid, - int flag, void *cb_data) - { - struct object *o = deref_tag(parse_object(oid), refname, 0); - - if (o && o->type == OBJ_COMMIT) - clear_commit_marks((struct commit *)o, - COMMON | COMMON_REF | SEEN | POPPED); - return 0; - } - - /* - This function marks a rev and its ancestors as common. - In some cases, it is desirable to mark only the ancestors (for example - when only the server does not yet know that they are common). - */ - - static void mark_common(struct commit *commit, - int ancestors_only, int dont_parse) - { - if (commit != NULL && !(commit->object.flags & COMMON)) { - struct object *o = (struct object *)commit; - - if (!ancestors_only) - o->flags |= COMMON; - - if (!(o->flags & SEEN)) - rev_list_push(commit, SEEN); - else { - struct commit_list *parents; - - if (!ancestors_only && !(o->flags & POPPED)) - non_common_revs--; - if (!o->parsed && !dont_parse) - if (parse_commit(commit)) - return; - - for (parents = commit->parents; - parents; - parents = parents->next) - mark_common(parents->item, 0, dont_parse); - } - } - } - - /* - Get the next rev to send, ignoring the common. - */ - - static const struct object_id *get_rev(void) - { - struct commit *commit = NULL; - - while (commit == NULL) { - unsigned int mark; - struct commit_list *parents; - - if (rev_list.nr == 0 || non_common_revs == 0) - return NULL; - - commit = prio_queue_get(&rev_list); - parse_commit(commit); - parents = commit->parents; - - commit->object.flags |= POPPED; - if (!(commit->object.flags & COMMON)) - non_common_revs--; - - if (commit->object.flags & COMMON) { - /* do not send "have", and ignore ancestors */ - commit = NULL; - mark = COMMON | SEEN; - } else if (commit->object.flags & COMMON_REF) - /* send "have", and ignore ancestors */ - mark = COMMON | SEEN; - else - /* send "have", also for its ancestors */ - mark = SEEN; - - while (parents) { - if (!(parents->item->object.flags & SEEN)) - rev_list_push(parents->item, mark); - if (mark & COMMON) - mark_common(parents->item, 1, 0); - parents = parents->next; - } - } - - return &commit->object.oid; + return rev_list_insert_ref(cb_data, refname, oid); } enum ack_type { @@@ -298,9 -187,10 +189,10 @@@ static void send_request(struct fetch_p write_or_die(fd, buf->buf, buf->len); } - static void insert_one_alternate_object(struct object *obj) + static void insert_one_alternate_object(struct fetch_negotiator *negotiator, + struct object *obj) { - rev_list_insert_ref(NULL, &obj->oid); + rev_list_insert_ref(negotiator, NULL, &obj->oid); } #define INITIAL_FLUSH 16 @@@ -323,7 -213,8 +215,8 @@@ static int next_flush(int stateless_rpc return count; } - static int find_common(struct fetch_pack_args *args, + static int find_common(struct fetch_negotiator *negotiator, + struct fetch_pack_args *args, int fd[2], struct object_id *result_oid, struct ref *refs) { @@@ -338,12 -229,9 +231,9 @@@ if (args->stateless_rpc && multi_ack == 1) die(_("--stateless-rpc requires multi_ack_detailed")); - if (marked) - for_each_ref(clear_marks, NULL); - marked = 1; - for_each_ref(rev_list_insert_ref_oid, NULL); - for_each_cached_alternate(insert_one_alternate_object); + for_each_ref(rev_list_insert_ref_oid, negotiator); + for_each_cached_alternate(negotiator, insert_one_alternate_object); fetching = 0; for ( ; refs ; refs = refs->next) { @@@ -398,7 -286,7 +288,7 @@@ return 1; } - if (is_repository_shallow()) + if (is_repository_shallow(the_repository)) write_shallow_commits(&req_buf, 1, NULL); if (args->depth > 0) packet_buf_write(&req_buf, "deepen %d", args->depth); @@@ -429,7 -317,7 +319,7 @@@ if (skip_prefix(line, "shallow ", &arg)) { if (get_oid_hex(arg, &oid)) die(_("invalid shallow line: %s"), line); - register_shallow(&oid); + register_shallow(the_repository, &oid); continue; } if (skip_prefix(line, "unshallow ", &arg)) { @@@ -461,7 -349,7 +351,7 @@@ retval = -1; if (args->no_dependents) goto done; - while ((oid = get_rev())) { + while ((oid = negotiator->next(negotiator))) { packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid)); print_verbose(args, "have %s", oid_to_hex(oid)); in_vain++; @@@ -498,11 -386,13 +388,13 @@@ case ACK_continue: { struct commit *commit = lookup_commit(result_oid); + int was_common; if (!commit) die(_("invalid commit %s"), oid_to_hex(result_oid)); + was_common = negotiator->ack(negotiator, commit); if (args->stateless_rpc && ack == ACK_common - && !(commit->object.flags & COMMON)) { + && !was_common) { /* We need to replay the have for this object * on the next RPC request so the peer knows * it is in common with us. @@@ -519,13 -409,10 +411,10 @@@ } else if (!args->stateless_rpc || ack != ACK_common) in_vain = 0; - mark_common(commit, 0, 1); retval = 0; got_continue = 1; - if (ack == ACK_ready) { - clear_prio_queue(&rev_list); + if (ack == ACK_ready) got_ready = 1; - } break; } } @@@ -535,6 -422,8 +424,8 @@@ print_verbose(args, _("giving up")); break; /* give up */ } + if (got_ready) + break; } } done: @@@ -659,11 -548,11 +550,11 @@@ static void filter_refs(struct fetch_pa } i++; } - } - if (!keep && args->fetch_all && - (!args->deepen || !starts_with(ref->name, "refs/tags/"))) - keep = 1; + if (!keep && args->fetch_all && + (!args->deepen || !starts_with(ref->name, "refs/tags/"))) + keep = 1; + } if (keep) { *newtail = ref; @@@ -709,7 -598,8 +600,8 @@@ *refs = newlist; } - static void mark_alternate_complete(struct object *obj) + static void mark_alternate_complete(struct fetch_negotiator *unused, + struct object *obj) { mark_complete(&obj->oid); } @@@ -736,12 -626,21 +628,21 @@@ static int add_loose_objects_to_set(con return 0; } - static int everything_local(struct fetch_pack_args *args, - struct ref **refs, - struct ref **sought, int nr_sought) + /* + * Mark recent commits available locally and reachable from a local ref as + * COMPLETE. If args->no_dependents is false, also mark COMPLETE remote refs as + * COMMON_REF (otherwise, we are not planning to participate in negotiation, and + * thus do not need COMMON_REF marks). + * + * The cutoff time for recency is determined by this heuristic: it is the + * earliest commit time of the objects in refs that are commits and that we know + * the commit time of. + */ + static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator, + struct fetch_pack_args *args, + struct ref **refs) { struct ref *ref; - int retval; int old_save_commit_buffer = save_commit_buffer; timestamp_t cutoff = 0; struct oidset loose_oid_set = OIDSET_INIT; @@@ -789,7 -688,7 +690,7 @@@ if (!args->no_dependents) { if (!args->deepen) { for_each_ref(mark_complete_oid, NULL); - for_each_cached_alternate(mark_alternate_complete); + for_each_cached_alternate(NULL, mark_alternate_complete); commit_list_sort_by_date(&complete); if (cutoff) mark_recent_complete_commits(args, cutoff); @@@ -806,15 -705,23 +707,23 @@@ if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE)) continue; - if (!(o->flags & SEEN)) { - rev_list_push((struct commit *)o, COMMON_REF | SEEN); - - mark_common((struct commit *)o, 1, 1); - } + negotiator->known_common(negotiator, + (struct commit *)o); } } - filter_refs(args, refs, sought, nr_sought); + save_commit_buffer = old_save_commit_buffer; + } + + /* + * Returns 1 if every object pointed to by the given remote refs is available + * locally and reachable from a local ref, and 0 otherwise. + */ + static int everything_local(struct fetch_pack_args *args, + struct ref **refs) + { + struct ref *ref; + int retval; for (retval = 1, ref = *refs; ref ; ref = ref->next) { const struct object_id *remote = &ref->old_oid; @@@ -831,8 -738,6 +740,6 @@@ ref->name); } - save_commit_buffer = old_save_commit_buffer; - return retval; } @@@ -983,11 -888,13 +890,13 @@@ static struct ref *do_fetch_pack(struc struct object_id oid; const char *agent_feature; int agent_len; + struct fetch_negotiator negotiator; + fetch_negotiator_init(&negotiator); sort_ref_list(&ref, ref_compare_name); QSORT(sought, nr_sought, cmp_ref_by_name); - if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow")) + if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow")) die(_("Server does not support shallow clients")); if (args->depth > 0 || args->deepen_since || args->deepen_not) args->deepen = 1; @@@ -1055,11 -962,13 +964,13 @@@ if (!server_supports("deepen-relative") && args->deepen_relative) die(_("Server does not support --deepen")); - if (everything_local(args, &ref, sought, nr_sought)) { + mark_complete_and_common_ref(&negotiator, args, &ref); + filter_refs(args, &ref, sought, nr_sought); + if (everything_local(args, &ref)) { packet_flush(fd[1]); goto all_done; } - if (find_common(args, fd, &oid, ref) < 0) + if (find_common(&negotiator, args, fd, &oid, ref) < 0) if (!args->keep_pack) /* When cloning, it is not unusual to have * no common commit. @@@ -1079,13 -988,14 +990,14 @@@ die(_("git fetch-pack: fetch failed.")); all_done: + negotiator.release(&negotiator); return ref; } static void add_shallow_requests(struct strbuf *req_buf, const struct fetch_pack_args *args) { - if (is_repository_shallow()) + if (is_repository_shallow(the_repository)) write_shallow_commits(req_buf, 1, NULL); if (args->depth > 0) packet_buf_write(req_buf, "deepen %d", args->depth); @@@ -1104,10 -1014,9 +1016,10 @@@ static void add_wants(const struct ref *wants, struct strbuf *req_buf) { + int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0); + for ( ; wants ; wants = wants->next) { const struct object_id *remote = &wants->old_oid; - const char *remote_hex; struct object *o; /* @@@ -1125,10 -1034,8 +1037,10 @@@ continue; } - remote_hex = oid_to_hex(remote); - packet_buf_write(req_buf, "want %s\n", remote_hex); + if (!use_ref_in_want || wants->exact_oid) + packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote)); + else + packet_buf_write(req_buf, "want-ref %s\n", wants->name); } } @@@ -1143,13 -1050,15 +1055,15 @@@ static void add_common(struct strbuf *r } } - static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain) + static int add_haves(struct fetch_negotiator *negotiator, + struct strbuf *req_buf, + int *haves_to_send, int *in_vain) { int ret = 0; int haves_added = 0; const struct object_id *oid; - while ((oid = get_rev())) { + while ((oid = negotiator->next(negotiator))) { packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid)); if (++haves_added >= *haves_to_send) break; @@@ -1168,7 -1077,8 +1082,8 @@@ return ret; } - static int send_fetch_request(int fd_out, const struct fetch_pack_args *args, + static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, + const struct fetch_pack_args *args, const struct ref *wants, struct oidset *common, int *haves_to_send, int *in_vain) { @@@ -1200,7 -1110,7 +1115,7 @@@ /* Add shallow-info and deepen request */ if (server_supports_feature("fetch", "shallow", 0)) add_shallow_requests(&req_buf, args); - else if (is_repository_shallow() || args->deepen) + else if (is_repository_shallow(the_repository) || args->deepen) die(_("Server does not support shallow requests")); /* Add filter */ @@@ -1224,7 -1134,7 +1139,7 @@@ add_common(&req_buf, common); /* Add initial haves */ - ret = add_haves(&req_buf, haves_to_send, in_vain); + ret = add_haves(negotiator, &req_buf, haves_to_send, in_vain); } /* Send request */ @@@ -1261,7 -1171,9 +1176,9 @@@ static int process_section_header(struc return ret; } - static int process_acks(struct packet_reader *reader, struct oidset *common) + static int process_acks(struct fetch_negotiator *negotiator, + struct packet_reader *reader, + struct oidset *common) { /* received */ int received_ready = 0; @@@ -1280,13 -1192,12 +1197,12 @@@ struct commit *commit; oidset_insert(common, &oid); commit = lookup_commit(&oid); - mark_common(commit, 0, 1); + negotiator->ack(negotiator, commit); } continue; } if (!strcmp(reader->line, "ready")) { - clear_prio_queue(&rev_list); received_ready = 1; continue; } @@@ -1313,7 -1224,7 +1229,7 @@@ static void receive_shallow_info(struc if (skip_prefix(reader->line, "shallow ", &arg)) { if (get_oid_hex(arg, &oid)) die(_("invalid shallow line: %s"), reader->line); - register_shallow(&oid); + register_shallow(the_repository, &oid); continue; } if (skip_prefix(reader->line, "unshallow ", &arg)) { @@@ -1339,32 -1250,6 +1255,32 @@@ args->deepen = 1; } +static void receive_wanted_refs(struct packet_reader *reader, struct ref *refs) +{ + process_section_header(reader, "wanted-refs", 0); + while (packet_reader_read(reader) == PACKET_READ_NORMAL) { + struct object_id oid; + const char *end; + struct ref *r = NULL; + + if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ') + die("expected wanted-ref, got '%s'", reader->line); + + for (r = refs; r; r = r->next) { + if (!strcmp(end, r->name)) { + oidcpy(&r->old_oid, &oid); + break; + } + } + + if (!r) + die("unexpected wanted-ref: '%s'", reader->line); + } + + if (reader->status != PACKET_READ_DELIM) + die("error processing wanted refs: %d", reader->status); +} + enum fetch_state { FETCH_CHECK_LOCAL = 0, FETCH_SEND_REQUEST, @@@ -1385,6 -1270,8 +1301,8 @@@ static struct ref *do_fetch_pack_v2(str struct packet_reader reader; int in_vain = 0; int haves_to_send = INITIAL_FLUSH; + struct fetch_negotiator negotiator; + fetch_negotiator_init(&negotiator); packet_reader_init(&reader, fd[0], NULL, 0, PACKET_READ_CHOMP_NEWLINE); @@@ -1400,21 -1287,21 +1318,21 @@@ if (args->depth > 0 || args->deepen_since || args->deepen_not) args->deepen = 1; - if (marked) - for_each_ref(clear_marks, NULL); - marked = 1; - - for_each_ref(rev_list_insert_ref_oid, NULL); - for_each_cached_alternate(insert_one_alternate_object); - /* Filter 'ref' by 'sought' and those that aren't local */ - if (everything_local(args, &ref, sought, nr_sought)) + mark_complete_and_common_ref(&negotiator, args, &ref); + filter_refs(args, &ref, sought, nr_sought); + if (everything_local(args, &ref)) state = FETCH_DONE; else state = FETCH_SEND_REQUEST; + + for_each_ref(rev_list_insert_ref_oid, &negotiator); + for_each_cached_alternate(&negotiator, + insert_one_alternate_object); break; case FETCH_SEND_REQUEST: - if (send_fetch_request(fd[1], args, ref, &common, + if (send_fetch_request(&negotiator, fd[1], args, ref, + &common, &haves_to_send, &in_vain)) state = FETCH_GET_PACK; else @@@ -1422,7 -1309,7 +1340,7 @@@ break; case FETCH_PROCESS_ACKS: /* Process ACKs/NAKs */ - switch (process_acks(&reader, &common)) { + switch (process_acks(&negotiator, &reader, &common)) { case 2: state = FETCH_GET_PACK; break; @@@ -1439,9 -1326,6 +1357,9 @@@ if (process_section_header(&reader, "shallow-info", 1)) receive_shallow_info(args, &reader); + if (process_section_header(&reader, "wanted-refs", 1)) + receive_wanted_refs(&reader, ref); + /* get the pack */ process_section_header(&reader, "packfile", 0); if (get_pack(args, fd, pack_lockfile)) @@@ -1454,6 -1338,7 +1372,7 @@@ } } + negotiator.release(&negotiator); oidset_clear(&common); return ref; } @@@ -1504,17 -1389,16 +1423,17 @@@ static int remove_duplicates_in_refs(st } static void update_shallow(struct fetch_pack_args *args, - struct ref **sought, int nr_sought, + struct ref *refs, struct shallow_info *si) { struct oid_array ref = OID_ARRAY_INIT; int *status; int i; + struct ref *r; if (args->deepen && alternate_shallow_file) { if (*alternate_shallow_file == '\0') { /* --unshallow */ - unlink_or_warn(git_path_shallow()); + unlink_or_warn(git_path_shallow(the_repository)); rollback_lock_file(&shallow_lock); } else commit_lock_file(&shallow_lock); @@@ -1552,8 -1436,8 +1471,8 @@@ remove_nonexistent_theirs_shallow(si); if (!si->nr_ours && !si->nr_theirs) return; - for (i = 0; i < nr_sought; i++) - oid_array_append(&ref, &sought[i]->old_oid); + for (r = refs; r; r = r->next) + oid_array_append(&ref, &r->old_oid); si->ref = &ref; if (args->update_shallow) { @@@ -1587,29 -1471,17 +1506,29 @@@ * remote is also shallow, check what ref is safe to update * without updating .git/shallow */ - status = xcalloc(nr_sought, sizeof(*status)); + status = xcalloc(ref.nr, sizeof(*status)); assign_shallow_commits_to_refs(si, NULL, status); if (si->nr_ours || si->nr_theirs) { - for (i = 0; i < nr_sought; i++) + for (r = refs, i = 0; r; r = r->next, i++) if (status[i]) - sought[i]->status = REF_STATUS_REJECT_SHALLOW; + r->status = REF_STATUS_REJECT_SHALLOW; } free(status); oid_array_clear(&ref); } +static int iterate_ref_map(void *cb_data, struct object_id *oid) +{ + struct ref **rm = cb_data; + struct ref *ref = *rm; + + if (!ref) + return -1; /* end of the list */ + *rm = ref->next; + oidcpy(oid, &ref->old_oid); + return 0; +} + struct ref *fetch_pack(struct fetch_pack_args *args, int fd[], struct child_process *conn, const struct ref *ref, @@@ -1638,25 -1510,7 +1557,25 @@@ ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, &si, pack_lockfile); reprepare_packed_git(the_repository); - update_shallow(args, sought, nr_sought, &si); + + if (!args->cloning && args->deepen) { + struct check_connected_options opt = CHECK_CONNECTED_INIT; + struct ref *iterator = ref_cpy; + opt.shallow_file = alternate_shallow_file; + if (args->deepen) + opt.is_deepening_fetch = 1; + if (check_connected(iterate_ref_map, &iterator, &opt)) { + error(_("remote did not send all necessary objects")); + free_refs(ref_cpy); + ref_cpy = NULL; + rollback_lock_file(&shallow_lock); + goto cleanup; + } + args->connectivity_checked = 1; + } + + update_shallow(args, ref_cpy, &si); +cleanup: clear_shallow_info(&si); return ref_cpy; } diff --combined object.h index 3afd123d73,7db4941d6d..7227b97166 --- a/object.h +++ b/object.h @@@ -1,32 -1,6 +1,32 @@@ #ifndef OBJECT_H #define OBJECT_H +struct parsed_object_pool { + struct object **obj_hash; + int nr_objs, obj_hash_size; + + /* TODO: migrate alloc_states to mem-pool? */ + struct alloc_state *blob_state; + struct alloc_state *tree_state; + struct alloc_state *commit_state; + struct alloc_state *tag_state; + struct alloc_state *object_state; + unsigned commit_count; + + /* parent substitutions from .git/info/grafts and .git/shallow */ + struct commit_graft **grafts; + int grafts_alloc, grafts_nr; + + int is_shallow; + struct stat_validity *shallow_stat; + char *alternate_shallow_file; + + int commit_graft_prepared; +}; + +struct parsed_object_pool *parsed_object_pool_new(void); +void parsed_object_pool_clear(struct parsed_object_pool *o); + struct object_list { struct object *item; struct object_list *next; @@@ -53,8 -27,9 +53,9 @@@ struct object_array /* * object flag allocation: - * revision.h: 0---------10 26 + * revision.h: 0---------10 2526 - * fetch-pack.c: 0----5 + * fetch-pack.c: 01 + * negotiator/default.c: 2--5 * walker.c: 0-2 * upload-pack.c: 4 11----------------19 * builtin/blame.c: 12-13 @@@ -68,7 -43,6 +69,7 @@@ * builtin/index-pack.c: 2021 * builtin/pack-objects.c: 20 * builtin/reflog.c: 10--12 + * builtin/show-branch.c: 0-------------------------------------------26 * builtin/unpack-objects.c: 2021 */ #define FLAG_BITS 27 @@@ -111,7 -85,7 +112,7 @@@ extern struct object *get_indexed_objec */ struct object *lookup_object(const unsigned char *sha1); -extern void *create_object(const unsigned char *sha1, void *obj); +extern void *create_object(struct repository *r, const unsigned char *sha1, void *obj); void *object_as_type(struct object *obj, enum object_type type, int quiet); diff --combined t/t5500-fetch-pack.sh index 0f0fefad1e,e0cdc295d8..fa03f56a20 --- a/t/t5500-fetch-pack.sh +++ b/t/t5500-fetch-pack.sh @@@ -259,7 -259,7 +259,7 @@@ test_expect_success 'clone shallow obje test_expect_success 'pull in shallow repo with missing merge base' ' ( cd shallow && - git fetch --depth 4 .. A + git fetch --depth 4 .. A && test_must_fail git merge --allow-unrelated-histories FETCH_HEAD ) ' @@@ -518,54 -518,6 +518,54 @@@ test_expect_success 'test --all, --dept ) >out-adt 2>error-adt ' +test_expect_success 'test --all with tag to non-tip' ' + git commit --allow-empty -m non-tip && + git commit --allow-empty -m tip && + git tag -m "annotated" non-tip HEAD^ && + ( + cd client && + git fetch-pack --all .. + ) +' + +test_expect_success 'test --all wrt tag to non-commits' ' + # create tag-to-{blob,tree,commit,tag}, making sure all tagged objects + # are reachable only via created tag references. + blob=$(echo "hello blob" | git hash-object -t blob -w --stdin) && + git tag -a -m "tag -> blob" tag-to-blob $blob && + + tree=$(printf "100644 blob $blob\tfile" | git mktree) && + git tag -a -m "tag -> tree" tag-to-tree $tree && + + tree2=$(printf "100644 blob $blob\tfile2" | git mktree) && + commit=$(git commit-tree -m "hello commit" $tree) && + git tag -a -m "tag -> commit" tag-to-commit $commit && + + blob2=$(echo "hello blob2" | git hash-object -t blob -w --stdin) && + tag=$(git mktag <<-EOF + object $blob2 + type blob + tag tag-to-blob2 + tagger author A U Thor 0 +0000 + + hello tag + EOF + ) && + git tag -a -m "tag -> tag" tag-to-tag $tag && + + # `fetch-pack --all` should succeed fetching all those objects. + mkdir fetchall && + ( + cd fetchall && + git init && + git fetch-pack --all .. && + git cat-file blob $blob >/dev/null && + git cat-file tree $tree >/dev/null && + git cat-file commit $commit >/dev/null && + git cat-file tag $tag >/dev/null + ) +' + test_expect_success 'shallow fetch with tags does not break the repository' ' mkdir repo1 && ( @@@ -759,17 -711,6 +759,17 @@@ test_expect_success 'fetch shallow sinc test_cmp expected actual ' +test_expect_success 'clone shallow since selects no commits' ' + test_create_repo shallow-since-the-future && + ( + cd shallow-since-the-future && + GIT_COMMITTER_DATE="100000000 +0700" git commit --allow-empty -m one && + GIT_COMMITTER_DATE="200000000 +0700" git commit --allow-empty -m two && + GIT_COMMITTER_DATE="300000000 +0700" git commit --allow-empty -m three && + test_must_fail git clone --shallow-since "900000000 +0700" "file://$(pwd)/." ../shallow111 + ) +' + test_expect_success 'shallow clone exclude tag two' ' test_create_repo shallow-exclude && ( @@@ -814,6 -755,39 +814,39 @@@ test_expect_success 'fetching deepen' ) ' + test_expect_success 'use ref advertisement to prune "have" lines sent' ' + rm -rf server client && + git init server && + test_commit -C server both_have_1 && + git -C server tag -d both_have_1 && + test_commit -C server both_have_2 && + + git clone server client && + test_commit -C server server_has && + test_commit -C client client_has && + + # In both protocol v0 and v2, ensure that the parent of both_have_2 is + # not sent as a "have" line. The client should know that the server has + # both_have_2, so it only needs to inform the server that it has + # both_have_2, and the server can infer the rest. + + rm -f trace && + cp -r client clientv0 && + GIT_TRACE_PACKET="$(pwd)/trace" git -C clientv0 \ + fetch origin server_has both_have_2 && + grep "have $(git -C client rev-parse client_has)" trace && + grep "have $(git -C client rev-parse both_have_2)" trace && + ! grep "have $(git -C client rev-parse both_have_2^)" trace && + + rm -f trace && + cp -r client clientv2 && + GIT_TRACE_PACKET="$(pwd)/trace" git -C clientv2 -c protocol.version=2 \ + fetch origin server_has both_have_2 && + grep "have $(git -C client rev-parse client_has)" trace && + grep "have $(git -C client rev-parse both_have_2)" trace && + ! grep "have $(git -C client rev-parse both_have_2^)" trace + ' + test_expect_success 'filtering by size' ' rm -rf server client && test_create_repo server &&