From: Junio C Hamano Date: Wed, 23 Oct 2013 20:32:17 +0000 (-0700) Subject: Merge branch 'nd/fetch-into-shallow' into maint X-Git-Tag: v1.8.4.2~10 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/6ba0d9551a2963a6fed83b90b55a373d79ef46d0?hp=-c Merge branch 'nd/fetch-into-shallow' into maint When there is no sufficient overlap between old and new history during a "git fetch" into a shallow repository, objects that the sending side knows the receiving end has were unnecessarily sent. * nd/fetch-into-shallow: Add testcase for needless objects during a shallow fetch list-objects: mark more commits as edges in mark_edges_uninteresting list-objects: reduce one argument in mark_edges_uninteresting upload-pack: delegate rev walking in shallow fetch to pack-objects shallow: add setup_temporary_shallow() shallow: only add shallow graft points to new shallow file move setup_alternate_shallow and write_shallow_commits to shallow.c --- 6ba0d9551a2963a6fed83b90b55a373d79ef46d0 diff --combined commit.h index f9504f70cc,c4d324c955..c308674120 --- a/commit.h +++ b/commit.h @@@ -62,9 -62,6 +62,9 @@@ struct commit_list *commit_list_insert_ struct commit_list **list); void commit_list_sort_by_date(struct commit_list **list); +/* Shallow copy of the input list */ +struct commit_list *copy_commit_list(struct commit_list *list); + void free_commit_list(struct commit_list *list); /* Commit formats */ @@@ -201,6 -198,10 +201,10 @@@ extern struct commit_list *get_shallow_ int depth, int shallow_flag, int not_shallow_flag); extern void check_shallow_file_for_update(void); extern void set_alternate_shallow_file(const char *path); + extern int write_shallow_commits(struct strbuf *out, int use_pack_protocol); + extern void setup_alternate_shallow(struct lock_file *shallow_lock, + const char **alternate_shallow_file); + extern char *setup_temporary_shallow(void); int is_descendant_of(struct commit *, struct commit_list *); int in_merge_bases(struct commit *, struct commit *); diff --combined fetch-pack.c index f5d99c1181,28195ed78b..aff4f5abab --- a/fetch-pack.c +++ b/fetch-pack.c @@@ -184,36 -184,6 +184,6 @@@ static void consume_shallow_list(struc } } - struct write_shallow_data { - struct strbuf *out; - int use_pack_protocol; - int count; - }; - - static int write_one_shallow(const struct commit_graft *graft, void *cb_data) - { - struct write_shallow_data *data = cb_data; - const char *hex = sha1_to_hex(graft->sha1); - data->count++; - if (data->use_pack_protocol) - packet_buf_write(data->out, "shallow %s", hex); - else { - strbuf_addstr(data->out, hex); - strbuf_addch(data->out, '\n'); - } - return 0; - } - - static int write_shallow_commits(struct strbuf *out, int use_pack_protocol) - { - struct write_shallow_data data; - data.out = out; - data.use_pack_protocol = use_pack_protocol; - data.count = 0; - for_each_commit_graft(write_one_shallow, &data); - return data.count; - } - static enum ack_type get_ack(int fd, unsigned char *result_sha1) { int len; @@@ -795,27 -765,6 +765,6 @@@ static int cmp_ref_by_name(const void * return strcmp(a->name, b->name); } - static void setup_alternate_shallow(void) - { - struct strbuf sb = STRBUF_INIT; - int fd; - - check_shallow_file_for_update(); - fd = hold_lock_file_for_update(&shallow_lock, git_path("shallow"), - LOCK_DIE_ON_ERROR); - if (write_shallow_commits(&sb, 0)) { - if (write_in_full(fd, sb.buf, sb.len) != sb.len) - die_errno("failed to write to %s", shallow_lock.filename); - alternate_shallow_file = shallow_lock.filename; - } else - /* - * is_repository_shallow() sees empty string as "no - * shallow file". - */ - alternate_shallow_file = ""; - strbuf_release(&sb); - } - static struct ref *do_fetch_pack(struct fetch_pack_args *args, int fd[2], const struct ref *orig_ref, @@@ -896,9 -845,7 +845,9 @@@ if (args->stateless_rpc) packet_flush(fd[1]); if (args->depth > 0) - setup_alternate_shallow(); + setup_alternate_shallow(&shallow_lock, &alternate_shallow_file); + else + alternate_shallow_file = NULL; if (get_pack(args, fd, pack_lockfile)) die("git fetch-pack: fetch failed."); @@@ -989,7 -936,7 +938,7 @@@ struct ref *fetch_pack(struct fetch_pac } ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile); - if (alternate_shallow_file) { + if (args->depth > 0 && alternate_shallow_file) { if (*alternate_shallow_file == '\0') { /* --unshallow */ unlink_or_warn(git_path("shallow")); rollback_lock_file(&shallow_lock); diff --combined t/t5500-fetch-pack.sh index a80584ea0e,7a22f557b9..d87ddf73b7 --- a/t/t5500-fetch-pack.sh +++ b/t/t5500-fetch-pack.sh @@@ -393,6 -393,17 +393,17 @@@ test_expect_success 'fetch in shallow r git fsck --no-dangling ) ' + test_expect_success 'fetch creating new shallow root' ' + ( + git clone "file://$(pwd)/." shallow10 && + git commit --allow-empty -m empty && + cd shallow10 && + git fetch --depth=1 --progress 2>actual && + # This should fetch only the empty commit, no tree or + # blob objects + grep "remote: Total 1" actual + ) + ' test_expect_success 'setup tests for the --stdin parameter' ' for head in C D E F @@@ -505,20 -516,4 +516,20 @@@ test_expect_success 'test --all, --dept ) >out-adt 2>error-adt ' +test_expect_success 'shallow fetch with tags does not break the repository' ' + mkdir repo1 && + ( + cd repo1 && + git init && + test_commit 1 && + test_commit 2 && + test_commit 3 && + mkdir repo2 && + cd repo2 && + git init && + git fetch --depth=2 ../.git master:branch && + git fsck + ) +' + test_done diff --combined upload-pack.c index 04a8707bb5,d5a003ad1f..8327dc0b7c --- a/upload-pack.c +++ b/upload-pack.c @@@ -40,7 -40,6 +40,7 @@@ static struct object_array have_obj static struct object_array want_obj; static struct object_array extra_edge_obj; static unsigned int timeout; +static int keepalive = 5; /* 0 for no sideband, * otherwise maximum packet size (up to 65520 bytes). */ @@@ -69,87 -68,28 +69,28 @@@ static ssize_t send_client_data(int fd return sz; } - static FILE *pack_pipe = NULL; - static void show_commit(struct commit *commit, void *data) - { - if (commit->object.flags & BOUNDARY) - fputc('-', pack_pipe); - if (fputs(sha1_to_hex(commit->object.sha1), pack_pipe) < 0) - die("broken output pipe"); - fputc('\n', pack_pipe); - fflush(pack_pipe); - free(commit->buffer); - commit->buffer = NULL; - } - - static void show_object(struct object *obj, - const struct name_path *path, const char *component, - void *cb_data) - { - show_object_with_name(pack_pipe, obj, path, component); - } - - static void show_edge(struct commit *commit) - { - fprintf(pack_pipe, "-%s\n", sha1_to_hex(commit->object.sha1)); - } - - static int do_rev_list(int in, int out, void *user_data) - { - int i; - struct rev_info revs; - - pack_pipe = xfdopen(out, "w"); - init_revisions(&revs, NULL); - revs.tag_objects = 1; - revs.tree_objects = 1; - revs.blob_objects = 1; - if (use_thin_pack) - revs.edge_hint = 1; - - for (i = 0; i < want_obj.nr; i++) { - struct object *o = want_obj.objects[i].item; - /* why??? */ - o->flags &= ~UNINTERESTING; - add_pending_object(&revs, o, NULL); - } - for (i = 0; i < have_obj.nr; i++) { - struct object *o = have_obj.objects[i].item; - o->flags |= UNINTERESTING; - add_pending_object(&revs, o, NULL); - } - setup_revisions(0, NULL, &revs, NULL); - if (prepare_revision_walk(&revs)) - die("revision walk setup failed"); - mark_edges_uninteresting(revs.commits, &revs, show_edge); - if (use_thin_pack) - for (i = 0; i < extra_edge_obj.nr; i++) - fprintf(pack_pipe, "-%s\n", sha1_to_hex( - extra_edge_obj.objects[i].item->sha1)); - traverse_commit_list(&revs, show_commit, show_object, NULL); - fflush(pack_pipe); - fclose(pack_pipe); - return 0; - } - static void create_pack_file(void) { - struct async rev_list; struct child_process pack_objects; char data[8193], progress[128]; char abort_msg[] = "aborting due to possible repository " "corruption on the remote side."; int buffered = -1; ssize_t sz; - const char *argv[10]; - int arg = 0; + const char *argv[12]; + int i, arg = 0; + FILE *pipe_fd; + char *shallow_file = NULL; - argv[arg++] = "pack-objects"; - if (!shallow_nr) { - argv[arg++] = "--revs"; - if (use_thin_pack) - argv[arg++] = "--thin"; + if (shallow_nr) { + shallow_file = setup_temporary_shallow(); + argv[arg++] = "--shallow-file"; + argv[arg++] = shallow_file; } + argv[arg++] = "pack-objects"; + argv[arg++] = "--revs"; + if (use_thin_pack) + argv[arg++] = "--thin"; argv[arg++] = "--stdout"; if (!no_progress) @@@ -170,29 -110,21 +111,21 @@@ if (start_command(&pack_objects)) die("git upload-pack: unable to fork git-pack-objects"); - if (shallow_nr) { - memset(&rev_list, 0, sizeof(rev_list)); - rev_list.proc = do_rev_list; - rev_list.out = pack_objects.in; - if (start_async(&rev_list)) - die("git upload-pack: unable to fork git-rev-list"); - } - else { - FILE *pipe_fd = xfdopen(pack_objects.in, "w"); - int i; - - for (i = 0; i < want_obj.nr; i++) - fprintf(pipe_fd, "%s\n", - sha1_to_hex(want_obj.objects[i].item->sha1)); - fprintf(pipe_fd, "--not\n"); - for (i = 0; i < have_obj.nr; i++) - fprintf(pipe_fd, "%s\n", - sha1_to_hex(have_obj.objects[i].item->sha1)); - fprintf(pipe_fd, "\n"); - fflush(pipe_fd); - fclose(pipe_fd); - } + pipe_fd = xfdopen(pack_objects.in, "w"); + for (i = 0; i < want_obj.nr; i++) + fprintf(pipe_fd, "%s\n", + sha1_to_hex(want_obj.objects[i].item->sha1)); + fprintf(pipe_fd, "--not\n"); + for (i = 0; i < have_obj.nr; i++) + fprintf(pipe_fd, "%s\n", + sha1_to_hex(have_obj.objects[i].item->sha1)); + for (i = 0; i < extra_edge_obj.nr; i++) + fprintf(pipe_fd, "%s\n", + sha1_to_hex(extra_edge_obj.objects[i].item->sha1)); + fprintf(pipe_fd, "\n"); + fflush(pipe_fd); + fclose(pipe_fd); /* We read from pack_objects.err to capture stderr output for * progress bar, and pack_objects.out to capture the pack data. @@@ -201,7 -133,6 +134,7 @@@ while (1) { struct pollfd pfd[2]; int pe, pu, pollsize; + int ret; reset_timeout(); @@@ -224,8 -155,7 +157,8 @@@ if (!pollsize) break; - if (poll(pfd, pollsize, -1) < 0) { + ret = poll(pfd, pollsize, 1000 * keepalive); + if (ret < 0) { if (errno != EINTR) { error("poll failed, resuming: %s", strerror(errno)); @@@ -287,29 -217,17 +220,32 @@@ if (sz < 0) goto fail; } + + /* + * We hit the keepalive timeout without saying anything; send + * an empty message on the data sideband just to let the other + * side know we're still working on it, but don't have any data + * yet. + * + * If we don't have a sideband channel, there's no room in the + * protocol to say anything, so those clients are just out of + * luck. + */ + if (!ret && use_sideband) { + static const char buf[] = "0005\1"; + write_or_die(1, buf, 5); + } } if (finish_command(&pack_objects)) { error("git upload-pack: git-pack-objects died with error."); goto fail; } - if (shallow_nr && finish_async(&rev_list)) - goto fail; /* error was already reported */ + if (shallow_file) { + if (*shallow_file) + unlink(shallow_file); + free(shallow_file); + } /* flush the data */ if (0 <= buffered) { @@@ -803,11 -721,6 +739,11 @@@ static int upload_pack_config(const cha { if (!strcmp("uploadpack.allowtipsha1inwant", var)) allow_tip_sha1_in_want = git_config_bool(var, value); + else if (!strcmp("uploadpack.keepalive", var)) { + keepalive = git_config_int(var, value); + if (!keepalive) + keepalive = -1; + } return parse_hide_refs_config(var, value, "uploadpack"); }