From: Junio C Hamano Date: Fri, 20 Sep 2013 19:25:32 +0000 (-0700) Subject: Merge branch 'nd/fetch-into-shallow' X-Git-Tag: v1.8.5-rc0~104 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/238504b014230d0bc244fb0de84990863fcddd59?ds=inline;hp=-c Merge branch 'nd/fetch-into-shallow' When there is no sufficient overlap between old and new history during a fetch into a shallow repository, we unnecessarily sent objects the sending side knows the receiving end has. * nd/fetch-into-shallow: Add testcase for needless objects during a shallow fetch list-objects: mark more commits as edges in mark_edges_uninteresting list-objects: reduce one argument in mark_edges_uninteresting upload-pack: delegate rev walking in shallow fetch to pack-objects shallow: add setup_temporary_shallow() shallow: only add shallow graft points to new shallow file move setup_alternate_shallow and write_shallow_commits to shallow.c --- 238504b014230d0bc244fb0de84990863fcddd59 diff --combined builtin/pack-objects.c index 4eb0521c81,dd117b379a..e86cd5729f --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@@ -1809,7 -1809,7 +1809,7 @@@ static void find_deltas(struct object_e static void try_to_free_from_threads(size_t size) { read_lock(); - release_pack_memory(size, -1); + release_pack_memory(size); read_unlock(); } @@@ -2378,7 -2378,7 +2378,7 @@@ static void get_object_list(int ac, con if (prepare_revision_walk(&revs)) die("revision walk setup failed"); - mark_edges_uninteresting(revs.commits, &revs, show_edge); + mark_edges_uninteresting(&revs, show_edge); traverse_commit_list(&revs, show_commit, show_object, NULL); if (keep_unreachable) diff --combined commit.h index 90a5a3c361,c4d324c955..bd841f4d0c --- a/commit.h +++ b/commit.h @@@ -62,9 -62,6 +62,9 @@@ struct commit_list *commit_list_insert_ struct commit_list **list); void commit_list_sort_by_date(struct commit_list **list); +/* Shallow copy of the input list */ +struct commit_list *copy_commit_list(struct commit_list *list); + void free_commit_list(struct commit_list *list); /* Commit formats */ @@@ -201,6 -198,10 +201,10 @@@ extern struct commit_list *get_shallow_ int depth, int shallow_flag, int not_shallow_flag); extern void check_shallow_file_for_update(void); extern void set_alternate_shallow_file(const char *path); + extern int write_shallow_commits(struct strbuf *out, int use_pack_protocol); + extern void setup_alternate_shallow(struct lock_file *shallow_lock, + const char **alternate_shallow_file); + extern char *setup_temporary_shallow(void); int is_descendant_of(struct commit *, struct commit_list *); int in_merge_bases(struct commit *, struct commit *); @@@ -208,7 -209,7 +212,7 @@@ int in_merge_bases_many(struct commit * extern int interactive_add(int argc, const char **argv, const char *prefix, int patch); extern int run_add_interactive(const char *revision, const char *patch_mode, - const char **pathspec); + const struct pathspec *pathspec); static inline int single_parent(struct commit *commit) { diff --combined fetch-pack.c index 094267fd80,28195ed78b..13b5b43bfa --- a/fetch-pack.c +++ b/fetch-pack.c @@@ -9,7 -9,6 +9,7 @@@ #include "fetch-pack.h" #include "remote.h" #include "run-command.h" +#include "connect.h" #include "transport.h" #include "version.h" #include "prio-queue.h" @@@ -185,36 -184,6 +185,6 @@@ static void consume_shallow_list(struc } } - struct write_shallow_data { - struct strbuf *out; - int use_pack_protocol; - int count; - }; - - static int write_one_shallow(const struct commit_graft *graft, void *cb_data) - { - struct write_shallow_data *data = cb_data; - const char *hex = sha1_to_hex(graft->sha1); - data->count++; - if (data->use_pack_protocol) - packet_buf_write(data->out, "shallow %s", hex); - else { - strbuf_addstr(data->out, hex); - strbuf_addch(data->out, '\n'); - } - return 0; - } - - static int write_shallow_commits(struct strbuf *out, int use_pack_protocol) - { - struct write_shallow_data data; - data.out = out; - data.use_pack_protocol = use_pack_protocol; - data.count = 0; - for_each_commit_graft(write_one_shallow, &data); - return data.count; - } - static enum ack_type get_ack(int fd, unsigned char *result_sha1) { int len; @@@ -796,27 -765,6 +766,6 @@@ static int cmp_ref_by_name(const void * return strcmp(a->name, b->name); } - static void setup_alternate_shallow(void) - { - struct strbuf sb = STRBUF_INIT; - int fd; - - check_shallow_file_for_update(); - fd = hold_lock_file_for_update(&shallow_lock, git_path("shallow"), - LOCK_DIE_ON_ERROR); - if (write_shallow_commits(&sb, 0)) { - if (write_in_full(fd, sb.buf, sb.len) != sb.len) - die_errno("failed to write to %s", shallow_lock.filename); - alternate_shallow_file = shallow_lock.filename; - } else - /* - * is_repository_shallow() sees empty string as "no - * shallow file". - */ - alternate_shallow_file = ""; - strbuf_release(&sb); - } - static struct ref *do_fetch_pack(struct fetch_pack_args *args, int fd[2], const struct ref *orig_ref, @@@ -897,9 -845,7 +846,9 @@@ if (args->stateless_rpc) packet_flush(fd[1]); if (args->depth > 0) - setup_alternate_shallow(); + setup_alternate_shallow(&shallow_lock, &alternate_shallow_file); + else + alternate_shallow_file = NULL; if (get_pack(args, fd, pack_lockfile)) die("git fetch-pack: fetch failed."); @@@ -990,7 -936,7 +939,7 @@@ struct ref *fetch_pack(struct fetch_pac } ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile); - if (alternate_shallow_file) { + if (args->depth > 0 && alternate_shallow_file) { if (*alternate_shallow_file == '\0') { /* --unshallow */ unlink_or_warn(git_path("shallow")); rollback_lock_file(&shallow_lock); diff --combined http-push.c index eea158a878,cde6416d37..69200baf76 --- a/http-push.c +++ b/http-push.c @@@ -1330,7 -1330,8 +1330,7 @@@ static struct object_list **process_tre break; } - free(tree->buffer); - tree->buffer = NULL; + free_tree_buffer(tree); return p; } @@@ -1975,7 -1976,7 +1975,7 @@@ int main(int argc, char **argv pushing = 0; if (prepare_revision_walk(&revs)) die("revision walk setup failed"); - mark_edges_uninteresting(revs.commits, &revs, NULL); + mark_edges_uninteresting(&revs, NULL); objects_to_send = get_delta(&revs, ref_lock); finish_all_active_slots(); diff --combined list-objects.c index c8c3463cad,05c8c5c616..6cbedf0280 --- a/list-objects.c +++ b/list-objects.c @@@ -123,7 -123,8 +123,7 @@@ static void process_tree(struct rev_inf cb_data); } strbuf_setlen(base, baselen); - free(tree->buffer); - tree->buffer = NULL; + free_tree_buffer(tree); } static void mark_edge_parents_uninteresting(struct commit *commit, @@@ -144,19 -145,35 +144,35 @@@ } } - void mark_edges_uninteresting(struct commit_list *list, - struct rev_info *revs, - show_edge_fn show_edge) + void mark_edges_uninteresting(struct rev_info *revs, show_edge_fn show_edge) { - for ( ; list; list = list->next) { + struct commit_list *list; + int i; + + for (list = revs->commits; list; list = list->next) { struct commit *commit = list->item; if (commit->object.flags & UNINTERESTING) { mark_tree_uninteresting(commit->tree); + if (revs->edge_hint && !(commit->object.flags & SHOWN)) { + commit->object.flags |= SHOWN; + show_edge(commit); + } continue; } mark_edge_parents_uninteresting(commit, revs, show_edge); } + for (i = 0; i < revs->cmdline.nr; i++) { + struct object *obj = revs->cmdline.rev[i].item; + struct commit *commit = (struct commit *)obj; + if (obj->type != OBJ_COMMIT || !(obj->flags & UNINTERESTING)) + continue; + mark_tree_uninteresting(commit->tree); + if (revs->edge_hint && !(obj->flags & SHOWN)) { + obj->flags |= SHOWN; + show_edge(commit); + } + } } static void add_pending_tree(struct rev_info *revs, struct tree *tree) diff --combined t/t5500-fetch-pack.sh index a80584ea0e,7a22f557b9..d87ddf73b7 --- a/t/t5500-fetch-pack.sh +++ b/t/t5500-fetch-pack.sh @@@ -393,6 -393,17 +393,17 @@@ test_expect_success 'fetch in shallow r git fsck --no-dangling ) ' + test_expect_success 'fetch creating new shallow root' ' + ( + git clone "file://$(pwd)/." shallow10 && + git commit --allow-empty -m empty && + cd shallow10 && + git fetch --depth=1 --progress 2>actual && + # This should fetch only the empty commit, no tree or + # blob objects + grep "remote: Total 1" actual + ) + ' test_expect_success 'setup tests for the --stdin parameter' ' for head in C D E F @@@ -505,20 -516,4 +516,20 @@@ test_expect_success 'test --all, --dept ) >out-adt 2>error-adt ' +test_expect_success 'shallow fetch with tags does not break the repository' ' + mkdir repo1 && + ( + cd repo1 && + git init && + test_commit 1 && + test_commit 2 && + test_commit 3 && + mkdir repo2 && + cd repo2 && + git init && + git fetch --depth=2 ../.git master:branch && + git fsck + ) +' + test_done diff --combined upload-pack.c index b03492e664,d5a003ad1f..4959dbc5fe --- a/upload-pack.c +++ b/upload-pack.c @@@ -10,7 -10,6 +10,7 @@@ #include "revision.h" #include "list-objects.h" #include "run-command.h" +#include "connect.h" #include "sigchain.h" #include "version.h" #include "string-list.h" @@@ -69,87 -68,28 +69,28 @@@ static ssize_t send_client_data(int fd return sz; } - static FILE *pack_pipe = NULL; - static void show_commit(struct commit *commit, void *data) - { - if (commit->object.flags & BOUNDARY) - fputc('-', pack_pipe); - if (fputs(sha1_to_hex(commit->object.sha1), pack_pipe) < 0) - die("broken output pipe"); - fputc('\n', pack_pipe); - fflush(pack_pipe); - free(commit->buffer); - commit->buffer = NULL; - } - - static void show_object(struct object *obj, - const struct name_path *path, const char *component, - void *cb_data) - { - show_object_with_name(pack_pipe, obj, path, component); - } - - static void show_edge(struct commit *commit) - { - fprintf(pack_pipe, "-%s\n", sha1_to_hex(commit->object.sha1)); - } - - static int do_rev_list(int in, int out, void *user_data) - { - int i; - struct rev_info revs; - - pack_pipe = xfdopen(out, "w"); - init_revisions(&revs, NULL); - revs.tag_objects = 1; - revs.tree_objects = 1; - revs.blob_objects = 1; - if (use_thin_pack) - revs.edge_hint = 1; - - for (i = 0; i < want_obj.nr; i++) { - struct object *o = want_obj.objects[i].item; - /* why??? */ - o->flags &= ~UNINTERESTING; - add_pending_object(&revs, o, NULL); - } - for (i = 0; i < have_obj.nr; i++) { - struct object *o = have_obj.objects[i].item; - o->flags |= UNINTERESTING; - add_pending_object(&revs, o, NULL); - } - setup_revisions(0, NULL, &revs, NULL); - if (prepare_revision_walk(&revs)) - die("revision walk setup failed"); - mark_edges_uninteresting(revs.commits, &revs, show_edge); - if (use_thin_pack) - for (i = 0; i < extra_edge_obj.nr; i++) - fprintf(pack_pipe, "-%s\n", sha1_to_hex( - extra_edge_obj.objects[i].item->sha1)); - traverse_commit_list(&revs, show_commit, show_object, NULL); - fflush(pack_pipe); - fclose(pack_pipe); - return 0; - } - static void create_pack_file(void) { - struct async rev_list; struct child_process pack_objects; char data[8193], progress[128]; char abort_msg[] = "aborting due to possible repository " "corruption on the remote side."; int buffered = -1; ssize_t sz; - const char *argv[10]; - int arg = 0; + const char *argv[12]; + int i, arg = 0; + FILE *pipe_fd; + char *shallow_file = NULL; - argv[arg++] = "pack-objects"; - if (!shallow_nr) { - argv[arg++] = "--revs"; - if (use_thin_pack) - argv[arg++] = "--thin"; + if (shallow_nr) { + shallow_file = setup_temporary_shallow(); + argv[arg++] = "--shallow-file"; + argv[arg++] = shallow_file; } + argv[arg++] = "pack-objects"; + argv[arg++] = "--revs"; + if (use_thin_pack) + argv[arg++] = "--thin"; argv[arg++] = "--stdout"; if (!no_progress) @@@ -170,29 -110,21 +111,21 @@@ if (start_command(&pack_objects)) die("git upload-pack: unable to fork git-pack-objects"); - if (shallow_nr) { - memset(&rev_list, 0, sizeof(rev_list)); - rev_list.proc = do_rev_list; - rev_list.out = pack_objects.in; - if (start_async(&rev_list)) - die("git upload-pack: unable to fork git-rev-list"); - } - else { - FILE *pipe_fd = xfdopen(pack_objects.in, "w"); - int i; - - for (i = 0; i < want_obj.nr; i++) - fprintf(pipe_fd, "%s\n", - sha1_to_hex(want_obj.objects[i].item->sha1)); - fprintf(pipe_fd, "--not\n"); - for (i = 0; i < have_obj.nr; i++) - fprintf(pipe_fd, "%s\n", - sha1_to_hex(have_obj.objects[i].item->sha1)); - fprintf(pipe_fd, "\n"); - fflush(pipe_fd); - fclose(pipe_fd); - } - + pipe_fd = xfdopen(pack_objects.in, "w"); + + for (i = 0; i < want_obj.nr; i++) + fprintf(pipe_fd, "%s\n", + sha1_to_hex(want_obj.objects[i].item->sha1)); + fprintf(pipe_fd, "--not\n"); + for (i = 0; i < have_obj.nr; i++) + fprintf(pipe_fd, "%s\n", + sha1_to_hex(have_obj.objects[i].item->sha1)); + for (i = 0; i < extra_edge_obj.nr; i++) + fprintf(pipe_fd, "%s\n", + sha1_to_hex(extra_edge_obj.objects[i].item->sha1)); + fprintf(pipe_fd, "\n"); + fflush(pipe_fd); + fclose(pipe_fd); /* We read from pack_objects.err to capture stderr output for * progress bar, and pack_objects.out to capture the pack data. @@@ -291,8 -223,11 +224,11 @@@ error("git upload-pack: git-pack-objects died with error."); goto fail; } - if (shallow_nr && finish_async(&rev_list)) - goto fail; /* error was already reported */ + if (shallow_file) { + if (*shallow_file) + unlink(shallow_file); + free(shallow_file); + } /* flush the data */ if (0 <= buffered) {