Merge branch 'nd/fetch-into-shallow' into maint
authorJunio C Hamano <gitster@pobox.com>
Wed, 23 Oct 2013 20:32:17 +0000 (13:32 -0700)
committerJunio C Hamano <gitster@pobox.com>
Wed, 23 Oct 2013 20:32:17 +0000 (13:32 -0700)
When there is no sufficient overlap between old and new history
during a "git fetch" into a shallow repository, objects that the
sending side knows the receiving end has were unnecessarily sent.

* nd/fetch-into-shallow:
Add testcase for needless objects during a shallow fetch
list-objects: mark more commits as edges in mark_edges_uninteresting
list-objects: reduce one argument in mark_edges_uninteresting
upload-pack: delegate rev walking in shallow fetch to pack-objects
shallow: add setup_temporary_shallow()
shallow: only add shallow graft points to new shallow file
move setup_alternate_shallow and write_shallow_commits to shallow.c

1  2 
commit.h
fetch-pack.c
t/t5500-fetch-pack.sh
upload-pack.c
diff --combined commit.h
index f9504f70cc5b57de590099024556b9d2b6b2cdce,c4d324c9558bb7c785f17d35cf770be5e1f4bea9..c308674120112f09ff8a9625e357c62e2029bbba
+++ b/commit.h
@@@ -62,9 -62,6 +62,9 @@@ struct commit_list *commit_list_insert_
                                    struct commit_list **list);
  void commit_list_sort_by_date(struct commit_list **list);
  
 +/* Shallow copy of the input list */
 +struct commit_list *copy_commit_list(struct commit_list *list);
 +
  void free_commit_list(struct commit_list *list);
  
  /* Commit formats */
@@@ -201,6 -198,10 +201,10 @@@ extern struct commit_list *get_shallow_
                int depth, int shallow_flag, int not_shallow_flag);
  extern void check_shallow_file_for_update(void);
  extern void set_alternate_shallow_file(const char *path);
+ extern int write_shallow_commits(struct strbuf *out, int use_pack_protocol);
+ extern void setup_alternate_shallow(struct lock_file *shallow_lock,
+                                   const char **alternate_shallow_file);
+ extern char *setup_temporary_shallow(void);
  
  int is_descendant_of(struct commit *, struct commit_list *);
  int in_merge_bases(struct commit *, struct commit *);
diff --combined fetch-pack.c
index f5d99c11813b1ae2eee0bb7dfd94eab60c721b64,28195ed78b281204850de9c5bfeee03fa29efd79..aff4f5abab667d054c9ab1b925e4877a4fd8d243
@@@ -184,36 -184,6 +184,6 @@@ static void consume_shallow_list(struc
        }
  }
  
- struct write_shallow_data {
-       struct strbuf *out;
-       int use_pack_protocol;
-       int count;
- };
- static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
- {
-       struct write_shallow_data *data = cb_data;
-       const char *hex = sha1_to_hex(graft->sha1);
-       data->count++;
-       if (data->use_pack_protocol)
-               packet_buf_write(data->out, "shallow %s", hex);
-       else {
-               strbuf_addstr(data->out, hex);
-               strbuf_addch(data->out, '\n');
-       }
-       return 0;
- }
- static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
- {
-       struct write_shallow_data data;
-       data.out = out;
-       data.use_pack_protocol = use_pack_protocol;
-       data.count = 0;
-       for_each_commit_graft(write_one_shallow, &data);
-       return data.count;
- }
  static enum ack_type get_ack(int fd, unsigned char *result_sha1)
  {
        int len;
@@@ -795,27 -765,6 +765,6 @@@ static int cmp_ref_by_name(const void *
        return strcmp(a->name, b->name);
  }
  
- static void setup_alternate_shallow(void)
- {
-       struct strbuf sb = STRBUF_INIT;
-       int fd;
-       check_shallow_file_for_update();
-       fd = hold_lock_file_for_update(&shallow_lock, git_path("shallow"),
-                                      LOCK_DIE_ON_ERROR);
-       if (write_shallow_commits(&sb, 0)) {
-               if (write_in_full(fd, sb.buf, sb.len) != sb.len)
-                       die_errno("failed to write to %s", shallow_lock.filename);
-               alternate_shallow_file = shallow_lock.filename;
-       } else
-               /*
-                * is_repository_shallow() sees empty string as "no
-                * shallow file".
-                */
-               alternate_shallow_file = "";
-       strbuf_release(&sb);
- }
  static struct ref *do_fetch_pack(struct fetch_pack_args *args,
                                 int fd[2],
                                 const struct ref *orig_ref,
        if (args->stateless_rpc)
                packet_flush(fd[1]);
        if (args->depth > 0)
-               setup_alternate_shallow();
+               setup_alternate_shallow(&shallow_lock, &alternate_shallow_file);
 +      else
 +              alternate_shallow_file = NULL;
        if (get_pack(args, fd, pack_lockfile))
                die("git fetch-pack: fetch failed.");
  
@@@ -989,7 -936,7 +938,7 @@@ struct ref *fetch_pack(struct fetch_pac
        }
        ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile);
  
 -      if (alternate_shallow_file) {
 +      if (args->depth > 0 && alternate_shallow_file) {
                if (*alternate_shallow_file == '\0') { /* --unshallow */
                        unlink_or_warn(git_path("shallow"));
                        rollback_lock_file(&shallow_lock);
diff --combined t/t5500-fetch-pack.sh
index a80584ea0eaba854ff21318e3c9646f72325bee7,7a22f557b99224fd1cb64e9970272bee98f773d7..d87ddf73b7127bc624ca739653db9389831b817e
@@@ -393,6 -393,17 +393,17 @@@ test_expect_success 'fetch in shallow r
                git fsck --no-dangling
        )
  '
+ test_expect_success 'fetch creating new shallow root' '
+       (
+               git clone "file://$(pwd)/." shallow10 &&
+               git commit --allow-empty -m empty &&
+               cd shallow10 &&
+               git fetch --depth=1 --progress 2>actual &&
+               # This should fetch only the empty commit, no tree or
+               # blob objects
+               grep "remote: Total 1" actual
+       )
+ '
  
  test_expect_success 'setup tests for the --stdin parameter' '
        for head in C D E F
@@@ -505,20 -516,4 +516,20 @@@ test_expect_success 'test --all, --dept
        ) >out-adt 2>error-adt
  '
  
 +test_expect_success 'shallow fetch with tags does not break the repository' '
 +      mkdir repo1 &&
 +      (
 +              cd repo1 &&
 +              git init &&
 +              test_commit 1 &&
 +              test_commit 2 &&
 +              test_commit 3 &&
 +              mkdir repo2 &&
 +              cd repo2 &&
 +              git init &&
 +              git fetch --depth=2 ../.git master:branch &&
 +              git fsck
 +      )
 +'
 +
  test_done
diff --combined upload-pack.c
index 04a8707bb58e7411a2c272a2716e6b8b340a2c7e,d5a003ad1fe6f395c181a16d13b9f11b34efe2da..8327dc0b7c80b61fc24342a1816e323009818729
@@@ -40,7 -40,6 +40,7 @@@ static struct object_array have_obj
  static struct object_array want_obj;
  static struct object_array extra_edge_obj;
  static unsigned int timeout;
 +static int keepalive = 5;
  /* 0 for no sideband,
   * otherwise maximum packet size (up to 65520 bytes).
   */
@@@ -69,87 -68,28 +69,28 @@@ static ssize_t send_client_data(int fd
        return sz;
  }
  
- static FILE *pack_pipe = NULL;
- static void show_commit(struct commit *commit, void *data)
- {
-       if (commit->object.flags & BOUNDARY)
-               fputc('-', pack_pipe);
-       if (fputs(sha1_to_hex(commit->object.sha1), pack_pipe) < 0)
-               die("broken output pipe");
-       fputc('\n', pack_pipe);
-       fflush(pack_pipe);
-       free(commit->buffer);
-       commit->buffer = NULL;
- }
- static void show_object(struct object *obj,
-                       const struct name_path *path, const char *component,
-                       void *cb_data)
- {
-       show_object_with_name(pack_pipe, obj, path, component);
- }
- static void show_edge(struct commit *commit)
- {
-       fprintf(pack_pipe, "-%s\n", sha1_to_hex(commit->object.sha1));
- }
- static int do_rev_list(int in, int out, void *user_data)
- {
-       int i;
-       struct rev_info revs;
-       pack_pipe = xfdopen(out, "w");
-       init_revisions(&revs, NULL);
-       revs.tag_objects = 1;
-       revs.tree_objects = 1;
-       revs.blob_objects = 1;
-       if (use_thin_pack)
-               revs.edge_hint = 1;
-       for (i = 0; i < want_obj.nr; i++) {
-               struct object *o = want_obj.objects[i].item;
-               /* why??? */
-               o->flags &= ~UNINTERESTING;
-               add_pending_object(&revs, o, NULL);
-       }
-       for (i = 0; i < have_obj.nr; i++) {
-               struct object *o = have_obj.objects[i].item;
-               o->flags |= UNINTERESTING;
-               add_pending_object(&revs, o, NULL);
-       }
-       setup_revisions(0, NULL, &revs, NULL);
-       if (prepare_revision_walk(&revs))
-               die("revision walk setup failed");
-       mark_edges_uninteresting(revs.commits, &revs, show_edge);
-       if (use_thin_pack)
-               for (i = 0; i < extra_edge_obj.nr; i++)
-                       fprintf(pack_pipe, "-%s\n", sha1_to_hex(
-                                       extra_edge_obj.objects[i].item->sha1));
-       traverse_commit_list(&revs, show_commit, show_object, NULL);
-       fflush(pack_pipe);
-       fclose(pack_pipe);
-       return 0;
- }
  static void create_pack_file(void)
  {
-       struct async rev_list;
        struct child_process pack_objects;
        char data[8193], progress[128];
        char abort_msg[] = "aborting due to possible repository "
                "corruption on the remote side.";
        int buffered = -1;
        ssize_t sz;
-       const char *argv[10];
-       int arg = 0;
+       const char *argv[12];
+       int i, arg = 0;
+       FILE *pipe_fd;
+       char *shallow_file = NULL;
  
-       argv[arg++] = "pack-objects";
-       if (!shallow_nr) {
-               argv[arg++] = "--revs";
-               if (use_thin_pack)
-                       argv[arg++] = "--thin";
+       if (shallow_nr) {
+               shallow_file = setup_temporary_shallow();
+               argv[arg++] = "--shallow-file";
+               argv[arg++] = shallow_file;
        }
+       argv[arg++] = "pack-objects";
+       argv[arg++] = "--revs";
+       if (use_thin_pack)
+               argv[arg++] = "--thin";
  
        argv[arg++] = "--stdout";
        if (!no_progress)
        if (start_command(&pack_objects))
                die("git upload-pack: unable to fork git-pack-objects");
  
-       if (shallow_nr) {
-               memset(&rev_list, 0, sizeof(rev_list));
-               rev_list.proc = do_rev_list;
-               rev_list.out = pack_objects.in;
-               if (start_async(&rev_list))
-                       die("git upload-pack: unable to fork git-rev-list");
-       }
-       else {
-               FILE *pipe_fd = xfdopen(pack_objects.in, "w");
-               int i;
-               for (i = 0; i < want_obj.nr; i++)
-                       fprintf(pipe_fd, "%s\n",
-                               sha1_to_hex(want_obj.objects[i].item->sha1));
-               fprintf(pipe_fd, "--not\n");
-               for (i = 0; i < have_obj.nr; i++)
-                       fprintf(pipe_fd, "%s\n",
-                               sha1_to_hex(have_obj.objects[i].item->sha1));
-               fprintf(pipe_fd, "\n");
-               fflush(pipe_fd);
-               fclose(pipe_fd);
-       }
+       pipe_fd = xfdopen(pack_objects.in, "w");
  
+       for (i = 0; i < want_obj.nr; i++)
+               fprintf(pipe_fd, "%s\n",
+                       sha1_to_hex(want_obj.objects[i].item->sha1));
+       fprintf(pipe_fd, "--not\n");
+       for (i = 0; i < have_obj.nr; i++)
+               fprintf(pipe_fd, "%s\n",
+                       sha1_to_hex(have_obj.objects[i].item->sha1));
+       for (i = 0; i < extra_edge_obj.nr; i++)
+               fprintf(pipe_fd, "%s\n",
+                       sha1_to_hex(extra_edge_obj.objects[i].item->sha1));
+       fprintf(pipe_fd, "\n");
+       fflush(pipe_fd);
+       fclose(pipe_fd);
  
        /* We read from pack_objects.err to capture stderr output for
         * progress bar, and pack_objects.out to capture the pack data.
        while (1) {
                struct pollfd pfd[2];
                int pe, pu, pollsize;
 +              int ret;
  
                reset_timeout();
  
                if (!pollsize)
                        break;
  
 -              if (poll(pfd, pollsize, -1) < 0) {
 +              ret = poll(pfd, pollsize, 1000 * keepalive);
 +              if (ret < 0) {
                        if (errno != EINTR) {
                                error("poll failed, resuming: %s",
                                      strerror(errno));
                        if (sz < 0)
                                goto fail;
                }
 +
 +              /*
 +               * We hit the keepalive timeout without saying anything; send
 +               * an empty message on the data sideband just to let the other
 +               * side know we're still working on it, but don't have any data
 +               * yet.
 +               *
 +               * If we don't have a sideband channel, there's no room in the
 +               * protocol to say anything, so those clients are just out of
 +               * luck.
 +               */
 +              if (!ret && use_sideband) {
 +                      static const char buf[] = "0005\1";
 +                      write_or_die(1, buf, 5);
 +              }
        }
  
        if (finish_command(&pack_objects)) {
                error("git upload-pack: git-pack-objects died with error.");
                goto fail;
        }
-       if (shallow_nr && finish_async(&rev_list))
-               goto fail;      /* error was already reported */
+       if (shallow_file) {
+               if (*shallow_file)
+                       unlink(shallow_file);
+               free(shallow_file);
+       }
  
        /* flush the data */
        if (0 <= buffered) {
@@@ -803,11 -721,6 +739,11 @@@ static int upload_pack_config(const cha
  {
        if (!strcmp("uploadpack.allowtipsha1inwant", var))
                allow_tip_sha1_in_want = git_config_bool(var, value);
 +      else if (!strcmp("uploadpack.keepalive", var)) {
 +              keepalive = git_config_int(var, value);
 +              if (!keepalive)
 +                      keepalive = -1;
 +      }
        return parse_hide_refs_config(var, value, "uploadpack");
  }