Merge branch 'nd/fetch-into-shallow'
authorJunio C Hamano <gitster@pobox.com>
Fri, 20 Sep 2013 19:25:32 +0000 (12:25 -0700)
committerJunio C Hamano <gitster@pobox.com>
Fri, 20 Sep 2013 19:25:32 +0000 (12:25 -0700)
When there is no sufficient overlap between old and new history
during a fetch into a shallow repository, we unnecessarily sent
objects the sending side knows the receiving end has.

* nd/fetch-into-shallow:
Add testcase for needless objects during a shallow fetch
list-objects: mark more commits as edges in mark_edges_uninteresting
list-objects: reduce one argument in mark_edges_uninteresting
upload-pack: delegate rev walking in shallow fetch to pack-objects
shallow: add setup_temporary_shallow()
shallow: only add shallow graft points to new shallow file
move setup_alternate_shallow and write_shallow_commits to shallow.c

1  2 
builtin/pack-objects.c
commit.h
fetch-pack.c
http-push.c
list-objects.c
t/t5500-fetch-pack.sh
upload-pack.c
diff --combined builtin/pack-objects.c
index 4eb0521c815870abb670e8ddb035caa9c6507e67,dd117b379ac9d7b58b439a691b90c6cb4f1dac90..e86cd5729f38316bf6e304e4e97ce671e4ea135b
@@@ -1809,7 -1809,7 +1809,7 @@@ static void find_deltas(struct object_e
  static void try_to_free_from_threads(size_t size)
  {
        read_lock();
 -      release_pack_memory(size, -1);
 +      release_pack_memory(size);
        read_unlock();
  }
  
@@@ -2378,7 -2378,7 +2378,7 @@@ static void get_object_list(int ac, con
  
        if (prepare_revision_walk(&revs))
                die("revision walk setup failed");
-       mark_edges_uninteresting(revs.commits, &revs, show_edge);
+       mark_edges_uninteresting(&revs, show_edge);
        traverse_commit_list(&revs, show_commit, show_object, NULL);
  
        if (keep_unreachable)
diff --combined commit.h
index 90a5a3c361e02cd7a9562eff2ca1b15f77be0941,c4d324c9558bb7c785f17d35cf770be5e1f4bea9..bd841f4d0c5e2b7fb8e83d55ba0d0b7e2839bf8a
+++ b/commit.h
@@@ -62,9 -62,6 +62,9 @@@ struct commit_list *commit_list_insert_
                                    struct commit_list **list);
  void commit_list_sort_by_date(struct commit_list **list);
  
 +/* Shallow copy of the input list */
 +struct commit_list *copy_commit_list(struct commit_list *list);
 +
  void free_commit_list(struct commit_list *list);
  
  /* Commit formats */
@@@ -201,6 -198,10 +201,10 @@@ extern struct commit_list *get_shallow_
                int depth, int shallow_flag, int not_shallow_flag);
  extern void check_shallow_file_for_update(void);
  extern void set_alternate_shallow_file(const char *path);
+ extern int write_shallow_commits(struct strbuf *out, int use_pack_protocol);
+ extern void setup_alternate_shallow(struct lock_file *shallow_lock,
+                                   const char **alternate_shallow_file);
+ extern char *setup_temporary_shallow(void);
  
  int is_descendant_of(struct commit *, struct commit_list *);
  int in_merge_bases(struct commit *, struct commit *);
@@@ -208,7 -209,7 +212,7 @@@ int in_merge_bases_many(struct commit *
  
  extern int interactive_add(int argc, const char **argv, const char *prefix, int patch);
  extern int run_add_interactive(const char *revision, const char *patch_mode,
 -                             const char **pathspec);
 +                             const struct pathspec *pathspec);
  
  static inline int single_parent(struct commit *commit)
  {
diff --combined fetch-pack.c
index 094267fd80cdb09feec75c89aa2f4931f11e583d,28195ed78b281204850de9c5bfeee03fa29efd79..13b5b43bfa9092476d8715c005e06086b7343a54
@@@ -9,7 -9,6 +9,7 @@@
  #include "fetch-pack.h"
  #include "remote.h"
  #include "run-command.h"
 +#include "connect.h"
  #include "transport.h"
  #include "version.h"
  #include "prio-queue.h"
@@@ -185,36 -184,6 +185,6 @@@ static void consume_shallow_list(struc
        }
  }
  
- struct write_shallow_data {
-       struct strbuf *out;
-       int use_pack_protocol;
-       int count;
- };
- static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
- {
-       struct write_shallow_data *data = cb_data;
-       const char *hex = sha1_to_hex(graft->sha1);
-       data->count++;
-       if (data->use_pack_protocol)
-               packet_buf_write(data->out, "shallow %s", hex);
-       else {
-               strbuf_addstr(data->out, hex);
-               strbuf_addch(data->out, '\n');
-       }
-       return 0;
- }
- static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
- {
-       struct write_shallow_data data;
-       data.out = out;
-       data.use_pack_protocol = use_pack_protocol;
-       data.count = 0;
-       for_each_commit_graft(write_one_shallow, &data);
-       return data.count;
- }
  static enum ack_type get_ack(int fd, unsigned char *result_sha1)
  {
        int len;
@@@ -796,27 -765,6 +766,6 @@@ static int cmp_ref_by_name(const void *
        return strcmp(a->name, b->name);
  }
  
- static void setup_alternate_shallow(void)
- {
-       struct strbuf sb = STRBUF_INIT;
-       int fd;
-       check_shallow_file_for_update();
-       fd = hold_lock_file_for_update(&shallow_lock, git_path("shallow"),
-                                      LOCK_DIE_ON_ERROR);
-       if (write_shallow_commits(&sb, 0)) {
-               if (write_in_full(fd, sb.buf, sb.len) != sb.len)
-                       die_errno("failed to write to %s", shallow_lock.filename);
-               alternate_shallow_file = shallow_lock.filename;
-       } else
-               /*
-                * is_repository_shallow() sees empty string as "no
-                * shallow file".
-                */
-               alternate_shallow_file = "";
-       strbuf_release(&sb);
- }
  static struct ref *do_fetch_pack(struct fetch_pack_args *args,
                                 int fd[2],
                                 const struct ref *orig_ref,
        if (args->stateless_rpc)
                packet_flush(fd[1]);
        if (args->depth > 0)
-               setup_alternate_shallow();
+               setup_alternate_shallow(&shallow_lock, &alternate_shallow_file);
 +      else
 +              alternate_shallow_file = NULL;
        if (get_pack(args, fd, pack_lockfile))
                die("git fetch-pack: fetch failed.");
  
@@@ -990,7 -936,7 +939,7 @@@ struct ref *fetch_pack(struct fetch_pac
        }
        ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile);
  
 -      if (alternate_shallow_file) {
 +      if (args->depth > 0 && alternate_shallow_file) {
                if (*alternate_shallow_file == '\0') { /* --unshallow */
                        unlink_or_warn(git_path("shallow"));
                        rollback_lock_file(&shallow_lock);
diff --combined http-push.c
index eea158a8785c4d7c87724146c141edadac0f8dfe,cde6416d37751fe0724e7c021e858530a74cf466..69200baf7651c2bcb99d6bc2068429e22865f900
@@@ -1330,7 -1330,8 +1330,7 @@@ static struct object_list **process_tre
                        break;
                }
  
 -      free(tree->buffer);
 -      tree->buffer = NULL;
 +      free_tree_buffer(tree);
        return p;
  }
  
@@@ -1975,7 -1976,7 +1975,7 @@@ int main(int argc, char **argv
                pushing = 0;
                if (prepare_revision_walk(&revs))
                        die("revision walk setup failed");
-               mark_edges_uninteresting(revs.commits, &revs, NULL);
+               mark_edges_uninteresting(&revs, NULL);
                objects_to_send = get_delta(&revs, ref_lock);
                finish_all_active_slots();
  
diff --combined list-objects.c
index c8c3463cad6db4145434ee08f6c3a494e26a8675,05c8c5c61660fdf351246df4279b7642695f8c7e..6cbedf0280076d8fe6e49241d89938955ecddead
@@@ -123,7 -123,8 +123,7 @@@ static void process_tree(struct rev_inf
                                     cb_data);
        }
        strbuf_setlen(base, baselen);
 -      free(tree->buffer);
 -      tree->buffer = NULL;
 +      free_tree_buffer(tree);
  }
  
  static void mark_edge_parents_uninteresting(struct commit *commit,
        }
  }
  
- void mark_edges_uninteresting(struct commit_list *list,
-                             struct rev_info *revs,
-                             show_edge_fn show_edge)
+ void mark_edges_uninteresting(struct rev_info *revs, show_edge_fn show_edge)
  {
-       for ( ; list; list = list->next) {
+       struct commit_list *list;
+       int i;
+       for (list = revs->commits; list; list = list->next) {
                struct commit *commit = list->item;
  
                if (commit->object.flags & UNINTERESTING) {
                        mark_tree_uninteresting(commit->tree);
+                       if (revs->edge_hint && !(commit->object.flags & SHOWN)) {
+                               commit->object.flags |= SHOWN;
+                               show_edge(commit);
+                       }
                        continue;
                }
                mark_edge_parents_uninteresting(commit, revs, show_edge);
        }
+       for (i = 0; i < revs->cmdline.nr; i++) {
+               struct object *obj = revs->cmdline.rev[i].item;
+               struct commit *commit = (struct commit *)obj;
+               if (obj->type != OBJ_COMMIT || !(obj->flags & UNINTERESTING))
+                       continue;
+               mark_tree_uninteresting(commit->tree);
+               if (revs->edge_hint && !(obj->flags & SHOWN)) {
+                       obj->flags |= SHOWN;
+                       show_edge(commit);
+               }
+       }
  }
  
  static void add_pending_tree(struct rev_info *revs, struct tree *tree)
diff --combined t/t5500-fetch-pack.sh
index a80584ea0eaba854ff21318e3c9646f72325bee7,7a22f557b99224fd1cb64e9970272bee98f773d7..d87ddf73b7127bc624ca739653db9389831b817e
@@@ -393,6 -393,17 +393,17 @@@ test_expect_success 'fetch in shallow r
                git fsck --no-dangling
        )
  '
+ test_expect_success 'fetch creating new shallow root' '
+       (
+               git clone "file://$(pwd)/." shallow10 &&
+               git commit --allow-empty -m empty &&
+               cd shallow10 &&
+               git fetch --depth=1 --progress 2>actual &&
+               # This should fetch only the empty commit, no tree or
+               # blob objects
+               grep "remote: Total 1" actual
+       )
+ '
  
  test_expect_success 'setup tests for the --stdin parameter' '
        for head in C D E F
@@@ -505,20 -516,4 +516,20 @@@ test_expect_success 'test --all, --dept
        ) >out-adt 2>error-adt
  '
  
 +test_expect_success 'shallow fetch with tags does not break the repository' '
 +      mkdir repo1 &&
 +      (
 +              cd repo1 &&
 +              git init &&
 +              test_commit 1 &&
 +              test_commit 2 &&
 +              test_commit 3 &&
 +              mkdir repo2 &&
 +              cd repo2 &&
 +              git init &&
 +              git fetch --depth=2 ../.git master:branch &&
 +              git fsck
 +      )
 +'
 +
  test_done
diff --combined upload-pack.c
index b03492e664daa53f09d13787494c925af702d949,d5a003ad1fe6f395c181a16d13b9f11b34efe2da..4959dbc5fe17993e563bc4da926509000c327ca1
@@@ -10,7 -10,6 +10,7 @@@
  #include "revision.h"
  #include "list-objects.h"
  #include "run-command.h"
 +#include "connect.h"
  #include "sigchain.h"
  #include "version.h"
  #include "string-list.h"
@@@ -69,87 -68,28 +69,28 @@@ static ssize_t send_client_data(int fd
        return sz;
  }
  
- static FILE *pack_pipe = NULL;
- static void show_commit(struct commit *commit, void *data)
- {
-       if (commit->object.flags & BOUNDARY)
-               fputc('-', pack_pipe);
-       if (fputs(sha1_to_hex(commit->object.sha1), pack_pipe) < 0)
-               die("broken output pipe");
-       fputc('\n', pack_pipe);
-       fflush(pack_pipe);
-       free(commit->buffer);
-       commit->buffer = NULL;
- }
- static void show_object(struct object *obj,
-                       const struct name_path *path, const char *component,
-                       void *cb_data)
- {
-       show_object_with_name(pack_pipe, obj, path, component);
- }
- static void show_edge(struct commit *commit)
- {
-       fprintf(pack_pipe, "-%s\n", sha1_to_hex(commit->object.sha1));
- }
- static int do_rev_list(int in, int out, void *user_data)
- {
-       int i;
-       struct rev_info revs;
-       pack_pipe = xfdopen(out, "w");
-       init_revisions(&revs, NULL);
-       revs.tag_objects = 1;
-       revs.tree_objects = 1;
-       revs.blob_objects = 1;
-       if (use_thin_pack)
-               revs.edge_hint = 1;
-       for (i = 0; i < want_obj.nr; i++) {
-               struct object *o = want_obj.objects[i].item;
-               /* why??? */
-               o->flags &= ~UNINTERESTING;
-               add_pending_object(&revs, o, NULL);
-       }
-       for (i = 0; i < have_obj.nr; i++) {
-               struct object *o = have_obj.objects[i].item;
-               o->flags |= UNINTERESTING;
-               add_pending_object(&revs, o, NULL);
-       }
-       setup_revisions(0, NULL, &revs, NULL);
-       if (prepare_revision_walk(&revs))
-               die("revision walk setup failed");
-       mark_edges_uninteresting(revs.commits, &revs, show_edge);
-       if (use_thin_pack)
-               for (i = 0; i < extra_edge_obj.nr; i++)
-                       fprintf(pack_pipe, "-%s\n", sha1_to_hex(
-                                       extra_edge_obj.objects[i].item->sha1));
-       traverse_commit_list(&revs, show_commit, show_object, NULL);
-       fflush(pack_pipe);
-       fclose(pack_pipe);
-       return 0;
- }
  static void create_pack_file(void)
  {
-       struct async rev_list;
        struct child_process pack_objects;
        char data[8193], progress[128];
        char abort_msg[] = "aborting due to possible repository "
                "corruption on the remote side.";
        int buffered = -1;
        ssize_t sz;
-       const char *argv[10];
-       int arg = 0;
+       const char *argv[12];
+       int i, arg = 0;
+       FILE *pipe_fd;
+       char *shallow_file = NULL;
  
-       argv[arg++] = "pack-objects";
-       if (!shallow_nr) {
-               argv[arg++] = "--revs";
-               if (use_thin_pack)
-                       argv[arg++] = "--thin";
+       if (shallow_nr) {
+               shallow_file = setup_temporary_shallow();
+               argv[arg++] = "--shallow-file";
+               argv[arg++] = shallow_file;
        }
+       argv[arg++] = "pack-objects";
+       argv[arg++] = "--revs";
+       if (use_thin_pack)
+               argv[arg++] = "--thin";
  
        argv[arg++] = "--stdout";
        if (!no_progress)
        if (start_command(&pack_objects))
                die("git upload-pack: unable to fork git-pack-objects");
  
-       if (shallow_nr) {
-               memset(&rev_list, 0, sizeof(rev_list));
-               rev_list.proc = do_rev_list;
-               rev_list.out = pack_objects.in;
-               if (start_async(&rev_list))
-                       die("git upload-pack: unable to fork git-rev-list");
-       }
-       else {
-               FILE *pipe_fd = xfdopen(pack_objects.in, "w");
-               int i;
-               for (i = 0; i < want_obj.nr; i++)
-                       fprintf(pipe_fd, "%s\n",
-                               sha1_to_hex(want_obj.objects[i].item->sha1));
-               fprintf(pipe_fd, "--not\n");
-               for (i = 0; i < have_obj.nr; i++)
-                       fprintf(pipe_fd, "%s\n",
-                               sha1_to_hex(have_obj.objects[i].item->sha1));
-               fprintf(pipe_fd, "\n");
-               fflush(pipe_fd);
-               fclose(pipe_fd);
-       }
+       pipe_fd = xfdopen(pack_objects.in, "w");
+       for (i = 0; i < want_obj.nr; i++)
+               fprintf(pipe_fd, "%s\n",
+                       sha1_to_hex(want_obj.objects[i].item->sha1));
+       fprintf(pipe_fd, "--not\n");
+       for (i = 0; i < have_obj.nr; i++)
+               fprintf(pipe_fd, "%s\n",
+                       sha1_to_hex(have_obj.objects[i].item->sha1));
+       for (i = 0; i < extra_edge_obj.nr; i++)
+               fprintf(pipe_fd, "%s\n",
+                       sha1_to_hex(extra_edge_obj.objects[i].item->sha1));
+       fprintf(pipe_fd, "\n");
+       fflush(pipe_fd);
+       fclose(pipe_fd);
  
        /* We read from pack_objects.err to capture stderr output for
         * progress bar, and pack_objects.out to capture the pack data.
                error("git upload-pack: git-pack-objects died with error.");
                goto fail;
        }
-       if (shallow_nr && finish_async(&rev_list))
-               goto fail;      /* error was already reported */
+       if (shallow_file) {
+               if (*shallow_file)
+                       unlink(shallow_file);
+               free(shallow_file);
+       }
  
        /* flush the data */
        if (0 <= buffered) {