Merge branch 'js/shallow-and-fetch-prune'
authorJunio C Hamano <gitster@pobox.com>
Tue, 6 Nov 2018 06:50:18 +0000 (15:50 +0900)
committerJunio C Hamano <gitster@pobox.com>
Tue, 6 Nov 2018 06:50:18 +0000 (15:50 +0900)
"git repack" in a shallow clone did not correctly update the
shallow points in the repository, leading to a repository that
does not pass fsck.

* js/shallow-and-fetch-prune:
repack -ad: prune the list of shallow commits
shallow: offer to prune only non-existing entries
repack: point out a bug handling stale shallow info

1  2 
builtin/prune.c
builtin/repack.c
commit.h
shallow.c
diff --combined builtin/prune.c
index 41230f82157e48b2cd6741deae8de0e0d1c3be61,b29ce4abbce006ae63424352dec80de530202ec1..1ec9ddd751df6644d2c39ace41a1494800082638
@@@ -120,7 -120,7 +120,7 @@@ int cmd_prune(int argc, const char **ar
        save_commit_buffer = 0;
        read_replace_refs = 0;
        ref_paranoia = 1;
 -      init_revisions(&revs, prefix);
 +      repo_init_revisions(the_repository, &revs, prefix);
  
        argc = parse_options(argc, argv, prefix, options, prune_usage, 0);
  
        free(s);
  
        if (is_repository_shallow(the_repository))
-               prune_shallow(show_only);
+               prune_shallow(show_only ? PRUNE_SHOW_ONLY : 0);
  
        return 0;
  }
diff --combined builtin/repack.c
index 0223f2880ceaec57d10aadb4fcce7a28ce670cc4,3c30e16ec5483c2a88cbe96f67475746487b31a9..82c19b755509f83bf189dd70323a678c299c6b66
@@@ -8,14 -8,12 +8,14 @@@
  #include "strbuf.h"
  #include "string-list.h"
  #include "argv-array.h"
 +#include "midx.h"
  #include "packfile.h"
  #include "object-store.h"
  
  static int delta_base_offset = 1;
  static int pack_kept_objects = -1;
  static int write_bitmaps;
 +static int use_delta_islands;
  static char *packdir, *packtmp;
  
  static const char *const git_repack_usage[] = {
@@@ -44,10 -42,6 +44,10 @@@ static int repack_config(const char *va
                write_bitmaps = git_config_bool(var, value);
                return 0;
        }
 +      if (!strcmp(var, "repack.usedeltaislands")) {
 +              use_delta_islands = git_config_bool(var, value);
 +              return 0;
 +      }
        return git_default_config(var, value, cb);
  }
  
@@@ -235,8 -229,8 +235,8 @@@ static void repack_promisor_objects(con
        while (strbuf_getline_lf(&line, out) != EOF) {
                char *promisor_name;
                int fd;
 -              if (line.len != 40)
 -                      die("repack: Expecting 40 character sha1 lines only from pack-objects.");
 +              if (line.len != the_hash_algo->hexsz)
 +                      die("repack: Expecting full hex object ID lines only from pack-objects.");
                string_list_append(names, line.buf);
  
                /*
@@@ -286,7 -280,6 +286,7 @@@ int cmd_repack(int argc, const char **a
        int keep_unreachable = 0;
        struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;
        int no_update_server_info = 0;
 +      int midx_cleared = 0;
        struct pack_objects_args po_args = {NULL};
  
        struct option builtin_repack_options[] = {
                                N_("pass --local to git-pack-objects")),
                OPT_BOOL('b', "write-bitmap-index", &write_bitmaps,
                                N_("write bitmap index")),
 +              OPT_BOOL('i', "delta-islands", &use_delta_islands,
 +                              N_("pass --delta-islands to git-pack-objects")),
                OPT_STRING(0, "unpack-unreachable", &unpack_unreachable, N_("approxidate"),
                                N_("with -A, do not loosen objects older than this")),
                OPT_BOOL('k', "keep-unreachable", &keep_unreachable,
                argv_array_push(&cmd.args, "--exclude-promisor-objects");
        if (write_bitmaps)
                argv_array_push(&cmd.args, "--write-bitmap-index");
 +      if (use_delta_islands)
 +              argv_array_push(&cmd.args, "--delta-islands");
  
        if (pack_everything & ALL_INTO_ONE) {
                get_non_kept_pack_filenames(&existing_packs, &keep_pack_list);
  
        out = xfdopen(cmd.out, "r");
        while (strbuf_getline_lf(&line, out) != EOF) {
 -              if (line.len != 40)
 -                      die("repack: Expecting 40 character sha1 lines only from pack-objects.");
 +              if (line.len != the_hash_algo->hexsz)
 +                      die("repack: Expecting full hex object ID lines only from pack-objects.");
                string_list_append(&names, line.buf);
        }
        fclose(out);
        for_each_string_list_item(item, &names) {
                for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
                        char *fname, *fname_old;
 +
 +                      if (!midx_cleared) {
 +                              /* if we move a packfile, it will invalidated the midx */
 +                              clear_midx_file(get_object_directory());
 +                              midx_cleared = 1;
 +                      }
 +
                        fname = mkpathdup("%s/pack-%s%s", packdir,
                                                item->string, exts[ext].name);
                        if (!file_exists(fname)) {
        reprepare_packed_git(the_repository);
  
        if (delete_redundant) {
 +              const int hexsz = the_hash_algo->hexsz;
                int opts = 0;
                string_list_sort(&names);
                for_each_string_list_item(item, &existing_packs) {
                        char *sha1;
                        size_t len = strlen(item->string);
 -                      if (len < 40)
 +                      if (len < hexsz)
                                continue;
 -                      sha1 = item->string + len - 40;
 +                      sha1 = item->string + len - hexsz;
                        if (!string_list_has_string(&names, sha1))
                                remove_redundant_pack(packdir, item->string);
                }
                if (!po_args.quiet && isatty(2))
                        opts |= PRUNE_PACKED_VERBOSE;
                prune_packed_objects(opts);
+               if (!keep_unreachable &&
+                   (!(pack_everything & LOOSEN_UNREACHABLE) ||
+                    unpack_unreachable) &&
+                   is_repository_shallow(the_repository))
+                       prune_shallow(PRUNE_QUICK);
        }
  
        if (!no_update_server_info)
diff --combined commit.h
index 6c4428c5931a9ab9d77d96a8a21aba4d75d73108,861a1593142e9c1816fc47c464143864c7bfbbab..8f15cfd43b602f0b85eda82fb9af90eef7996b6c
+++ b/commit.h
@@@ -202,10 -202,14 +202,10 @@@ typedef int (*each_commit_graft_fn)(con
  
  struct commit_graft *read_graft_line(struct strbuf *line);
  int register_commit_graft(struct repository *r, struct commit_graft *, int);
 +void prepare_commit_graft(struct repository *r);
  struct commit_graft *lookup_commit_graft(struct repository *r, const struct object_id *oid);
  
 -extern struct commit_list *get_merge_bases(struct commit *rev1, struct commit *rev2);
 -extern struct commit_list *get_merge_bases_many(struct commit *one, int n, struct commit **twos);
 -extern struct commit_list *get_octopus_merge_bases(struct commit_list *in);
 -
 -/* To be used only when object flags after this call no longer matter */
 -extern struct commit_list *get_merge_bases_many_dirty(struct commit *one, int n, struct commit **twos);
 +struct commit *get_fork_point(const char *refname, struct commit *commit);
  
  /* largest positive number a signed 32-bit integer can contain */
  #define INFINITE_DEPTH 0x7fffffff
@@@ -251,13 -255,37 +251,15 @@@ extern void assign_shallow_commits_to_r
                                           uint32_t **used,
                                           int *ref_status);
  extern int delayed_reachability_test(struct shallow_info *si, int c);
- extern void prune_shallow(int show_only);
+ #define PRUNE_SHOW_ONLY 1
+ #define PRUNE_QUICK 2
+ extern void prune_shallow(unsigned options);
  extern struct trace_key trace_shallow;
  
 -int is_descendant_of(struct commit *, struct commit_list *);
 -int in_merge_bases(struct commit *, struct commit *);
 -int in_merge_bases_many(struct commit *, int, struct commit **);
 -
  extern int interactive_add(int argc, const char **argv, const char *prefix, int patch);
  extern int run_add_interactive(const char *revision, const char *patch_mode,
                               const struct pathspec *pathspec);
  
 -/*
 - * Takes a list of commits and returns a new list where those
 - * have been removed that can be reached from other commits in
 - * the list. It is useful for, e.g., reducing the commits
 - * randomly thrown at the git-merge command and removing
 - * redundant commits that the user shouldn't have given to it.
 - *
 - * This function destroys the STALE bit of the commit objects'
 - * flags.
 - */
 -extern struct commit_list *reduce_heads(struct commit_list *heads);
 -
 -/*
 - * Like `reduce_heads()`, except it replaces the list. Use this
 - * instead of `foo = reduce_heads(foo);` to avoid memory leaks.
 - */
 -extern void reduce_heads_replace(struct commit_list **heads);
 -
  struct commit_extra_header {
        struct commit_extra_header *next;
        char *key;
@@@ -296,7 -324,7 +298,7 @@@ extern const char *find_commit_header(c
                                      size_t *out_len);
  
  /* Find the end of the log message, the right place for a new trailer. */
 -extern int ignore_non_trailer(const char *buf, size_t len);
 +extern size_t ignore_non_trailer(const char *buf, size_t len);
  
  typedef int (*each_mergetag_fn)(struct commit *commit, struct commit_extra_header *extra,
                                 void *cb_data);
diff --combined shallow.c
index 732e18d54f3b88d8a081786eabd030d84eb7e0a3,c1b68533ca3d831afcef7fd655cce95d8db89547..02fdbfc554c462c1eecdf1ddc6d17edbc1d8d853
+++ b/shallow.c
@@@ -16,7 -16,6 +16,7 @@@
  #include "list-objects.h"
  #include "commit-slab.h"
  #include "repository.h"
 +#include "commit-reach.h"
  
  void set_alternate_shallow_file(struct repository *r, const char *path, int override)
  {
@@@ -185,7 -184,7 +185,7 @@@ struct commit_list *get_shallow_commits
  
        is_repository_shallow(the_repository); /* make sure shallows are read */
  
 -      init_revisions(&revs, NULL);
 +      repo_init_revisions(the_repository, &revs, NULL);
        save_commit_buffer = 0;
        setup_revisions(ac, av, &revs, NULL);
  
@@@ -247,6 -246,7 +247,7 @@@ static void check_shallow_file_for_upda
  
  #define SEEN_ONLY 1
  #define VERBOSE   2
+ #define QUICK 4
  
  struct write_shallow_data {
        struct strbuf *out;
@@@ -261,7 -261,10 +262,10 @@@ static int write_one_shallow(const stru
        const char *hex = oid_to_hex(&graft->oid);
        if (graft->nr_parent != -1)
                return 0;
-       if (data->flags & SEEN_ONLY) {
+       if (data->flags & QUICK) {
+               if (!has_object_file(&graft->oid))
+                       return 0;
+       } else if (data->flags & SEEN_ONLY) {
                struct commit *c = lookup_commit(the_repository, &graft->oid);
                if (!c || !(c->object.flags & SEEN)) {
                        if (data->flags & VERBOSE)
@@@ -371,16 -374,23 +375,23 @@@ void advertise_shallow_grafts(int fd
  
  /*
   * mark_reachable_objects() should have been run prior to this and all
-  * reachable commits marked as "SEEN".
+  * reachable commits marked as "SEEN", except when quick_prune is non-zero,
+  * in which case lines are excised from the shallow file if they refer to
+  * commits that do not exist (any longer).
   */
- void prune_shallow(int show_only)
+ void prune_shallow(unsigned options)
  {
        struct lock_file shallow_lock = LOCK_INIT;
        struct strbuf sb = STRBUF_INIT;
+       unsigned flags = SEEN_ONLY;
        int fd;
  
-       if (show_only) {
-               write_shallow_commits_1(&sb, 0, NULL, SEEN_ONLY | VERBOSE);
+       if (options & PRUNE_QUICK)
+               flags |= QUICK;
+       if (options & PRUNE_SHOW_ONLY) {
+               flags |= VERBOSE;
+               write_shallow_commits_1(&sb, 0, NULL, flags);
                strbuf_release(&sb);
                return;
        }
                                       git_path_shallow(the_repository),
                                       LOCK_DIE_ON_ERROR);
        check_shallow_file_for_update(the_repository);
-       if (write_shallow_commits_1(&sb, 0, NULL, SEEN_ONLY)) {
+       if (write_shallow_commits_1(&sb, 0, NULL, flags)) {
                if (write_in_full(fd, sb.buf, sb.len) < 0)
                        die_errno("failed to write to %s",
                                  get_lock_file_path(&shallow_lock));