Merge branch 'bp/untracked-cache-noflush'
authorJunio C Hamano <gitster@pobox.com>
Thu, 8 Mar 2018 20:36:30 +0000 (12:36 -0800)
committerJunio C Hamano <gitster@pobox.com>
Thu, 8 Mar 2018 20:36:30 +0000 (12:36 -0800)
Writing out the index file when the only thing that changed in it
is the untracked cache information is often wasteful, and this has
been optimized out.

* bp/untracked-cache-noflush:
untracked cache: use git_env_bool() not getenv() for customization
dir.c: don't flag the index as dirty for changes to the untracked cache

1  2 
dir.c
t/t7063-status-untracked-cache.sh
diff --combined dir.c
index 6dd91be8185d4012e11495a9238636913fedeeef,d445d77e626e84a56d44c73f0cf4118fc4f19600..dedbf5d476f207e39c1b7853ec8c97553181e5fb
--- 1/dir.c
--- 2/dir.c
+++ b/dir.c
@@@ -231,10 -231,12 +231,10 @@@ int within_depth(const char *name, int 
   *     1 along with { data, size } of the (possibly augmented) buffer
   *       when successful.
   *
 - * Optionally updates the given sha1_stat with the given OID (when valid).
 + * Optionally updates the given oid_stat with the given OID (when valid).
   */
 -static int do_read_blob(const struct object_id *oid,
 -                      struct sha1_stat *sha1_stat,
 -                      size_t *size_out,
 -                      char **data_out)
 +static int do_read_blob(const struct object_id *oid, struct oid_stat *oid_stat,
 +                      size_t *size_out, char **data_out)
  {
        enum object_type type;
        unsigned long sz;
                return -1;
        }
  
 -      if (sha1_stat) {
 -              memset(&sha1_stat->stat, 0, sizeof(sha1_stat->stat));
 -              hashcpy(sha1_stat->sha1, oid->hash);
 +      if (oid_stat) {
 +              memset(&oid_stat->stat, 0, sizeof(oid_stat->stat));
 +              oidcpy(&oid_stat->oid, oid);
        }
  
        if (sz == 0) {
@@@ -652,8 -654,9 +652,8 @@@ void add_exclude(const char *string, co
  
  static int read_skip_worktree_file_from_index(const struct index_state *istate,
                                              const char *path,
 -                                            size_t *size_out,
 -                                            char **data_out,
 -                                            struct sha1_stat *sha1_stat)
 +                                            size_t *size_out, char **data_out,
 +                                            struct oid_stat *oid_stat)
  {
        int pos, len;
  
        if (!ce_skip_worktree(istate->cache[pos]))
                return -1;
  
 -      return do_read_blob(&istate->cache[pos]->oid, sha1_stat, size_out, data_out);
 +      return do_read_blob(&istate->cache[pos]->oid, oid_stat, size_out, data_out);
  }
  
  /*
@@@ -744,8 -747,8 +744,8 @@@ static struct untracked_cache_dir *look
        FLEX_ALLOC_MEM(d, name, name, len);
  
        ALLOC_GROW(dir->dirs, dir->dirs_nr + 1, dir->dirs_alloc);
 -      memmove(dir->dirs + first + 1, dir->dirs + first,
 -              (dir->dirs_nr - first) * sizeof(*dir->dirs));
 +      MOVE_ARRAY(dir->dirs + first + 1, dir->dirs + first,
 +                 dir->dirs_nr - first);
        dir->dirs_nr++;
        dir->dirs[first] = d;
        return d;
@@@ -771,16 -774,7 +771,16 @@@ static void invalidate_directory(struc
                                 struct untracked_cache_dir *dir)
  {
        int i;
 -      uc->dir_invalidated++;
 +
 +      /*
 +       * Invalidation increment here is just roughly correct. If
 +       * untracked_nr or any of dirs[].recurse is non-zero, we
 +       * should increment dir_invalidated too. But that's more
 +       * expensive to do.
 +       */
 +      if (dir->valid)
 +              uc->dir_invalidated++;
 +
        dir->valid = 0;
        dir->untracked_nr = 0;
        for (i = 0; i < dir->dirs_nr; i++)
@@@ -801,8 -795,9 +801,8 @@@ static int add_excludes_from_buffer(cha
   * ss_valid is non-zero, "ss" must contain good value as input.
   */
  static int add_excludes(const char *fname, const char *base, int baselen,
 -                      struct exclude_list *el,
 -                      struct index_state *istate,
 -                      struct sha1_stat *sha1_stat)
 +                      struct exclude_list *el, struct index_state *istate,
 +                      struct oid_stat *oid_stat)
  {
        struct stat st;
        int r;
                        return -1;
                r = read_skip_worktree_file_from_index(istate, fname,
                                                       &size, &buf,
 -                                                     sha1_stat);
 +                                                     oid_stat);
                if (r != 1)
                        return r;
        } else {
                size = xsize_t(st.st_size);
                if (size == 0) {
 -                      if (sha1_stat) {
 -                              fill_stat_data(&sha1_stat->stat, &st);
 -                              hashcpy(sha1_stat->sha1, EMPTY_BLOB_SHA1_BIN);
 -                              sha1_stat->valid = 1;
 +                      if (oid_stat) {
 +                              fill_stat_data(&oid_stat->stat, &st);
 +                              oidcpy(&oid_stat->oid, &empty_blob_oid);
 +                              oid_stat->valid = 1;
                        }
                        close(fd);
                        return 0;
                }
                buf[size++] = '\n';
                close(fd);
 -              if (sha1_stat) {
 +              if (oid_stat) {
                        int pos;
 -                      if (sha1_stat->valid &&
 -                          !match_stat_data_racy(istate, &sha1_stat->stat, &st))
 +                      if (oid_stat->valid &&
 +                          !match_stat_data_racy(istate, &oid_stat->stat, &st))
                                ; /* no content change, ss->sha1 still good */
                        else if (istate &&
                                 (pos = index_name_pos(istate, fname, strlen(fname))) >= 0 &&
                                 !ce_stage(istate->cache[pos]) &&
                                 ce_uptodate(istate->cache[pos]) &&
                                 !would_convert_to_git(istate, fname))
 -                              hashcpy(sha1_stat->sha1,
 -                                      istate->cache[pos]->oid.hash);
 +                              oidcpy(&oid_stat->oid,
 +                                     &istate->cache[pos]->oid);
                        else
 -                              hash_sha1_file(buf, size, "blob", sha1_stat->sha1);
 -                      fill_stat_data(&sha1_stat->stat, &st);
 -                      sha1_stat->valid = 1;
 +                              hash_object_file(buf, size, "blob",
 +                                               &oid_stat->oid);
 +                      fill_stat_data(&oid_stat->stat, &st);
 +                      oid_stat->valid = 1;
                }
        }
  
@@@ -936,7 -930,7 +936,7 @@@ struct exclude_list *add_exclude_list(s
   * Used to set up core.excludesfile and .git/info/exclude lists.
   */
  static void add_excludes_from_file_1(struct dir_struct *dir, const char *fname,
 -                                   struct sha1_stat *sha1_stat)
 +                                   struct oid_stat *oid_stat)
  {
        struct exclude_list *el;
        /*
        if (!dir->untracked)
                dir->unmanaged_exclude_files++;
        el = add_exclude_list(dir, EXC_FILE, fname);
 -      if (add_excludes(fname, "", 0, el, NULL, sha1_stat) < 0)
 +      if (add_excludes(fname, "", 0, el, NULL, oid_stat) < 0)
                die("cannot use %s as an exclude file", fname);
  }
  
@@@ -1186,7 -1180,7 +1186,7 @@@ static void prep_exclude(struct dir_str
  
        while (current < baselen) {
                const char *cp;
 -              struct sha1_stat sha1_stat;
 +              struct oid_stat oid_stat;
  
                stk = xcalloc(1, sizeof(*stk));
                if (current < 0) {
                }
  
                /* Try to read per-directory file */
 -              hashclr(sha1_stat.sha1);
 -              sha1_stat.valid = 0;
 +              oidclr(&oid_stat.oid);
 +              oid_stat.valid = 0;
                if (dir->exclude_per_dir &&
                    /*
                     * If we know that no files have been added in
                        strbuf_addstr(&sb, dir->exclude_per_dir);
                        el->src = strbuf_detach(&sb, NULL);
                        add_excludes(el->src, el->src, stk->baselen, el, istate,
 -                                   untracked ? &sha1_stat : NULL);
 +                                   untracked ? &oid_stat : NULL);
                }
                /*
                 * NEEDSWORK: when untracked cache is enabled, prep_exclude()
                 * order, though, if you do that.
                 */
                if (untracked &&
 -                  hashcmp(sha1_stat.sha1, untracked->exclude_sha1)) {
 +                  hashcmp(oid_stat.oid.hash, untracked->exclude_sha1)) {
                        invalidate_gitignore(dir->untracked, untracked);
 -                      hashcpy(untracked->exclude_sha1, sha1_stat.sha1);
 +                      hashcpy(untracked->exclude_sha1, oid_stat.oid.hash);
                }
                dir->exclude_stack = stk;
                current = stk->baselen;
@@@ -1779,7 -1773,7 +1779,7 @@@ static enum path_treatment treat_path(s
        if (!de)
                return treat_path_fast(dir, untracked, cdir, istate, path,
                                       baselen, pathspec);
 -      if (is_dot_or_dotdot(de->d_name) || !strcmp(de->d_name, ".git"))
 +      if (is_dot_or_dotdot(de->d_name) || !fspathcmp(de->d_name, ".git"))
                return path_none;
        strbuf_setlen(path, baselen);
        strbuf_addstr(path, de->d_name);
@@@ -1815,19 -1809,24 +1815,19 @@@ static int valid_cached_dir(struct dir_
         */
        refresh_fsmonitor(istate);
        if (!(dir->untracked->use_fsmonitor && untracked->valid)) {
 -              if (stat(path->len ? path->buf : ".", &st)) {
 -                      invalidate_directory(dir->untracked, untracked);
 +              if (lstat(path->len ? path->buf : ".", &st)) {
                        memset(&untracked->stat_data, 0, sizeof(untracked->stat_data));
                        return 0;
                }
                if (!untracked->valid ||
                        match_stat_data_racy(istate, &untracked->stat_data, &st)) {
 -                      if (untracked->valid)
 -                              invalidate_directory(dir->untracked, untracked);
                        fill_stat_data(&untracked->stat_data, &st);
                        return 0;
                }
        }
  
 -      if (untracked->check_only != !!check_only) {
 -              invalidate_directory(dir->untracked, untracked);
 +      if (untracked->check_only != !!check_only)
                return 0;
 -      }
  
        /*
         * prep_exclude will be called eventually on this directory,
@@@ -1854,20 -1853,13 +1854,20 @@@ static int open_cached_dir(struct cache
                           struct strbuf *path,
                           int check_only)
  {
 +      const char *c_path;
 +
        memset(cdir, 0, sizeof(*cdir));
        cdir->untracked = untracked;
        if (valid_cached_dir(dir, untracked, istate, path, check_only))
                return 0;
 -      cdir->fdir = opendir(path->len ? path->buf : ".");
 -      if (dir->untracked)
 +      c_path = path->len ? path->buf : ".";
 +      cdir->fdir = opendir(c_path);
 +      if (!cdir->fdir)
 +              warning_errno(_("could not open directory '%s'"), c_path);
 +      if (dir->untracked) {
 +              invalidate_directory(dir->untracked, untracked);
                dir->untracked->dir_opened++;
 +      }
        if (!cdir->fdir)
                return -1;
        return 0;
@@@ -2172,8 -2164,13 +2172,13 @@@ static struct untracked_cache_dir *vali
                                                      const struct pathspec *pathspec)
  {
        struct untracked_cache_dir *root;
+       static int untracked_cache_disabled = -1;
  
-       if (!dir->untracked || getenv("GIT_DISABLE_UNTRACKED_CACHE"))
+       if (!dir->untracked)
+               return NULL;
+       if (untracked_cache_disabled < 0)
+               untracked_cache_disabled = git_env_bool("GIT_DISABLE_UNTRACKED_CACHE", 0);
+       if (untracked_cache_disabled)
                return NULL;
  
        /*
  
        /* Validate $GIT_DIR/info/exclude and core.excludesfile */
        root = dir->untracked->root;
 -      if (hashcmp(dir->ss_info_exclude.sha1,
 -                  dir->untracked->ss_info_exclude.sha1)) {
 +      if (oidcmp(&dir->ss_info_exclude.oid,
 +                 &dir->untracked->ss_info_exclude.oid)) {
                invalidate_gitignore(dir->untracked, root);
                dir->untracked->ss_info_exclude = dir->ss_info_exclude;
        }
 -      if (hashcmp(dir->ss_excludes_file.sha1,
 -                  dir->untracked->ss_excludes_file.sha1)) {
 +      if (oidcmp(&dir->ss_excludes_file.oid,
 +                 &dir->untracked->ss_excludes_file.oid)) {
                invalidate_gitignore(dir->untracked, root);
                dir->untracked->ss_excludes_file = dir->ss_excludes_file;
        }
@@@ -2256,7 -2253,6 +2261,7 @@@ int read_directory(struct dir_struct *d
                   const char *path, int len, const struct pathspec *pathspec)
  {
        struct untracked_cache_dir *untracked;
 +      uint64_t start = getnanotime();
  
        if (has_symlink_leading_path(path, len))
                return dir->nr;
                dir->nr = i;
        }
  
 +      trace_performance_since(start, "read directory %.*s", len, path);
        if (dir->untracked) {
+               static int force_untracked_cache = -1;
                static struct trace_key trace_untracked_stats = TRACE_KEY_INIT(UNTRACKED_STATS);
+               if (force_untracked_cache < 0)
+                       force_untracked_cache =
+                               git_env_bool("GIT_FORCE_UNTRACKED_CACHE", 0);
                trace_printf_key(&trace_untracked_stats,
                                 "node creation: %u\n"
                                 "gitignore invalidation: %u\n"
                                 dir->untracked->gitignore_invalidated,
                                 dir->untracked->dir_invalidated,
                                 dir->untracked->dir_opened);
-               if (dir->untracked == istate->untracked &&
+               if (force_untracked_cache &&
+                       dir->untracked == istate->untracked &&
                    (dir->untracked->dir_opened ||
                     dir->untracked->gitignore_invalidated ||
                     dir->untracked->dir_invalidated))
@@@ -2648,8 -2649,8 +2659,8 @@@ void write_untracked_extension(struct s
        FLEX_ALLOC_MEM(ouc, exclude_per_dir, untracked->exclude_per_dir, len);
        stat_data_to_disk(&ouc->info_exclude_stat, &untracked->ss_info_exclude.stat);
        stat_data_to_disk(&ouc->excludes_file_stat, &untracked->ss_excludes_file.stat);
 -      hashcpy(ouc->info_exclude_sha1, untracked->ss_info_exclude.sha1);
 -      hashcpy(ouc->excludes_file_sha1, untracked->ss_excludes_file.sha1);
 +      hashcpy(ouc->info_exclude_sha1, untracked->ss_info_exclude.oid.hash);
 +      hashcpy(ouc->excludes_file_sha1, untracked->ss_excludes_file.oid.hash);
        ouc->dir_flags = htonl(untracked->dir_flags);
  
        varint_len = encode_varint(untracked->ident.len, varbuf);
@@@ -2826,12 -2827,13 +2837,12 @@@ static void read_sha1(size_t pos, void 
        rd->data += 20;
  }
  
 -static void load_sha1_stat(struct sha1_stat *sha1_stat,
 -                         const unsigned char *data,
 -                         const unsigned char *sha1)
 +static void load_oid_stat(struct oid_stat *oid_stat, const unsigned char *data,
 +                        const unsigned char *sha1)
  {
 -      stat_data_from_disk(&sha1_stat->stat, data);
 -      hashcpy(sha1_stat->sha1, sha1);
 -      sha1_stat->valid = 1;
 +      stat_data_from_disk(&oid_stat->stat, data);
 +      hashcpy(oid_stat->oid.hash, sha1);
 +      oid_stat->valid = 1;
  }
  
  struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz)
        uc = xcalloc(1, sizeof(*uc));
        strbuf_init(&uc->ident, ident_len);
        strbuf_add(&uc->ident, ident, ident_len);
 -      load_sha1_stat(&uc->ss_info_exclude,
 -                     next + ouc_offset(info_exclude_stat),
 -                     next + ouc_offset(info_exclude_sha1));
 -      load_sha1_stat(&uc->ss_excludes_file,
 -                     next + ouc_offset(excludes_file_stat),
 -                     next + ouc_offset(excludes_file_sha1));
 +      load_oid_stat(&uc->ss_info_exclude,
 +                    next + ouc_offset(info_exclude_stat),
 +                    next + ouc_offset(info_exclude_sha1));
 +      load_oid_stat(&uc->ss_excludes_file,
 +                    next + ouc_offset(excludes_file_stat),
 +                    next + ouc_offset(excludes_file_sha1));
        uc->dir_flags = get_be32(next + ouc_offset(dir_flags));
        exclude_per_dir = (const char *)next + ouc_offset(exclude_per_dir);
        uc->exclude_per_dir = xstrdup(exclude_per_dir);
@@@ -2977,12 -2979,10 +2988,12 @@@ static int invalidate_one_component(str
  }
  
  void untracked_cache_invalidate_path(struct index_state *istate,
 -                                   const char *path)
 +                                   const char *path, int safe_path)
  {
        if (!istate->untracked || !istate->untracked->root)
                return;
 +      if (!safe_path && !verify_path(path))
 +              return;
        invalidate_one_component(istate->untracked, istate->untracked->root,
                                 path, strlen(path));
  }
  void untracked_cache_remove_from_index(struct index_state *istate,
                                       const char *path)
  {
 -      untracked_cache_invalidate_path(istate, path);
 +      untracked_cache_invalidate_path(istate, path, 1);
  }
  
  void untracked_cache_add_to_index(struct index_state *istate,
                                  const char *path)
  {
 -      untracked_cache_invalidate_path(istate, path);
 +      untracked_cache_invalidate_path(istate, path, 1);
  }
  
  /* Update gitfile and core.worktree setting to connect work tree and git dir */
index 46b947824fd68150cfaa6018355cd086c5a8759e,9cb16ca36dae7b1d188318e741fdd538e575ba4f..c61e304e97376b09de51299a797dfb388335a365
@@@ -14,6 -14,9 +14,9 @@@ test_description='test untracked cache
  # See <20160803174522.5571-1-pclouds@gmail.com> if you want to know
  # more.
  
+ GIT_FORCE_UNTRACKED_CACHE=true
+ export GIT_FORCE_UNTRACKED_CACHE
  sync_mtime () {
        find . -type d -ls >/dev/null
  }
@@@ -22,12 -25,6 +25,12 @@@ avoid_racy() 
        sleep 1
  }
  
 +status_is_clean() {
 +      >../status.expect &&
 +      git status --porcelain >../status.actual &&
 +      test_cmp ../status.expect ../status.actual
 +}
 +
  test_lazy_prereq UNTRACKED_CACHE '
        { git update-index --test-untracked-cache; ret=$?; } &&
        test $ret -ne 1
@@@ -689,85 -686,4 +692,85 @@@ test_expect_success 'untracked cache su
        test_cmp ../before ../after
  '
  
 +test_expect_success 'teardown worktree' '
 +      cd ..
 +'
 +
 +test_expect_success SYMLINKS 'setup worktree for symlink test' '
 +      git init worktree-symlink &&
 +      cd worktree-symlink &&
 +      git config core.untrackedCache true &&
 +      mkdir one two &&
 +      touch one/file two/file &&
 +      git add one/file two/file &&
 +      git commit -m"first commit" &&
 +      git rm -rf one &&
 +      ln -s two one &&
 +      git add one &&
 +      git commit -m"second commit"
 +'
 +
 +test_expect_success SYMLINKS '"status" after symlink replacement should be clean with UC=true' '
 +      git checkout HEAD~ &&
 +      status_is_clean &&
 +      status_is_clean &&
 +      git checkout master &&
 +      avoid_racy &&
 +      status_is_clean &&
 +      status_is_clean
 +'
 +
 +test_expect_success SYMLINKS '"status" after symlink replacement should be clean with UC=false' '
 +      git config core.untrackedCache false &&
 +      git checkout HEAD~ &&
 +      status_is_clean &&
 +      status_is_clean &&
 +      git checkout master &&
 +      avoid_racy &&
 +      status_is_clean &&
 +      status_is_clean
 +'
 +
 +test_expect_success 'setup worktree for non-symlink test' '
 +      git init worktree-non-symlink &&
 +      cd worktree-non-symlink &&
 +      git config core.untrackedCache true &&
 +      mkdir one two &&
 +      touch one/file two/file &&
 +      git add one/file two/file &&
 +      git commit -m"first commit" &&
 +      git rm -rf one &&
 +      cp two/file one &&
 +      git add one &&
 +      git commit -m"second commit"
 +'
 +
 +test_expect_success '"status" after file replacement should be clean with UC=true' '
 +      git checkout HEAD~ &&
 +      status_is_clean &&
 +      status_is_clean &&
 +      git checkout master &&
 +      avoid_racy &&
 +      status_is_clean &&
 +      test-dump-untracked-cache >../actual &&
 +      grep -F "recurse valid" ../actual >../actual.grep &&
 +      cat >../expect.grep <<EOF &&
 +/ 0000000000000000000000000000000000000000 recurse valid
 +/two/ 0000000000000000000000000000000000000000 recurse valid
 +EOF
 +      status_is_clean &&
 +      test_cmp ../expect.grep ../actual.grep
 +'
 +
 +test_expect_success '"status" after file replacement should be clean with UC=false' '
 +      git config core.untrackedCache false &&
 +      git checkout HEAD~ &&
 +      status_is_clean &&
 +      status_is_clean &&
 +      git checkout master &&
 +      avoid_racy &&
 +      status_is_clean &&
 +      status_is_clean
 +'
 +
  test_done