t5526: test recursive submodules when fetching moved submodules
[gitweb.git] / dir.c
diff --git a/dir.c b/dir.c
index 9987011da57bcf171ba645788b8ccd2151561e5c..ccf8b4975e8645182da783b2367e31401a4cdadb 100644 (file)
--- a/dir.c
+++ b/dir.c
@@ -18,6 +18,8 @@
 #include "utf8.h"
 #include "varint.h"
 #include "ewah/ewok.h"
+#include "fsmonitor.h"
+#include "submodule-config.h"
 
 /*
  * Tells read_directory_recursive how a file or directory should be treated.
@@ -220,6 +222,55 @@ int within_depth(const char *name, int namelen,
        return 1;
 }
 
+/*
+ * Read the contents of the blob with the given OID into a buffer.
+ * Append a trailing LF to the end if the last line doesn't have one.
+ *
+ * Returns:
+ *    -1 when the OID is invalid or unknown or does not refer to a blob.
+ *     0 when the blob is empty.
+ *     1 along with { data, size } of the (possibly augmented) buffer
+ *       when successful.
+ *
+ * Optionally updates the given oid_stat with the given OID (when valid).
+ */
+static int do_read_blob(const struct object_id *oid, struct oid_stat *oid_stat,
+                       size_t *size_out, char **data_out)
+{
+       enum object_type type;
+       unsigned long sz;
+       char *data;
+
+       *size_out = 0;
+       *data_out = NULL;
+
+       data = read_object_file(oid, &type, &sz);
+       if (!data || type != OBJ_BLOB) {
+               free(data);
+               return -1;
+       }
+
+       if (oid_stat) {
+               memset(&oid_stat->stat, 0, sizeof(oid_stat->stat));
+               oidcpy(&oid_stat->oid, oid);
+       }
+
+       if (sz == 0) {
+               free(data);
+               return 0;
+       }
+
+       if (data[sz - 1] != '\n') {
+               data = xrealloc(data, st_add(sz, 1));
+               data[sz++] = '\n';
+       }
+
+       *size_out = xsize_t(sz);
+       *data_out = data;
+
+       return 1;
+}
+
 #define DO_MATCH_EXCLUDE   (1<<0)
 #define DO_MATCH_DIRECTORY (1<<1)
 #define DO_MATCH_SUBMODULE (1<<2)
@@ -600,32 +651,21 @@ void add_exclude(const char *string, const char *base,
        x->el = el;
 }
 
-static void *read_skip_worktree_file_from_index(const struct index_state *istate,
-                                               const char *path, size_t *size,
-                                               struct sha1_stat *sha1_stat)
+static int read_skip_worktree_file_from_index(const struct index_state *istate,
+                                             const char *path,
+                                             size_t *size_out, char **data_out,
+                                             struct oid_stat *oid_stat)
 {
        int pos, len;
-       unsigned long sz;
-       enum object_type type;
-       void *data;
 
        len = strlen(path);
        pos = index_name_pos(istate, path, len);
        if (pos < 0)
-               return NULL;
+               return -1;
        if (!ce_skip_worktree(istate->cache[pos]))
-               return NULL;
-       data = read_sha1_file(istate->cache[pos]->oid.hash, &type, &sz);
-       if (!data || type != OBJ_BLOB) {
-               free(data);
-               return NULL;
-       }
-       *size = xsize_t(sz);
-       if (sha1_stat) {
-               memset(&sha1_stat->stat, 0, sizeof(sha1_stat->stat));
-               hashcpy(sha1_stat->sha1, istate->cache[pos]->oid.hash);
-       }
-       return data;
+               return -1;
+
+       return do_read_blob(&istate->cache[pos]->oid, oid_stat, size_out, data_out);
 }
 
 /*
@@ -705,8 +745,8 @@ static struct untracked_cache_dir *lookup_untracked(struct untracked_cache *uc,
        FLEX_ALLOC_MEM(d, name, name, len);
 
        ALLOC_GROW(dir->dirs, dir->dirs_nr + 1, dir->dirs_alloc);
-       memmove(dir->dirs + first + 1, dir->dirs + first,
-               (dir->dirs_nr - first) * sizeof(*dir->dirs));
+       MOVE_ARRAY(dir->dirs + first + 1, dir->dirs + first,
+                  dir->dirs_nr - first);
        dir->dirs_nr++;
        dir->dirs[first] = d;
        return d;
@@ -732,13 +772,26 @@ static void invalidate_directory(struct untracked_cache *uc,
                                 struct untracked_cache_dir *dir)
 {
        int i;
-       uc->dir_invalidated++;
+
+       /*
+        * Invalidation increment here is just roughly correct. If
+        * untracked_nr or any of dirs[].recurse is non-zero, we
+        * should increment dir_invalidated too. But that's more
+        * expensive to do.
+        */
+       if (dir->valid)
+               uc->dir_invalidated++;
+
        dir->valid = 0;
        dir->untracked_nr = 0;
        for (i = 0; i < dir->dirs_nr; i++)
                dir->dirs[i]->recurse = 0;
 }
 
+static int add_excludes_from_buffer(char *buf, size_t size,
+                                   const char *base, int baselen,
+                                   struct exclude_list *el);
+
 /*
  * Given a file with name "fname", read it (either from disk, or from
  * an index if 'istate' is non-null), parse it and store the
@@ -749,14 +802,14 @@ static void invalidate_directory(struct untracked_cache *uc,
  * ss_valid is non-zero, "ss" must contain good value as input.
  */
 static int add_excludes(const char *fname, const char *base, int baselen,
-                       struct exclude_list *el,
-                       struct index_state *istate,
-                       struct sha1_stat *sha1_stat)
+                       struct exclude_list *el, struct index_state *istate,
+                       struct oid_stat *oid_stat)
 {
        struct stat st;
-       int fd, i, lineno = 1;
+       int r;
+       int fd;
        size_t size = 0;
-       char *buf, *entry;
+       char *buf;
 
        fd = open(fname, O_RDONLY);
        if (fd < 0 || fstat(fd, &st) < 0) {
@@ -764,24 +817,20 @@ static int add_excludes(const char *fname, const char *base, int baselen,
                        warn_on_fopen_errors(fname);
                else
                        close(fd);
-               if (!istate ||
-                   (buf = read_skip_worktree_file_from_index(istate, fname, &size, sha1_stat)) == NULL)
+               if (!istate)
                        return -1;
-               if (size == 0) {
-                       free(buf);
-                       return 0;
-               }
-               if (buf[size-1] != '\n') {
-                       buf = xrealloc(buf, st_add(size, 1));
-                       buf[size++] = '\n';
-               }
+               r = read_skip_worktree_file_from_index(istate, fname,
+                                                      &size, &buf,
+                                                      oid_stat);
+               if (r != 1)
+                       return r;
        } else {
                size = xsize_t(st.st_size);
                if (size == 0) {
-                       if (sha1_stat) {
-                               fill_stat_data(&sha1_stat->stat, &st);
-                               hashcpy(sha1_stat->sha1, EMPTY_BLOB_SHA1_BIN);
-                               sha1_stat->valid = 1;
+                       if (oid_stat) {
+                               fill_stat_data(&oid_stat->stat, &st);
+                               oidcpy(&oid_stat->oid, the_hash_algo->empty_blob);
+                               oid_stat->valid = 1;
                        }
                        close(fd);
                        return 0;
@@ -794,25 +843,37 @@ static int add_excludes(const char *fname, const char *base, int baselen,
                }
                buf[size++] = '\n';
                close(fd);
-               if (sha1_stat) {
+               if (oid_stat) {
                        int pos;
-                       if (sha1_stat->valid &&
-                           !match_stat_data_racy(istate, &sha1_stat->stat, &st))
+                       if (oid_stat->valid &&
+                           !match_stat_data_racy(istate, &oid_stat->stat, &st))
                                ; /* no content change, ss->sha1 still good */
                        else if (istate &&
                                 (pos = index_name_pos(istate, fname, strlen(fname))) >= 0 &&
                                 !ce_stage(istate->cache[pos]) &&
                                 ce_uptodate(istate->cache[pos]) &&
                                 !would_convert_to_git(istate, fname))
-                               hashcpy(sha1_stat->sha1,
-                                       istate->cache[pos]->oid.hash);
+                               oidcpy(&oid_stat->oid,
+                                      &istate->cache[pos]->oid);
                        else
-                               hash_sha1_file(buf, size, "blob", sha1_stat->sha1);
-                       fill_stat_data(&sha1_stat->stat, &st);
-                       sha1_stat->valid = 1;
+                               hash_object_file(buf, size, "blob",
+                                                &oid_stat->oid);
+                       fill_stat_data(&oid_stat->stat, &st);
+                       oid_stat->valid = 1;
                }
        }
 
+       add_excludes_from_buffer(buf, size, base, baselen, el);
+       return 0;
+}
+
+static int add_excludes_from_buffer(char *buf, size_t size,
+                                   const char *base, int baselen,
+                                   struct exclude_list *el)
+{
+       int i, lineno = 1;
+       char *entry;
+
        el->filebuf = buf;
 
        if (skip_utf8_bom(&buf, size))
@@ -841,6 +902,23 @@ int add_excludes_from_file_to_list(const char *fname, const char *base,
        return add_excludes(fname, base, baselen, el, istate, NULL);
 }
 
+int add_excludes_from_blob_to_list(
+       struct object_id *oid,
+       const char *base, int baselen,
+       struct exclude_list *el)
+{
+       char *buf;
+       size_t size;
+       int r;
+
+       r = do_read_blob(oid, NULL, &size, &buf);
+       if (r != 1)
+               return r;
+
+       add_excludes_from_buffer(buf, size, base, baselen, el);
+       return 0;
+}
+
 struct exclude_list *add_exclude_list(struct dir_struct *dir,
                                      int group_type, const char *src)
 {
@@ -859,7 +937,7 @@ struct exclude_list *add_exclude_list(struct dir_struct *dir,
  * Used to set up core.excludesfile and .git/info/exclude lists.
  */
 static void add_excludes_from_file_1(struct dir_struct *dir, const char *fname,
-                                    struct sha1_stat *sha1_stat)
+                                    struct oid_stat *oid_stat)
 {
        struct exclude_list *el;
        /*
@@ -870,7 +948,7 @@ static void add_excludes_from_file_1(struct dir_struct *dir, const char *fname,
        if (!dir->untracked)
                dir->unmanaged_exclude_files++;
        el = add_exclude_list(dir, EXC_FILE, fname);
-       if (add_excludes(fname, "", 0, el, NULL, sha1_stat) < 0)
+       if (add_excludes(fname, "", 0, el, NULL, oid_stat) < 0)
                die("cannot use %s as an exclude file", fname);
 }
 
@@ -1109,7 +1187,7 @@ static void prep_exclude(struct dir_struct *dir,
 
        while (current < baselen) {
                const char *cp;
-               struct sha1_stat sha1_stat;
+               struct oid_stat oid_stat;
 
                stk = xcalloc(1, sizeof(*stk));
                if (current < 0) {
@@ -1152,8 +1230,8 @@ static void prep_exclude(struct dir_struct *dir,
                }
 
                /* Try to read per-directory file */
-               hashclr(sha1_stat.sha1);
-               sha1_stat.valid = 0;
+               oidclr(&oid_stat.oid);
+               oid_stat.valid = 0;
                if (dir->exclude_per_dir &&
                    /*
                     * If we know that no files have been added in
@@ -1163,11 +1241,11 @@ static void prep_exclude(struct dir_struct *dir,
                    (!untracked || !untracked->valid ||
                     /*
                      * .. and .gitignore does not exist before
-                     * (i.e. null exclude_sha1). Then we can skip
+                     * (i.e. null exclude_oid). Then we can skip
                      * loading .gitignore, which would result in
                      * ENOENT anyway.
                      */
-                    !is_null_sha1(untracked->exclude_sha1))) {
+                    !is_null_oid(&untracked->exclude_oid))) {
                        /*
                         * dir->basebuf gets reused by the traversal, but we
                         * need fname to remain unchanged to ensure the src
@@ -1181,7 +1259,7 @@ static void prep_exclude(struct dir_struct *dir,
                        strbuf_addstr(&sb, dir->exclude_per_dir);
                        el->src = strbuf_detach(&sb, NULL);
                        add_excludes(el->src, el->src, stk->baselen, el, istate,
-                                    untracked ? &sha1_stat : NULL);
+                                    untracked ? &oid_stat : NULL);
                }
                /*
                 * NEEDSWORK: when untracked cache is enabled, prep_exclude()
@@ -1198,9 +1276,9 @@ static void prep_exclude(struct dir_struct *dir,
                 * order, though, if you do that.
                 */
                if (untracked &&
-                   hashcmp(sha1_stat.sha1, untracked->exclude_sha1)) {
+                   oidcmp(&oid_stat.oid, &untracked->exclude_oid)) {
                        invalidate_gitignore(dir->untracked, untracked);
-                       hashcpy(untracked->exclude_sha1, sha1_stat.sha1);
+                       oidcpy(&untracked->exclude_oid, &oid_stat.oid);
                }
                dir->exclude_stack = stk;
                current = stk->baselen;
@@ -1389,9 +1467,33 @@ static enum path_treatment treat_directory(struct dir_struct *dir,
        case index_nonexistent:
                if (dir->flags & DIR_SHOW_OTHER_DIRECTORIES)
                        break;
+               if (exclude &&
+                       (dir->flags & DIR_SHOW_IGNORED_TOO) &&
+                       (dir->flags & DIR_SHOW_IGNORED_TOO_MODE_MATCHING)) {
+
+                       /*
+                        * This is an excluded directory and we are
+                        * showing ignored paths that match an exclude
+                        * pattern.  (e.g. show directory as ignored
+                        * only if it matches an exclude pattern).
+                        * This path will either be 'path_excluded`
+                        * (if we are showing empty directories or if
+                        * the directory is not empty), or will be
+                        * 'path_none' (empty directory, and we are
+                        * not showing empty directories).
+                        */
+                       if (!(dir->flags & DIR_HIDE_EMPTY_DIRECTORIES))
+                               return path_excluded;
+
+                       if (read_directory_recursive(dir, istate, dirname, len,
+                                                    untracked, 1, 1, pathspec) == path_excluded)
+                               return path_excluded;
+
+                       return path_none;
+               }
                if (!(dir->flags & DIR_NO_GITLINKS)) {
-                       unsigned char sha1[20];
-                       if (resolve_gitlink_ref(dirname, "HEAD", sha1) == 0)
+                       struct object_id oid;
+                       if (resolve_gitlink_ref(dirname, "HEAD", &oid) == 0)
                                return exclude ? path_excluded : path_untracked;
                }
                return path_recurse;
@@ -1561,6 +1663,7 @@ static enum path_treatment treat_one_path(struct dir_struct *dir,
 {
        int exclude;
        int has_path_in_index = !!index_file_exists(istate, path->buf, path->len, ignore_case);
+       enum path_treatment path_treatment;
 
        if (dtype == DT_UNKNOWN)
                dtype = get_dtype(de, istate, path->buf, path->len);
@@ -1607,8 +1710,23 @@ static enum path_treatment treat_one_path(struct dir_struct *dir,
                return path_none;
        case DT_DIR:
                strbuf_addch(path, '/');
-               return treat_directory(dir, istate, untracked, path->buf, path->len,
-                                      baselen, exclude, pathspec);
+               path_treatment = treat_directory(dir, istate, untracked,
+                                                path->buf, path->len,
+                                                baselen, exclude, pathspec);
+               /*
+                * If 1) we only want to return directories that
+                * match an exclude pattern and 2) this directory does
+                * not match an exclude pattern but all of its
+                * contents are excluded, then indicate that we should
+                * recurse into this directory (instead of marking the
+                * directory itself as an ignored path).
+                */
+               if (!exclude &&
+                   path_treatment == path_excluded &&
+                   (dir->flags & DIR_SHOW_IGNORED_TOO) &&
+                   (dir->flags & DIR_SHOW_IGNORED_TOO_MODE_MATCHING))
+                       return path_recurse;
+               return path_treatment;
        case DT_REG:
        case DT_LNK:
                return exclude ? path_excluded : path_untracked;
@@ -1662,7 +1780,7 @@ static enum path_treatment treat_path(struct dir_struct *dir,
        if (!de)
                return treat_path_fast(dir, untracked, cdir, istate, path,
                                       baselen, pathspec);
-       if (is_dot_or_dotdot(de->d_name) || !strcmp(de->d_name, ".git"))
+       if (is_dot_or_dotdot(de->d_name) || !fspathcmp(de->d_name, ".git"))
                return path_none;
        strbuf_setlen(path, baselen);
        strbuf_addstr(path, de->d_name);
@@ -1693,23 +1811,24 @@ static int valid_cached_dir(struct dir_struct *dir,
        if (!untracked)
                return 0;
 
-       if (stat(path->len ? path->buf : ".", &st)) {
-               invalidate_directory(dir->untracked, untracked);
-               memset(&untracked->stat_data, 0, sizeof(untracked->stat_data));
-               return 0;
-       }
-       if (!untracked->valid ||
-           match_stat_data_racy(istate, &untracked->stat_data, &st)) {
-               if (untracked->valid)
-                       invalidate_directory(dir->untracked, untracked);
-               fill_stat_data(&untracked->stat_data, &st);
-               return 0;
+       /*
+        * With fsmonitor, we can trust the untracked cache's valid field.
+        */
+       refresh_fsmonitor(istate);
+       if (!(dir->untracked->use_fsmonitor && untracked->valid)) {
+               if (lstat(path->len ? path->buf : ".", &st)) {
+                       memset(&untracked->stat_data, 0, sizeof(untracked->stat_data));
+                       return 0;
+               }
+               if (!untracked->valid ||
+                       match_stat_data_racy(istate, &untracked->stat_data, &st)) {
+                       fill_stat_data(&untracked->stat_data, &st);
+                       return 0;
+               }
        }
 
-       if (untracked->check_only != !!check_only) {
-               invalidate_directory(dir->untracked, untracked);
+       if (untracked->check_only != !!check_only)
                return 0;
-       }
 
        /*
         * prep_exclude will be called eventually on this directory,
@@ -1736,13 +1855,20 @@ static int open_cached_dir(struct cached_dir *cdir,
                           struct strbuf *path,
                           int check_only)
 {
+       const char *c_path;
+
        memset(cdir, 0, sizeof(*cdir));
        cdir->untracked = untracked;
        if (valid_cached_dir(dir, untracked, istate, path, check_only))
                return 0;
-       cdir->fdir = opendir(path->len ? path->buf : ".");
-       if (dir->untracked)
+       c_path = path->len ? path->buf : ".";
+       cdir->fdir = opendir(c_path);
+       if (!cdir->fdir)
+               warning_errno(_("could not open directory '%s'"), c_path);
+       if (dir->untracked) {
+               invalidate_directory(dir->untracked, untracked);
                dir->untracked->dir_opened++;
+       }
        if (!cdir->fdir)
                return -1;
        return 0;
@@ -2047,8 +2173,13 @@ static struct untracked_cache_dir *validate_untracked_cache(struct dir_struct *d
                                                      const struct pathspec *pathspec)
 {
        struct untracked_cache_dir *root;
+       static int untracked_cache_disabled = -1;
 
-       if (!dir->untracked || getenv("GIT_DISABLE_UNTRACKED_CACHE"))
+       if (!dir->untracked)
+               return NULL;
+       if (untracked_cache_disabled < 0)
+               untracked_cache_disabled = git_env_bool("GIT_DISABLE_UNTRACKED_CACHE", 0);
+       if (untracked_cache_disabled)
                return NULL;
 
        /*
@@ -2111,13 +2242,13 @@ static struct untracked_cache_dir *validate_untracked_cache(struct dir_struct *d
 
        /* Validate $GIT_DIR/info/exclude and core.excludesfile */
        root = dir->untracked->root;
-       if (hashcmp(dir->ss_info_exclude.sha1,
-                   dir->untracked->ss_info_exclude.sha1)) {
+       if (oidcmp(&dir->ss_info_exclude.oid,
+                  &dir->untracked->ss_info_exclude.oid)) {
                invalidate_gitignore(dir->untracked, root);
                dir->untracked->ss_info_exclude = dir->ss_info_exclude;
        }
-       if (hashcmp(dir->ss_excludes_file.sha1,
-                   dir->untracked->ss_excludes_file.sha1)) {
+       if (oidcmp(&dir->ss_excludes_file.oid,
+                  &dir->untracked->ss_excludes_file.oid)) {
                invalidate_gitignore(dir->untracked, root);
                dir->untracked->ss_excludes_file = dir->ss_excludes_file;
        }
@@ -2131,6 +2262,7 @@ int read_directory(struct dir_struct *dir, struct index_state *istate,
                   const char *path, int len, const struct pathspec *pathspec)
 {
        struct untracked_cache_dir *untracked;
+       uint64_t start = getnanotime();
 
        if (has_symlink_leading_path(path, len))
                return dir->nr;
@@ -2169,8 +2301,14 @@ int read_directory(struct dir_struct *dir, struct index_state *istate,
                dir->nr = i;
        }
 
+       trace_performance_since(start, "read directory %.*s", len, path);
        if (dir->untracked) {
+               static int force_untracked_cache = -1;
                static struct trace_key trace_untracked_stats = TRACE_KEY_INIT(UNTRACKED_STATS);
+
+               if (force_untracked_cache < 0)
+                       force_untracked_cache =
+                               git_env_bool("GIT_FORCE_UNTRACKED_CACHE", 0);
                trace_printf_key(&trace_untracked_stats,
                                 "node creation: %u\n"
                                 "gitignore invalidation: %u\n"
@@ -2180,7 +2318,8 @@ int read_directory(struct dir_struct *dir, struct index_state *istate,
                                 dir->untracked->gitignore_invalidated,
                                 dir->untracked->dir_invalidated,
                                 dir->untracked->dir_opened);
-               if (dir->untracked == istate->untracked &&
+               if (force_untracked_cache &&
+                       dir->untracked == istate->untracked &&
                    (dir->untracked->dir_opened ||
                     dir->untracked->gitignore_invalidated ||
                     dir->untracked->dir_invalidated))
@@ -2279,10 +2418,10 @@ static int remove_dir_recurse(struct strbuf *path, int flag, int *kept_up)
        int ret = 0, original_len = path->len, len, kept_down = 0;
        int only_empty = (flag & REMOVE_DIR_EMPTY_ONLY);
        int keep_toplevel = (flag & REMOVE_DIR_KEEP_TOPLEVEL);
-       unsigned char submodule_head[20];
+       struct object_id submodule_head;
 
        if ((flag & REMOVE_DIR_KEEP_NESTED_GIT) &&
-           !resolve_gitlink_ref(path->buf, "HEAD", submodule_head)) {
+           !resolve_gitlink_ref(path->buf, "HEAD", &submodule_head)) {
                /* Do not descend and nuke a nested git work tree. */
                if (kept_up)
                        *kept_up = 1;
@@ -2484,9 +2623,10 @@ static void write_one_dir(struct untracked_cache_dir *untracked,
                stat_data_to_disk(&stat_data, &untracked->stat_data);
                strbuf_add(&wd->sb_stat, &stat_data, sizeof(stat_data));
        }
-       if (!is_null_sha1(untracked->exclude_sha1)) {
+       if (!is_null_oid(&untracked->exclude_oid)) {
                ewah_set(wd->sha1_valid, i);
-               strbuf_add(&wd->sb_sha1, untracked->exclude_sha1, 20);
+               strbuf_add(&wd->sb_sha1, untracked->exclude_oid.hash,
+                          the_hash_algo->rawsz);
        }
 
        intlen = encode_varint(untracked->untracked_nr, intbuf);
@@ -2521,8 +2661,8 @@ void write_untracked_extension(struct strbuf *out, struct untracked_cache *untra
        FLEX_ALLOC_MEM(ouc, exclude_per_dir, untracked->exclude_per_dir, len);
        stat_data_to_disk(&ouc->info_exclude_stat, &untracked->ss_info_exclude.stat);
        stat_data_to_disk(&ouc->excludes_file_stat, &untracked->ss_excludes_file.stat);
-       hashcpy(ouc->info_exclude_sha1, untracked->ss_info_exclude.sha1);
-       hashcpy(ouc->excludes_file_sha1, untracked->ss_excludes_file.sha1);
+       hashcpy(ouc->info_exclude_sha1, untracked->ss_info_exclude.oid.hash);
+       hashcpy(ouc->excludes_file_sha1, untracked->ss_excludes_file.oid.hash);
        ouc->dir_flags = htonl(untracked->dir_flags);
 
        varint_len = encode_varint(untracked->ident.len, varbuf);
@@ -2687,25 +2827,24 @@ static void read_stat(size_t pos, void *cb)
        ud->valid = 1;
 }
 
-static void read_sha1(size_t pos, void *cb)
+static void read_oid(size_t pos, void *cb)
 {
        struct read_data *rd = cb;
        struct untracked_cache_dir *ud = rd->ucd[pos];
-       if (rd->data + 20 > rd->end) {
+       if (rd->data + the_hash_algo->rawsz > rd->end) {
                rd->data = rd->end + 1;
                return;
        }
-       hashcpy(ud->exclude_sha1, rd->data);
-       rd->data += 20;
+       hashcpy(ud->exclude_oid.hash, rd->data);
+       rd->data += the_hash_algo->rawsz;
 }
 
-static void load_sha1_stat(struct sha1_stat *sha1_stat,
-                          const unsigned char *data,
-                          const unsigned char *sha1)
+static void load_oid_stat(struct oid_stat *oid_stat, const unsigned char *data,
+                         const unsigned char *sha1)
 {
-       stat_data_from_disk(&sha1_stat->stat, data);
-       hashcpy(sha1_stat->sha1, sha1);
-       sha1_stat->valid = 1;
+       stat_data_from_disk(&oid_stat->stat, data);
+       hashcpy(oid_stat->oid.hash, sha1);
+       oid_stat->valid = 1;
 }
 
 struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz)
@@ -2733,12 +2872,12 @@ struct untracked_cache *read_untracked_extension(const void *data, unsigned long
        uc = xcalloc(1, sizeof(*uc));
        strbuf_init(&uc->ident, ident_len);
        strbuf_add(&uc->ident, ident, ident_len);
-       load_sha1_stat(&uc->ss_info_exclude,
-                      next + ouc_offset(info_exclude_stat),
-                      next + ouc_offset(info_exclude_sha1));
-       load_sha1_stat(&uc->ss_excludes_file,
-                      next + ouc_offset(excludes_file_stat),
-                      next + ouc_offset(excludes_file_sha1));
+       load_oid_stat(&uc->ss_info_exclude,
+                     next + ouc_offset(info_exclude_stat),
+                     next + ouc_offset(info_exclude_sha1));
+       load_oid_stat(&uc->ss_excludes_file,
+                     next + ouc_offset(excludes_file_stat),
+                     next + ouc_offset(excludes_file_sha1));
        uc->dir_flags = get_be32(next + ouc_offset(dir_flags));
        exclude_per_dir = (const char *)next + ouc_offset(exclude_per_dir);
        uc->exclude_per_dir = xstrdup(exclude_per_dir);
@@ -2780,7 +2919,7 @@ struct untracked_cache *read_untracked_extension(const void *data, unsigned long
        ewah_each_bit(rd.check_only, set_check_only, &rd);
        rd.data = next + len;
        ewah_each_bit(rd.valid, read_stat, &rd);
-       ewah_each_bit(rd.sha1_valid, read_sha1, &rd);
+       ewah_each_bit(rd.sha1_valid, read_oid, &rd);
        next = rd.data;
 
 done:
@@ -2851,10 +2990,12 @@ static int invalidate_one_component(struct untracked_cache *uc,
 }
 
 void untracked_cache_invalidate_path(struct index_state *istate,
-                                    const char *path)
+                                    const char *path, int safe_path)
 {
        if (!istate->untracked || !istate->untracked->root)
                return;
+       if (!safe_path && !verify_path(path, 0))
+               return;
        invalidate_one_component(istate->untracked, istate->untracked->root,
                                 path, strlen(path));
 }
@@ -2862,17 +3003,66 @@ void untracked_cache_invalidate_path(struct index_state *istate,
 void untracked_cache_remove_from_index(struct index_state *istate,
                                       const char *path)
 {
-       untracked_cache_invalidate_path(istate, path);
+       untracked_cache_invalidate_path(istate, path, 1);
 }
 
 void untracked_cache_add_to_index(struct index_state *istate,
                                  const char *path)
 {
-       untracked_cache_invalidate_path(istate, path);
+       untracked_cache_invalidate_path(istate, path, 1);
+}
+
+static void connect_wt_gitdir_in_nested(const char *sub_worktree,
+                                       const char *sub_gitdir)
+{
+       int i;
+       struct repository subrepo;
+       struct strbuf sub_wt = STRBUF_INIT;
+       struct strbuf sub_gd = STRBUF_INIT;
+
+       const struct submodule *sub;
+
+       /* If the submodule has no working tree, we can ignore it. */
+       if (repo_init(&subrepo, sub_gitdir, sub_worktree))
+               return;
+
+       if (repo_read_index(&subrepo) < 0)
+               die("index file corrupt in repo %s", subrepo.gitdir);
+
+       for (i = 0; i < subrepo.index->cache_nr; i++) {
+               const struct cache_entry *ce = subrepo.index->cache[i];
+
+               if (!S_ISGITLINK(ce->ce_mode))
+                       continue;
+
+               while (i + 1 < subrepo.index->cache_nr &&
+                      !strcmp(ce->name, subrepo.index->cache[i + 1]->name))
+                       /*
+                        * Skip entries with the same name in different stages
+                        * to make sure an entry is returned only once.
+                        */
+                       i++;
+
+               sub = submodule_from_path(&subrepo, &null_oid, ce->name);
+               if (!sub || !is_submodule_active(&subrepo, ce->name))
+                       /* .gitmodules broken or inactive sub */
+                       continue;
+
+               strbuf_reset(&sub_wt);
+               strbuf_reset(&sub_gd);
+               strbuf_addf(&sub_wt, "%s/%s", sub_worktree, sub->path);
+               strbuf_addf(&sub_gd, "%s/modules/%s", sub_gitdir, sub->name);
+
+               connect_work_tree_and_git_dir(sub_wt.buf, sub_gd.buf, 1);
+       }
+       strbuf_release(&sub_wt);
+       strbuf_release(&sub_gd);
+       repo_clear(&subrepo);
 }
 
-/* Update gitfile and core.worktree setting to connect work tree and git dir */
-void connect_work_tree_and_git_dir(const char *work_tree_, const char *git_dir_)
+void connect_work_tree_and_git_dir(const char *work_tree_,
+                                  const char *git_dir_,
+                                  int recurse_into_nested)
 {
        struct strbuf gitfile_sb = STRBUF_INIT;
        struct strbuf cfg_sb = STRBUF_INIT;
@@ -2902,6 +3092,10 @@ void connect_work_tree_and_git_dir(const char *work_tree_, const char *git_dir_)
        strbuf_release(&gitfile_sb);
        strbuf_release(&cfg_sb);
        strbuf_release(&rel_path);
+
+       if (recurse_into_nested)
+               connect_wt_gitdir_in_nested(work_tree, git_dir);
+
        free(work_tree);
        free(git_dir);
 }
@@ -2915,5 +3109,5 @@ void relocate_gitdir(const char *path, const char *old_git_dir, const char *new_
                die_errno(_("could not migrate git directory from '%s' to '%s'"),
                        old_git_dir, new_git_dir);
 
-       connect_work_tree_and_git_dir(path, new_git_dir);
+       connect_work_tree_and_git_dir(path, new_git_dir, 0);
 }