Merge branch 'nd/exclusion-regression-fix'
authorJunio C Hamano <gitster@pobox.com>
Wed, 24 Feb 2016 21:25:59 +0000 (13:25 -0800)
committerJunio C Hamano <gitster@pobox.com>
Wed, 24 Feb 2016 21:25:59 +0000 (13:25 -0800)
Another try to add support to the ignore mechanism that lets you
say "this is excluded" and then later say "oh, no, this part (that
is a subset of the previous part) is not excluded".

* nd/exclusion-regression-fix:
dir.c: don't exclude whole dir prematurely
dir.c: support marking some patterns already matched
dir.c: support tracing exclude
dir.c: fix match_pathname()

Documentation/git-check-ignore.txt
Documentation/git.txt
Documentation/gitignore.txt
dir.c
dir.h
t/t3001-ls-files-others-exclude.sh
t/t3007-ls-files-other-negative.sh [new file with mode: 0755]
index e94367a5ed8e8b94bca7036ff7616bc3cfa9ca97..f60ee051f8c0e930e790add0bd1be39f49c82785 100644 (file)
@@ -114,6 +114,7 @@ SEE ALSO
 linkgit:gitignore[5]
 linkgit:gitconfig[5]
 linkgit:git-ls-files[1]
+GIT_TRACE_EXCLUDE in linkgit:git[1]
 
 GIT
 ---
index 9dffb4c03577770b0a2942938211ee05876003db..2754af8f7782d4a13fce4f5752737e3106491f22 100644 (file)
@@ -1065,6 +1065,11 @@ of clones and fetches.
        cloning of shallow repositories.
        See 'GIT_TRACE' for available trace output options.
 
+'GIT_TRACE_EXCLUDE'::
+       Enables trace messages that can help debugging .gitignore
+       processing. See 'GIT_TRACE' for available trace output
+       options.
+
 'GIT_LITERAL_PATHSPECS'::
        Setting this variable to `1` will cause Git to treat all
        pathspecs literally, rather than as glob patterns. For example,
index 473623d6318a859c9ed2cf600222ea6cb4a25d4c..3ded6fdc9972bb72a0efea73f0a2c73ae3e9906d 100644 (file)
@@ -82,12 +82,12 @@ PATTERN FORMAT
 
  - An optional prefix "`!`" which negates the pattern; any
    matching file excluded by a previous pattern will become
-   included again. It is not possible to re-include a file if a parent
-   directory of that file is excluded. Git doesn't list excluded
-   directories for performance reasons, so any patterns on contained
-   files have no effect, no matter where they are defined.
+   included again.
    Put a backslash ("`\`") in front of the first "`!`" for patterns
    that begin with a literal "`!`", for example, "`\!important!.txt`".
+   It is possible to re-include a file if a parent directory of that
+   file is excluded if certain conditions are met. See section NOTES
+   for detail.
 
  - If the pattern ends with a slash, it is removed for the
    purpose of the following description, but it would only find
@@ -141,6 +141,15 @@ not tracked by Git remain untracked.
 To stop tracking a file that is currently tracked, use
 'git rm --cached'.
 
+To re-include files or directories when their parent directory is
+excluded, the following conditions must be met:
+
+ - The rules to exclude a directory and re-include a subset back must
+   be in the same .gitignore file.
+
+ - The directory part in the re-include rules must be literal (i.e. no
+   wildcards)
+
 EXAMPLES
 --------
 
diff --git a/dir.c b/dir.c
index f0b6d0a3eab704311ed2c93a3da5db0683b3125c..552af237040b3bab95d94f43b169a6278623cea3 100644 (file)
--- a/dir.c
+++ b/dir.c
@@ -53,6 +53,8 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir,
        int check_only, const struct path_simplify *simplify);
 static int get_dtype(struct dirent *de, const char *path, int len);
 
+static struct trace_key trace_exclude = TRACE_KEY_INIT(EXCLUDE);
+
 /* helper string functions with support for the ignore_case flag */
 int strcmp_icase(const char *a, const char *b)
 {
@@ -519,6 +521,7 @@ void add_exclude(const char *string, const char *base,
        x->baselen = baselen;
        x->flags = flags;
        x->srcpos = srcpos;
+       string_list_init(&x->sticky_paths, 1);
        ALLOC_GROW(el->excludes, el->nr + 1, el->alloc);
        el->excludes[el->nr++] = x;
        x->el = el;
@@ -559,8 +562,10 @@ void clear_exclude_list(struct exclude_list *el)
 {
        int i;
 
-       for (i = 0; i < el->nr; i++)
+       for (i = 0; i < el->nr; i++) {
+               string_list_clear(&el->excludes[i]->sticky_paths, 0);
                free(el->excludes[i]);
+       }
        free(el->excludes);
        free(el->filebuf);
 
@@ -878,7 +883,7 @@ int match_pathname(const char *pathname, int pathlen,
                 * then our prefix match is all we need; we
                 * do not need to call fnmatch at all.
                 */
-               if (!patternlen && !namelen)
+               if (!patternlen && (!namelen || *name == '/'))
                        return 1;
        }
 
@@ -887,6 +892,113 @@ int match_pathname(const char *pathname, int pathlen,
                                 WM_PATHNAME) == 0;
 }
 
+static void add_sticky(struct exclude *exc, const char *pathname, int pathlen)
+{
+       struct strbuf sb = STRBUF_INIT;
+       int i;
+
+       for (i = exc->sticky_paths.nr - 1; i >= 0; i--) {
+               const char *sticky = exc->sticky_paths.items[i].string;
+               int len = strlen(sticky);
+
+               if (pathlen < len && sticky[pathlen] == '/' &&
+                   !strncmp(pathname, sticky, pathlen))
+                       return;
+       }
+
+       strbuf_add(&sb, pathname, pathlen);
+       string_list_append_nodup(&exc->sticky_paths, strbuf_detach(&sb, NULL));
+}
+
+static int match_sticky(struct exclude *exc, const char *pathname, int pathlen, int dtype)
+{
+       int i;
+
+       for (i = exc->sticky_paths.nr - 1; i >= 0; i--) {
+               const char *sticky = exc->sticky_paths.items[i].string;
+               int len = strlen(sticky);
+
+               if (pathlen == len && dtype == DT_DIR &&
+                   !strncmp(pathname, sticky, len))
+                       return 1;
+
+               if (pathlen > len && pathname[len] == '/' &&
+                   !strncmp(pathname, sticky, len))
+                       return 1;
+       }
+
+       return 0;
+}
+
+static inline int different_decisions(const struct exclude *a,
+                                     const struct exclude *b)
+{
+       return (a->flags & EXC_FLAG_NEGATIVE) != (b->flags & EXC_FLAG_NEGATIVE);
+}
+
+/*
+ * Return non-zero if pathname is a directory and an ancestor of the
+ * literal path in a pattern.
+ */
+static int match_directory_part(const char *pathname, int pathlen,
+                               int *dtype, struct exclude *x)
+{
+       const char      *base       = x->base;
+       int              baselen    = x->baselen ? x->baselen - 1 : 0;
+       const char      *pattern    = x->pattern;
+       int              prefix     = x->nowildcardlen;
+       int              patternlen = x->patternlen;
+
+       if (*dtype == DT_UNKNOWN)
+               *dtype = get_dtype(NULL, pathname, pathlen);
+       if (*dtype != DT_DIR)
+               return 0;
+
+       if (*pattern == '/') {
+               pattern++;
+               patternlen--;
+               prefix--;
+       }
+
+       if (baselen) {
+               if (((pathlen < baselen && base[pathlen] == '/') ||
+                    pathlen == baselen) &&
+                   !strncmp_icase(pathname, base, pathlen))
+                       return 1;
+               pathname += baselen + 1;
+               pathlen  -= baselen + 1;
+       }
+
+
+       if (prefix &&
+           (((pathlen < prefix && pattern[pathlen] == '/') ||
+             pathlen == prefix) &&
+            !strncmp_icase(pathname, pattern, pathlen)))
+               return 1;
+
+       return 0;
+}
+
+static struct exclude *should_descend(const char *pathname, int pathlen,
+                                     int *dtype, struct exclude_list *el,
+                                     struct exclude *exc)
+{
+       int i;
+
+       for (i = el->nr - 1; 0 <= i; i--) {
+               struct exclude *x = el->excludes[i];
+
+               if (x == exc)
+                       break;
+
+               if (!(x->flags & EXC_FLAG_NODIR) &&
+                   different_decisions(x, exc) &&
+                   match_directory_part(pathname, pathlen, dtype, x))
+                       return x;
+       }
+       return NULL;
+}
+
 /*
  * Scan the given exclude list in reverse to see whether pathname
  * should be ignored.  The first match (i.e. the last on the list), if
@@ -900,16 +1012,32 @@ static struct exclude *last_exclude_matching_from_list(const char *pathname,
                                                       struct exclude_list *el)
 {
        struct exclude *exc = NULL; /* undecided */
-       int i;
+       int i, maybe_descend = 0;
 
        if (!el->nr)
                return NULL;    /* undefined */
 
+       trace_printf_key(&trace_exclude, "exclude: from %s\n", el->src);
+
        for (i = el->nr - 1; 0 <= i; i--) {
                struct exclude *x = el->excludes[i];
                const char *exclude = x->pattern;
                int prefix = x->nowildcardlen;
 
+               if (!maybe_descend && i < el->nr - 1 &&
+                   different_decisions(x, el->excludes[i+1]))
+                       maybe_descend = 1;
+
+               if (x->sticky_paths.nr) {
+                       if (*dtype == DT_UNKNOWN)
+                               *dtype = get_dtype(NULL, pathname, pathlen);
+                       if (match_sticky(x, pathname, pathlen, *dtype)) {
+                               exc = x;
+                               break;
+                       }
+                       continue;
+               }
+
                if (x->flags & EXC_FLAG_MUSTBEDIR) {
                        if (*dtype == DT_UNKNOWN)
                                *dtype = get_dtype(NULL, pathname, pathlen);
@@ -936,6 +1064,45 @@ static struct exclude *last_exclude_matching_from_list(const char *pathname,
                        break;
                }
        }
+
+       if (!exc) {
+               trace_printf_key(&trace_exclude, "exclude: %.*s => n/a\n",
+                                pathlen, pathname);
+               return NULL;
+       }
+
+       /*
+        * We have found a matching pattern "exc" that may exclude whole
+        * directory. We also found that there may be a pattern that matches
+        * something inside the directory and reincludes stuff.
+        *
+        * Go through the patterns again, find that pattern and double check.
+        * If it's true, return "undecided" and keep descending in. "exc" is
+        * marked sticky so that it continues to match inside the directory.
+        */
+       if (!(exc->flags & EXC_FLAG_NEGATIVE) && maybe_descend) {
+               struct exclude *x;
+
+               if (*dtype == DT_UNKNOWN)
+                       *dtype = get_dtype(NULL, pathname, pathlen);
+
+               if (*dtype == DT_DIR &&
+                   (x = should_descend(pathname, pathlen, dtype, el, exc))) {
+                       add_sticky(exc, pathname, pathlen);
+                       trace_printf_key(&trace_exclude,
+                                        "exclude: %.*s vs %s at line %d => %s,"
+                                        " forced open by %s at line %d => n/a\n",
+                                        pathlen, pathname, exc->pattern, exc->srcpos,
+                                        exc->flags & EXC_FLAG_NEGATIVE ? "no" : "yes",
+                                        x->pattern, x->srcpos);
+                       return NULL;
+               }
+       }
+
+       trace_printf_key(&trace_exclude, "exclude: %.*s vs %s at line %d => %s%s\n",
+                        pathlen, pathname, exc->pattern, exc->srcpos,
+                        exc->flags & EXC_FLAG_NEGATIVE ? "no" : "yes",
+                        exc->sticky_paths.nr ? " (stuck)" : "");
        return exc;
 }
 
@@ -1683,9 +1850,13 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir,
        struct cached_dir cdir;
        enum path_treatment state, subdir_state, dir_state = path_none;
        struct strbuf path = STRBUF_INIT;
+       static int level = 0;
 
        strbuf_add(&path, base, baselen);
 
+       trace_printf_key(&trace_exclude, "exclude: [%d] enter '%.*s'\n",
+                        level++, baselen, base);
+
        if (open_cached_dir(&cdir, dir, untracked, &path, check_only))
                goto out;
 
@@ -1749,6 +1920,8 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir,
        }
        close_cached_dir(&cdir);
  out:
+       trace_printf_key(&trace_exclude, "exclude: [%d] leave '%.*s'\n",
+                        --level, baselen, base);
        strbuf_release(&path);
 
        return dir_state;
@@ -1985,6 +2158,25 @@ static struct untracked_cache_dir *validate_untracked_cache(struct dir_struct *d
        return root;
 }
 
+static void clear_sticky(struct dir_struct *dir)
+{
+       struct exclude_list_group *g;
+       struct exclude_list *el;
+       struct exclude *x;
+       int i, j, k;
+
+       for (i = EXC_CMDL; i <= EXC_FILE; i++) {
+               g = &dir->exclude_list_group[i];
+               for (j = g->nr - 1; j >= 0; j--) {
+                       el = &g->el[j];
+                       for (k = el->nr - 1; 0 <= k; k--) {
+                               x = el->excludes[k];
+                               string_list_clear(&x->sticky_paths, 0);
+                       }
+               }
+       }
+}
+
 int read_directory(struct dir_struct *dir, const char *path, int len, const struct pathspec *pathspec)
 {
        struct path_simplify *simplify;
@@ -2005,6 +2197,12 @@ int read_directory(struct dir_struct *dir, const char *path, int len, const stru
        if (has_symlink_leading_path(path, len))
                return dir->nr;
 
+       /*
+        * Stay on the safe side. if read_directory() has run once on
+        * "dir", some sticky flag may have been left. Clear them all.
+        */
+       clear_sticky(dir);
+
        /*
         * exclude patterns are treated like positive ones in
         * create_simplify. Usually exclude patterns should be a
diff --git a/dir.h b/dir.h
index cd46f30017ce239720926afdad4301b2ac402ccf..3ec3fb0dca22164134ea0700094bf63bdc6ad329 100644 (file)
--- a/dir.h
+++ b/dir.h
@@ -4,6 +4,7 @@
 /* See Documentation/technical/api-directory-listing.txt */
 
 #include "strbuf.h"
+#include "string-list.h"
 
 struct dir_entry {
        unsigned int len;
@@ -34,6 +35,8 @@ struct exclude {
         * and from -1 decrementing for patterns from CLI args.
         */
        int srcpos;
+
+       struct string_list sticky_paths;
 };
 
 /*
index 3fc484e8c3f8d910f02e409baf7bf1d5682b4f2f..d043078da526275c3b1505297f5300cf1ad26a22 100755 (executable)
@@ -175,13 +175,10 @@ test_expect_success 'negated exclude matches can override previous ones' '
        grep "^a.1" output
 '
 
-test_expect_success 'excluded directory overrides content patterns' '
+test_expect_success 'excluded directory does not override content patterns' '
 
        git ls-files --others --exclude="one" --exclude="!one/a.1" >output &&
-       if grep "^one/a.1" output
-       then
-               false
-       fi
+       grep "^one/a.1" output
 '
 
 test_expect_success 'negated directory doesn'\''t affect content patterns' '
diff --git a/t/t3007-ls-files-other-negative.sh b/t/t3007-ls-files-other-negative.sh
new file mode 100755 (executable)
index 0000000..0797b86
--- /dev/null
@@ -0,0 +1,153 @@
+#!/bin/sh
+
+test_description='test re-include patterns'
+
+. ./test-lib.sh
+
+test_expect_success 'setup' '
+       mkdir -p fooo foo/bar tmp &&
+       touch abc foo/def foo/bar/ghi foo/bar/bar
+'
+
+test_expect_success 'no match, do not enter subdir and waste cycles' '
+       cat >.gitignore <<-\EOF &&
+       /tmp
+       /foo
+       !fooo/bar/bar
+       EOF
+       GIT_TRACE_EXCLUDE="$(pwd)/tmp/trace" git ls-files -o --exclude-standard >tmp/actual &&
+       ! grep "enter .foo/.\$" tmp/trace &&
+       cat >tmp/expected <<-\EOF &&
+       .gitignore
+       abc
+       EOF
+       test_cmp tmp/expected tmp/actual
+'
+
+test_expect_success 'match, excluded by literal pathname pattern' '
+       cat >.gitignore <<-\EOF &&
+       /tmp
+       /fooo
+       /foo
+       !foo/bar/bar
+       EOF
+       cat >fooo/.gitignore <<-\EOF &&
+       !/*
+       EOF     git ls-files -o --exclude-standard >tmp/actual &&
+       cat >tmp/expected <<-\EOF &&
+       .gitignore
+       abc
+       foo/bar/bar
+       EOF
+       test_cmp tmp/expected tmp/actual
+'
+
+test_expect_success 'match, excluded by wildcard pathname pattern' '
+       cat >.gitignore <<-\EOF &&
+       /tmp
+       /fooo
+       /fo?
+       !foo/bar/bar
+       EOF
+       git ls-files -o --exclude-standard >tmp/actual &&
+       cat >tmp/expected <<-\EOF &&
+       .gitignore
+       abc
+       foo/bar/bar
+       EOF
+       test_cmp tmp/expected tmp/actual
+'
+
+test_expect_success 'match, excluded by literal basename pattern' '
+       cat >.gitignore <<-\EOF &&
+       /tmp
+       /fooo
+       foo
+       !foo/bar/bar
+       EOF
+       git ls-files -o --exclude-standard >tmp/actual &&
+       cat >tmp/expected <<-\EOF &&
+       .gitignore
+       abc
+       foo/bar/bar
+       EOF
+       test_cmp tmp/expected tmp/actual
+'
+
+test_expect_success 'match, excluded by wildcard basename pattern' '
+       cat >.gitignore <<-\EOF &&
+       /tmp
+       /fooo
+       fo?
+       !foo/bar/bar
+       EOF
+       git ls-files -o --exclude-standard >tmp/actual &&
+       cat >tmp/expected <<-\EOF &&
+       .gitignore
+       abc
+       foo/bar/bar
+       EOF
+       test_cmp tmp/expected tmp/actual
+'
+
+test_expect_success 'match, excluded by literal mustbedir, basename pattern' '
+       cat >.gitignore <<-\EOF &&
+       /tmp
+       /fooo
+       foo/
+       !foo/bar/bar
+       EOF
+       git ls-files -o --exclude-standard >tmp/actual &&
+       cat >tmp/expected <<-\EOF &&
+       .gitignore
+       abc
+       foo/bar/bar
+       EOF
+       test_cmp tmp/expected tmp/actual
+'
+
+test_expect_success 'match, excluded by literal mustbedir, pathname pattern' '
+       cat >.gitignore <<-\EOF &&
+       /tmp
+       /fooo
+       /foo/
+       !foo/bar/bar
+       EOF
+       git ls-files -o --exclude-standard >tmp/actual &&
+       cat >tmp/expected <<-\EOF &&
+       .gitignore
+       abc
+       foo/bar/bar
+       EOF
+       test_cmp tmp/expected tmp/actual
+'
+
+test_expect_success 'prepare for nested negatives' '
+       cat >.git/info/exclude <<-\EOF &&
+       /.gitignore
+       /tmp
+       /foo
+       /abc
+       EOF
+       git ls-files -o --exclude-standard >tmp/actual &&
+       test_must_be_empty tmp/actual &&
+       mkdir -p 1/2/3/4 &&
+       touch 1/f 1/2/f 1/2/3/f 1/2/3/4/f
+'
+
+test_expect_success 'match, literal pathname, nested negatives' '
+       cat >.gitignore <<-\EOF &&
+       /1
+       !1/2
+       1/2/3
+       !1/2/3/4
+       EOF
+       git ls-files -o --exclude-standard >tmp/actual &&
+       cat >tmp/expected <<-\EOF &&
+       1/2/3/4/f
+       1/2/f
+       EOF
+       test_cmp tmp/expected tmp/actual
+'
+
+test_done