fsmonitor: teach git to optionally utilize a file system monitor to speed up detecting new or changed files.
authorBen Peart <benpeart@microsoft.com>
Fri, 22 Sep 2017 16:35:40 +0000 (12:35 -0400)
committerJunio C Hamano <gitster@pobox.com>
Sun, 1 Oct 2017 08:23:01 +0000 (17:23 +0900)
When the index is read from disk, the fsmonitor index extension is used
to flag the last known potentially dirty index entries. The registered
core.fsmonitor command is called with the time the index was last
updated and returns the list of files changed since that time. This list
is used to flag any additional dirty cache entries and untracked cache
directories.

We can then use this valid state to speed up preload_index(),
ie_match_stat(), and refresh_cache_ent() as they do not need to lstat()
files to detect potential changes for those entries marked
CE_FSMONITOR_VALID.

In addition, if the untracked cache is turned on valid_cached_dir() can
skip checking directories for new or changed files as fsmonitor will
invalidate the cache only for those directories that have been
identified as having potential changes.

To keep the CE_FSMONITOR_VALID state accurate during git operations;
when git updates a cache entry to match the current state on disk,
it will now set the CE_FSMONITOR_VALID bit.

Inversely, anytime git changes a cache entry, the CE_FSMONITOR_VALID bit
is cleared and the corresponding untracked cache directory is marked
invalid.

Signed-off-by: Ben Peart <benpeart@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
16 files changed:
Makefile
builtin/update-index.c
cache.h
config.c
config.h
diff-lib.c
dir.c
dir.h
entry.c
environment.c
fsmonitor.c [new file with mode: 0644]
fsmonitor.h [new file with mode: 0644]
preload-index.c
read-cache.c
submodule.c
unpack-trees.c
index f2bb7f2f63e458af6261b4a151d87ef9e973222e..9d6ec9c1e9a1a7f4bf41c458684f3cd3a234d3ad 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -786,6 +786,7 @@ LIB_OBJS += ewah/ewah_rlw.o
 LIB_OBJS += exec_cmd.o
 LIB_OBJS += fetch-pack.o
 LIB_OBJS += fsck.o
 LIB_OBJS += exec_cmd.o
 LIB_OBJS += fetch-pack.o
 LIB_OBJS += fsck.o
+LIB_OBJS += fsmonitor.o
 LIB_OBJS += gettext.o
 LIB_OBJS += gpg-interface.o
 LIB_OBJS += graph.o
 LIB_OBJS += gettext.o
 LIB_OBJS += gpg-interface.o
 LIB_OBJS += graph.o
index e1ca0759d587b69d76d75cdda6790f8be4e13563..6f39ee9274adaa16da131bef45459196f3a92ba0 100644 (file)
@@ -16,6 +16,7 @@
 #include "pathspec.h"
 #include "dir.h"
 #include "split-index.h"
 #include "pathspec.h"
 #include "dir.h"
 #include "split-index.h"
+#include "fsmonitor.h"
 
 /*
  * Default to not allowing changes to the list of files. The
 
 /*
  * Default to not allowing changes to the list of files. The
@@ -233,6 +234,7 @@ static int mark_ce_flags(const char *path, int flag, int mark)
                else
                        active_cache[pos]->ce_flags &= ~flag;
                active_cache[pos]->ce_flags |= CE_UPDATE_IN_BASE;
                else
                        active_cache[pos]->ce_flags &= ~flag;
                active_cache[pos]->ce_flags |= CE_UPDATE_IN_BASE;
+               mark_fsmonitor_invalid(&the_index, active_cache[pos]);
                cache_tree_invalidate_path(&the_index, path);
                active_cache_changed |= CE_ENTRY_CHANGED;
                return 0;
                cache_tree_invalidate_path(&the_index, path);
                active_cache_changed |= CE_ENTRY_CHANGED;
                return 0;
diff --git a/cache.h b/cache.h
index a916bc79e3c1a77f55764f34f6357478e83b6cb0..f1c903e1b689be91109772fa27f5ec4d1aa97b77 100644 (file)
--- a/cache.h
+++ b/cache.h
@@ -203,6 +203,7 @@ struct cache_entry {
 #define CE_ADDED             (1 << 19)
 
 #define CE_HASHED            (1 << 20)
 #define CE_ADDED             (1 << 19)
 
 #define CE_HASHED            (1 << 20)
+#define CE_FSMONITOR_VALID   (1 << 21)
 #define CE_WT_REMOVE         (1 << 22) /* remove in work directory */
 #define CE_CONFLICTED        (1 << 23)
 
 #define CE_WT_REMOVE         (1 << 22) /* remove in work directory */
 #define CE_CONFLICTED        (1 << 23)
 
@@ -326,6 +327,7 @@ static inline unsigned int canon_mode(unsigned int mode)
 #define CACHE_TREE_CHANGED     (1 << 5)
 #define SPLIT_INDEX_ORDERED    (1 << 6)
 #define UNTRACKED_CHANGED      (1 << 7)
 #define CACHE_TREE_CHANGED     (1 << 5)
 #define SPLIT_INDEX_ORDERED    (1 << 6)
 #define UNTRACKED_CHANGED      (1 << 7)
+#define FSMONITOR_CHANGED      (1 << 8)
 
 struct split_index;
 struct untracked_cache;
 
 struct split_index;
 struct untracked_cache;
@@ -344,6 +346,7 @@ struct index_state {
        struct hashmap dir_hash;
        unsigned char sha1[20];
        struct untracked_cache *untracked;
        struct hashmap dir_hash;
        unsigned char sha1[20];
        struct untracked_cache *untracked;
+       uint64_t fsmonitor_last_update;
 };
 
 extern struct index_state the_index;
 };
 
 extern struct index_state the_index;
@@ -679,8 +682,10 @@ extern void *read_blob_data_from_index(const struct index_state *, const char *,
 #define CE_MATCH_IGNORE_MISSING                0x08
 /* enable stat refresh */
 #define CE_MATCH_REFRESH               0x10
 #define CE_MATCH_IGNORE_MISSING                0x08
 /* enable stat refresh */
 #define CE_MATCH_REFRESH               0x10
-extern int ie_match_stat(const struct index_state *, const struct cache_entry *, struct stat *, unsigned int);
-extern int ie_modified(const struct index_state *, const struct cache_entry *, struct stat *, unsigned int);
+/* don't refresh_fsmonitor state or do stat comparison even if CE_FSMONITOR_VALID is true */
+#define CE_MATCH_IGNORE_FSMONITOR 0X20
+extern int ie_match_stat(struct index_state *, const struct cache_entry *, struct stat *, unsigned int);
+extern int ie_modified(struct index_state *, const struct cache_entry *, struct stat *, unsigned int);
 
 #define HASH_WRITE_OBJECT 1
 #define HASH_FORMAT_CHECK 2
 
 #define HASH_WRITE_OBJECT 1
 #define HASH_FORMAT_CHECK 2
@@ -773,6 +778,7 @@ extern int core_apply_sparse_checkout;
 extern int precomposed_unicode;
 extern int protect_hfs;
 extern int protect_ntfs;
 extern int precomposed_unicode;
 extern int protect_hfs;
 extern int protect_ntfs;
+extern const char *core_fsmonitor;
 
 /*
  * Include broken refs in all ref iterations, which will
 
 /*
  * Include broken refs in all ref iterations, which will
index d0d8ce823ae6d8be4b9cc92e1d01e44ca7569e84..ddda96e5842e4b6bffedbaee8b50f5009a8d2361 100644 (file)
--- a/config.c
+++ b/config.c
@@ -2165,6 +2165,20 @@ int git_config_get_max_percent_split_change(void)
        return -1; /* default value */
 }
 
        return -1; /* default value */
 }
 
+int git_config_get_fsmonitor(void)
+{
+       if (git_config_get_pathname("core.fsmonitor", &core_fsmonitor))
+               core_fsmonitor = getenv("GIT_FSMONITOR_TEST");
+
+       if (core_fsmonitor && !*core_fsmonitor)
+               core_fsmonitor = NULL;
+
+       if (core_fsmonitor)
+               return 1;
+
+       return 0;
+}
+
 NORETURN
 void git_die_config_linenr(const char *key, const char *filename, int linenr)
 {
 NORETURN
 void git_die_config_linenr(const char *key, const char *filename, int linenr)
 {
index 97471b887320b48f1b4c2d5a2023f6ffd1acf94e..c9fcf691baea299ee86477772d5220198d9790aa 100644 (file)
--- a/config.h
+++ b/config.h
@@ -211,6 +211,7 @@ extern int git_config_get_pathname(const char *key, const char **dest);
 extern int git_config_get_untracked_cache(void);
 extern int git_config_get_split_index(void);
 extern int git_config_get_max_percent_split_change(void);
 extern int git_config_get_untracked_cache(void);
 extern int git_config_get_split_index(void);
 extern int git_config_get_max_percent_split_change(void);
+extern int git_config_get_fsmonitor(void);
 
 /* This dies if the configured or default date is in the future */
 extern int git_config_get_expiry(const char *key, const char **output);
 
 /* This dies if the configured or default date is in the future */
 extern int git_config_get_expiry(const char *key, const char **output);
index 2a52b079546cb01d8dfe708f7e7c7e38ec582435..23c6d03ca922c42c6d8efaf6d41c6ae5139efb10 100644 (file)
@@ -12,6 +12,7 @@
 #include "refs.h"
 #include "submodule.h"
 #include "dir.h"
 #include "refs.h"
 #include "submodule.h"
 #include "dir.h"
+#include "fsmonitor.h"
 
 /*
  * diff-files
 
 /*
  * diff-files
@@ -228,6 +229,7 @@ int run_diff_files(struct rev_info *revs, unsigned int option)
 
                if (!changed && !dirty_submodule) {
                        ce_mark_uptodate(ce);
 
                if (!changed && !dirty_submodule) {
                        ce_mark_uptodate(ce);
+                       mark_fsmonitor_valid(ce);
                        if (!DIFF_OPT_TST(&revs->diffopt, FIND_COPIES_HARDER))
                                continue;
                }
                        if (!DIFF_OPT_TST(&revs->diffopt, FIND_COPIES_HARDER))
                                continue;
                }
diff --git a/dir.c b/dir.c
index 1c55dc3e366f8c9e8bcbc62305d7085533059512..ac9833daec79a24a645f46626236676dc044f636 100644 (file)
--- a/dir.c
+++ b/dir.c
@@ -18,6 +18,7 @@
 #include "utf8.h"
 #include "varint.h"
 #include "ewah/ewok.h"
 #include "utf8.h"
 #include "varint.h"
 #include "ewah/ewok.h"
+#include "fsmonitor.h"
 
 /*
  * Tells read_directory_recursive how a file or directory should be treated.
 
 /*
  * Tells read_directory_recursive how a file or directory should be treated.
@@ -1688,17 +1689,23 @@ static int valid_cached_dir(struct dir_struct *dir,
        if (!untracked)
                return 0;
 
        if (!untracked)
                return 0;
 
-       if (stat(path->len ? path->buf : ".", &st)) {
-               invalidate_directory(dir->untracked, untracked);
-               memset(&untracked->stat_data, 0, sizeof(untracked->stat_data));
-               return 0;
-       }
-       if (!untracked->valid ||
-           match_stat_data_racy(istate, &untracked->stat_data, &st)) {
-               if (untracked->valid)
+       /*
+        * With fsmonitor, we can trust the untracked cache's valid field.
+        */
+       refresh_fsmonitor(istate);
+       if (!(dir->untracked->use_fsmonitor && untracked->valid)) {
+               if (stat(path->len ? path->buf : ".", &st)) {
                        invalidate_directory(dir->untracked, untracked);
                        invalidate_directory(dir->untracked, untracked);
-               fill_stat_data(&untracked->stat_data, &st);
-               return 0;
+                       memset(&untracked->stat_data, 0, sizeof(untracked->stat_data));
+                       return 0;
+               }
+               if (!untracked->valid ||
+                       match_stat_data_racy(istate, &untracked->stat_data, &st)) {
+                       if (untracked->valid)
+                               invalidate_directory(dir->untracked, untracked);
+                       fill_stat_data(&untracked->stat_data, &st);
+                       return 0;
+               }
        }
 
        if (untracked->check_only != !!check_only) {
        }
 
        if (untracked->check_only != !!check_only) {
diff --git a/dir.h b/dir.h
index e3717055d193366e6780b50d68f6a3c2360d32c4..fab8fc1561b582bbdb2d1706c437c6c0290204a1 100644 (file)
--- a/dir.h
+++ b/dir.h
@@ -139,6 +139,8 @@ struct untracked_cache {
        int gitignore_invalidated;
        int dir_invalidated;
        int dir_opened;
        int gitignore_invalidated;
        int dir_invalidated;
        int dir_opened;
+       /* fsmonitor invalidation data */
+       unsigned int use_fsmonitor : 1;
 };
 
 struct dir_struct {
 };
 
 struct dir_struct {
diff --git a/entry.c b/entry.c
index cb291aa88bf148608184edeff68943315b44e6ab..3a7b6673738302e2c549053b024e31ab9f8df56c 100644 (file)
--- a/entry.c
+++ b/entry.c
@@ -4,6 +4,7 @@
 #include "streaming.h"
 #include "submodule.h"
 #include "progress.h"
 #include "streaming.h"
 #include "submodule.h"
 #include "progress.h"
+#include "fsmonitor.h"
 
 static void create_directories(const char *path, int path_len,
                               const struct checkout *state)
 
 static void create_directories(const char *path, int path_len,
                               const struct checkout *state)
@@ -357,6 +358,7 @@ static int write_entry(struct cache_entry *ce,
                        lstat(ce->name, &st);
                fill_stat_cache_info(ce, &st);
                ce->ce_flags |= CE_UPDATE_IN_BASE;
                        lstat(ce->name, &st);
                fill_stat_cache_info(ce, &st);
                ce->ce_flags |= CE_UPDATE_IN_BASE;
+               mark_fsmonitor_invalid(state->istate, ce);
                state->istate->cache_changed |= CE_ENTRY_CHANGED;
        }
        return 0;
                state->istate->cache_changed |= CE_ENTRY_CHANGED;
        }
        return 0;
index 3fd4b1084590d0118cad5e7ae32fb37e570e7523..d0b9fc64d4dd1677a759afebb8f9a92559c120d8 100644 (file)
@@ -76,6 +76,7 @@ int protect_hfs = PROTECT_HFS_DEFAULT;
 #define PROTECT_NTFS_DEFAULT 0
 #endif
 int protect_ntfs = PROTECT_NTFS_DEFAULT;
 #define PROTECT_NTFS_DEFAULT 0
 #endif
 int protect_ntfs = PROTECT_NTFS_DEFAULT;
+const char *core_fsmonitor;
 
 /*
  * The character that begins a commented line in user-editable file
 
 /*
  * The character that begins a commented line in user-editable file
diff --git a/fsmonitor.c b/fsmonitor.c
new file mode 100644 (file)
index 0000000..7c1540c
--- /dev/null
@@ -0,0 +1,253 @@
+#include "cache.h"
+#include "config.h"
+#include "dir.h"
+#include "ewah/ewok.h"
+#include "fsmonitor.h"
+#include "run-command.h"
+#include "strbuf.h"
+
+#define INDEX_EXTENSION_VERSION        (1)
+#define HOOK_INTERFACE_VERSION (1)
+
+struct trace_key trace_fsmonitor = TRACE_KEY_INIT(FSMONITOR);
+
+static void fsmonitor_ewah_callback(size_t pos, void *is)
+{
+       struct index_state *istate = (struct index_state *)is;
+       struct cache_entry *ce = istate->cache[pos];
+
+       ce->ce_flags &= ~CE_FSMONITOR_VALID;
+}
+
+int read_fsmonitor_extension(struct index_state *istate, const void *data,
+       unsigned long sz)
+{
+       const char *index = data;
+       uint32_t hdr_version;
+       uint32_t ewah_size;
+       struct ewah_bitmap *fsmonitor_dirty;
+       int i;
+       int ret;
+
+       if (sz < sizeof(uint32_t) + sizeof(uint64_t) + sizeof(uint32_t))
+               return error("corrupt fsmonitor extension (too short)");
+
+       hdr_version = get_be32(index);
+       index += sizeof(uint32_t);
+       if (hdr_version != INDEX_EXTENSION_VERSION)
+               return error("bad fsmonitor version %d", hdr_version);
+
+       istate->fsmonitor_last_update = get_be64(index);
+       index += sizeof(uint64_t);
+
+       ewah_size = get_be32(index);
+       index += sizeof(uint32_t);
+
+       fsmonitor_dirty = ewah_new();
+       ret = ewah_read_mmap(fsmonitor_dirty, index, ewah_size);
+       if (ret != ewah_size) {
+               ewah_free(fsmonitor_dirty);
+               return error("failed to parse ewah bitmap reading fsmonitor index extension");
+       }
+
+       if (git_config_get_fsmonitor()) {
+               /* Mark all entries valid */
+               for (i = 0; i < istate->cache_nr; i++)
+                       istate->cache[i]->ce_flags |= CE_FSMONITOR_VALID;
+
+               /* Mark all previously saved entries as dirty */
+               ewah_each_bit(fsmonitor_dirty, fsmonitor_ewah_callback, istate);
+
+               /* Now mark the untracked cache for fsmonitor usage */
+               if (istate->untracked)
+                       istate->untracked->use_fsmonitor = 1;
+       }
+       ewah_free(fsmonitor_dirty);
+
+       trace_printf_key(&trace_fsmonitor, "read fsmonitor extension successful");
+       return 0;
+}
+
+void write_fsmonitor_extension(struct strbuf *sb, struct index_state *istate)
+{
+       uint32_t hdr_version;
+       uint64_t tm;
+       struct ewah_bitmap *bitmap;
+       int i;
+       uint32_t ewah_start;
+       uint32_t ewah_size = 0;
+       int fixup = 0;
+
+       put_be32(&hdr_version, INDEX_EXTENSION_VERSION);
+       strbuf_add(sb, &hdr_version, sizeof(uint32_t));
+
+       put_be64(&tm, istate->fsmonitor_last_update);
+       strbuf_add(sb, &tm, sizeof(uint64_t));
+       fixup = sb->len;
+       strbuf_add(sb, &ewah_size, sizeof(uint32_t)); /* we'll fix this up later */
+
+       ewah_start = sb->len;
+       bitmap = ewah_new();
+       for (i = 0; i < istate->cache_nr; i++)
+               if (!(istate->cache[i]->ce_flags & CE_FSMONITOR_VALID))
+                       ewah_set(bitmap, i);
+       ewah_serialize_strbuf(bitmap, sb);
+       ewah_free(bitmap);
+
+       /* fix up size field */
+       put_be32(&ewah_size, sb->len - ewah_start);
+       memcpy(sb->buf + fixup, &ewah_size, sizeof(uint32_t));
+
+       trace_printf_key(&trace_fsmonitor, "write fsmonitor extension successful");
+}
+
+/*
+ * Call the query-fsmonitor hook passing the time of the last saved results.
+ */
+static int query_fsmonitor(int version, uint64_t last_update, struct strbuf *query_result)
+{
+       struct child_process cp = CHILD_PROCESS_INIT;
+       char ver[64];
+       char date[64];
+       const char *argv[4];
+
+       if (!(argv[0] = core_fsmonitor))
+               return -1;
+
+       snprintf(ver, sizeof(version), "%d", version);
+       snprintf(date, sizeof(date), "%" PRIuMAX, (uintmax_t)last_update);
+       argv[1] = ver;
+       argv[2] = date;
+       argv[3] = NULL;
+       cp.argv = argv;
+       cp.use_shell = 1;
+
+       return capture_command(&cp, query_result, 1024);
+}
+
+static void fsmonitor_refresh_callback(struct index_state *istate, const char *name)
+{
+       int pos = index_name_pos(istate, name, strlen(name));
+
+       if (pos >= 0) {
+               struct cache_entry *ce = istate->cache[pos];
+               ce->ce_flags &= ~CE_FSMONITOR_VALID;
+       }
+
+       /*
+        * Mark the untracked cache dirty even if it wasn't found in the index
+        * as it could be a new untracked file.
+        */
+       trace_printf_key(&trace_fsmonitor, "fsmonitor_refresh_callback '%s'", name);
+       untracked_cache_invalidate_path(istate, name);
+}
+
+void refresh_fsmonitor(struct index_state *istate)
+{
+       static int has_run_once = 0;
+       struct strbuf query_result = STRBUF_INIT;
+       int query_success = 0;
+       size_t bol; /* beginning of line */
+       uint64_t last_update;
+       char *buf;
+       int i;
+
+       if (!core_fsmonitor || has_run_once)
+               return;
+       has_run_once = 1;
+
+       trace_printf_key(&trace_fsmonitor, "refresh fsmonitor");
+       /*
+        * This could be racy so save the date/time now and query_fsmonitor
+        * should be inclusive to ensure we don't miss potential changes.
+        */
+       last_update = getnanotime();
+
+       /*
+        * If we have a last update time, call query_fsmonitor for the set of
+        * changes since that time, else assume everything is possibly dirty
+        * and check it all.
+        */
+       if (istate->fsmonitor_last_update) {
+               query_success = !query_fsmonitor(HOOK_INTERFACE_VERSION,
+                       istate->fsmonitor_last_update, &query_result);
+               trace_performance_since(last_update, "fsmonitor process '%s'", core_fsmonitor);
+               trace_printf_key(&trace_fsmonitor, "fsmonitor process '%s' returned %s",
+                       core_fsmonitor, query_success ? "success" : "failure");
+       }
+
+       /* a fsmonitor process can return '/' to indicate all entries are invalid */
+       if (query_success && query_result.buf[0] != '/') {
+               /* Mark all entries returned by the monitor as dirty */
+               buf = query_result.buf;
+               bol = 0;
+               for (i = 0; i < query_result.len; i++) {
+                       if (buf[i] != '\0')
+                               continue;
+                       fsmonitor_refresh_callback(istate, buf + bol);
+                       bol = i + 1;
+               }
+               if (bol < query_result.len)
+                       fsmonitor_refresh_callback(istate, buf + bol);
+       } else {
+               /* Mark all entries invalid */
+               for (i = 0; i < istate->cache_nr; i++)
+                       istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID;
+
+               if (istate->untracked)
+                       istate->untracked->use_fsmonitor = 0;
+       }
+       strbuf_release(&query_result);
+
+       /* Now that we've updated istate, save the last_update time */
+       istate->fsmonitor_last_update = last_update;
+}
+
+void add_fsmonitor(struct index_state *istate)
+{
+       int i;
+
+       if (!istate->fsmonitor_last_update) {
+               trace_printf_key(&trace_fsmonitor, "add fsmonitor");
+               istate->cache_changed |= FSMONITOR_CHANGED;
+               istate->fsmonitor_last_update = getnanotime();
+
+               /* reset the fsmonitor state */
+               for (i = 0; i < istate->cache_nr; i++)
+                       istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID;
+
+               /* reset the untracked cache */
+               if (istate->untracked) {
+                       add_untracked_cache(istate);
+                       istate->untracked->use_fsmonitor = 1;
+               }
+
+               /* Update the fsmonitor state */
+               refresh_fsmonitor(istate);
+       }
+}
+
+void remove_fsmonitor(struct index_state *istate)
+{
+       if (istate->fsmonitor_last_update) {
+               trace_printf_key(&trace_fsmonitor, "remove fsmonitor");
+               istate->cache_changed |= FSMONITOR_CHANGED;
+               istate->fsmonitor_last_update = 0;
+       }
+}
+
+void tweak_fsmonitor(struct index_state *istate)
+{
+       switch (git_config_get_fsmonitor()) {
+       case -1: /* keep: do nothing */
+               break;
+       case 0: /* false */
+               remove_fsmonitor(istate);
+               break;
+       case 1: /* true */
+               add_fsmonitor(istate);
+               break;
+       default: /* unknown value: do nothing */
+               break;
+       }
+}
diff --git a/fsmonitor.h b/fsmonitor.h
new file mode 100644 (file)
index 0000000..0de644e
--- /dev/null
@@ -0,0 +1,66 @@
+#ifndef FSMONITOR_H
+#define FSMONITOR_H
+
+extern struct trace_key trace_fsmonitor;
+
+/*
+ * Read the fsmonitor index extension and (if configured) restore the
+ * CE_FSMONITOR_VALID state.
+ */
+extern int read_fsmonitor_extension(struct index_state *istate, const void *data, unsigned long sz);
+
+/*
+ * Write the CE_FSMONITOR_VALID state into the fsmonitor index extension.
+ */
+extern void write_fsmonitor_extension(struct strbuf *sb, struct index_state *istate);
+
+/*
+ * Add/remove the fsmonitor index extension
+ */
+extern void add_fsmonitor(struct index_state *istate);
+extern void remove_fsmonitor(struct index_state *istate);
+
+/*
+ * Add/remove the fsmonitor index extension as necessary based on the current
+ * core.fsmonitor setting.
+ */
+extern void tweak_fsmonitor(struct index_state *istate);
+
+/*
+ * Run the configured fsmonitor integration script and clear the
+ * CE_FSMONITOR_VALID bit for any files returned as dirty.  Also invalidate
+ * any corresponding untracked cache directory structures. Optimized to only
+ * run the first time it is called.
+ */
+extern void refresh_fsmonitor(struct index_state *istate);
+
+/*
+ * Set the given cache entries CE_FSMONITOR_VALID bit. This should be
+ * called any time the cache entry has been updated to reflect the
+ * current state of the file on disk.
+ */
+static inline void mark_fsmonitor_valid(struct cache_entry *ce)
+{
+       if (core_fsmonitor) {
+               ce->ce_flags |= CE_FSMONITOR_VALID;
+               trace_printf_key(&trace_fsmonitor, "mark_fsmonitor_clean '%s'", ce->name);
+       }
+}
+
+/*
+ * Clear the given cache entry's CE_FSMONITOR_VALID bit and invalidate
+ * any corresponding untracked cache directory structures. This should
+ * be called any time git creates or modifies a file that should
+ * trigger an lstat() or invalidate the untracked cache for the
+ * corresponding directory
+ */
+static inline void mark_fsmonitor_invalid(struct index_state *istate, struct cache_entry *ce)
+{
+       if (core_fsmonitor) {
+               ce->ce_flags &= ~CE_FSMONITOR_VALID;
+               untracked_cache_invalidate_path(istate, ce->name);
+               trace_printf_key(&trace_fsmonitor, "mark_fsmonitor_invalid '%s'", ce->name);
+       }
+}
+
+#endif
index 75564c497a6e55f6f5f7d13d1ab25c40ed5ed423..2a83255e4eeaf8e8ce74516880b0333b23d6f583 100644 (file)
@@ -4,6 +4,7 @@
 #include "cache.h"
 #include "pathspec.h"
 #include "dir.h"
 #include "cache.h"
 #include "pathspec.h"
 #include "dir.h"
+#include "fsmonitor.h"
 
 #ifdef NO_PTHREADS
 static void preload_index(struct index_state *index,
 
 #ifdef NO_PTHREADS
 static void preload_index(struct index_state *index,
@@ -55,15 +56,18 @@ static void *preload_thread(void *_data)
                        continue;
                if (ce_skip_worktree(ce))
                        continue;
                        continue;
                if (ce_skip_worktree(ce))
                        continue;
+               if (ce->ce_flags & CE_FSMONITOR_VALID)
+                       continue;
                if (!ce_path_match(ce, &p->pathspec, NULL))
                        continue;
                if (threaded_has_symlink_leading_path(&cache, ce->name, ce_namelen(ce)))
                        continue;
                if (lstat(ce->name, &st))
                        continue;
                if (!ce_path_match(ce, &p->pathspec, NULL))
                        continue;
                if (threaded_has_symlink_leading_path(&cache, ce->name, ce_namelen(ce)))
                        continue;
                if (lstat(ce->name, &st))
                        continue;
-               if (ie_match_stat(index, ce, &st, CE_MATCH_RACY_IS_DIRTY))
+               if (ie_match_stat(index, ce, &st, CE_MATCH_RACY_IS_DIRTY|CE_MATCH_IGNORE_FSMONITOR))
                        continue;
                ce_mark_uptodate(ce);
                        continue;
                ce_mark_uptodate(ce);
+               mark_fsmonitor_valid(ce);
        } while (--nr > 0);
        cache_def_clear(&cache);
        return NULL;
        } while (--nr > 0);
        cache_def_clear(&cache);
        return NULL;
index 40da87ea71f088114c5a40893850cc88efee6953..05c0a33fddfa6989f1cd630f744686500dce12ea 100644 (file)
@@ -19,6 +19,7 @@
 #include "varint.h"
 #include "split-index.h"
 #include "utf8.h"
 #include "varint.h"
 #include "split-index.h"
 #include "utf8.h"
+#include "fsmonitor.h"
 
 /* Mask for the name length in ce_flags in the on-disk index */
 
 
 /* Mask for the name length in ce_flags in the on-disk index */
 
 #define CACHE_EXT_RESOLVE_UNDO 0x52455543 /* "REUC" */
 #define CACHE_EXT_LINK 0x6c696e6b        /* "link" */
 #define CACHE_EXT_UNTRACKED 0x554E5452   /* "UNTR" */
 #define CACHE_EXT_RESOLVE_UNDO 0x52455543 /* "REUC" */
 #define CACHE_EXT_LINK 0x6c696e6b        /* "link" */
 #define CACHE_EXT_UNTRACKED 0x554E5452   /* "UNTR" */
+#define CACHE_EXT_FSMONITOR 0x46534D4E   /* "FSMN" */
 
 /* changes that can be kept in $GIT_DIR/index (basically all extensions) */
 #define EXTMASK (RESOLVE_UNDO_CHANGED | CACHE_TREE_CHANGED | \
                 CE_ENTRY_ADDED | CE_ENTRY_REMOVED | CE_ENTRY_CHANGED | \
 
 /* changes that can be kept in $GIT_DIR/index (basically all extensions) */
 #define EXTMASK (RESOLVE_UNDO_CHANGED | CACHE_TREE_CHANGED | \
                 CE_ENTRY_ADDED | CE_ENTRY_REMOVED | CE_ENTRY_CHANGED | \
-                SPLIT_INDEX_ORDERED | UNTRACKED_CHANGED)
+                SPLIT_INDEX_ORDERED | UNTRACKED_CHANGED | FSMONITOR_CHANGED)
 
 struct index_state the_index;
 static const char *alternate_index_output;
 
 struct index_state the_index;
 static const char *alternate_index_output;
@@ -62,6 +64,7 @@ static void replace_index_entry(struct index_state *istate, int nr, struct cache
        free(old);
        set_index_entry(istate, nr, ce);
        ce->ce_flags |= CE_UPDATE_IN_BASE;
        free(old);
        set_index_entry(istate, nr, ce);
        ce->ce_flags |= CE_UPDATE_IN_BASE;
+       mark_fsmonitor_invalid(istate, ce);
        istate->cache_changed |= CE_ENTRY_CHANGED;
 }
 
        istate->cache_changed |= CE_ENTRY_CHANGED;
 }
 
@@ -150,8 +153,10 @@ void fill_stat_cache_info(struct cache_entry *ce, struct stat *st)
        if (assume_unchanged)
                ce->ce_flags |= CE_VALID;
 
        if (assume_unchanged)
                ce->ce_flags |= CE_VALID;
 
-       if (S_ISREG(st->st_mode))
+       if (S_ISREG(st->st_mode)) {
                ce_mark_uptodate(ce);
                ce_mark_uptodate(ce);
+               mark_fsmonitor_valid(ce);
+       }
 }
 
 static int ce_compare_data(const struct cache_entry *ce, struct stat *st)
 }
 
 static int ce_compare_data(const struct cache_entry *ce, struct stat *st)
@@ -300,7 +305,7 @@ int match_stat_data_racy(const struct index_state *istate,
        return match_stat_data(sd, st);
 }
 
        return match_stat_data(sd, st);
 }
 
-int ie_match_stat(const struct index_state *istate,
+int ie_match_stat(struct index_state *istate,
                  const struct cache_entry *ce, struct stat *st,
                  unsigned int options)
 {
                  const struct cache_entry *ce, struct stat *st,
                  unsigned int options)
 {
@@ -308,7 +313,10 @@ int ie_match_stat(const struct index_state *istate,
        int ignore_valid = options & CE_MATCH_IGNORE_VALID;
        int ignore_skip_worktree = options & CE_MATCH_IGNORE_SKIP_WORKTREE;
        int assume_racy_is_modified = options & CE_MATCH_RACY_IS_DIRTY;
        int ignore_valid = options & CE_MATCH_IGNORE_VALID;
        int ignore_skip_worktree = options & CE_MATCH_IGNORE_SKIP_WORKTREE;
        int assume_racy_is_modified = options & CE_MATCH_RACY_IS_DIRTY;
+       int ignore_fsmonitor = options & CE_MATCH_IGNORE_FSMONITOR;
 
 
+       if (!ignore_fsmonitor)
+               refresh_fsmonitor(istate);
        /*
         * If it's marked as always valid in the index, it's
         * valid whatever the checked-out copy says.
        /*
         * If it's marked as always valid in the index, it's
         * valid whatever the checked-out copy says.
@@ -319,6 +327,8 @@ int ie_match_stat(const struct index_state *istate,
                return 0;
        if (!ignore_valid && (ce->ce_flags & CE_VALID))
                return 0;
                return 0;
        if (!ignore_valid && (ce->ce_flags & CE_VALID))
                return 0;
+       if (!ignore_fsmonitor && (ce->ce_flags & CE_FSMONITOR_VALID))
+               return 0;
 
        /*
         * Intent-to-add entries have not been added, so the index entry
 
        /*
         * Intent-to-add entries have not been added, so the index entry
@@ -356,7 +366,7 @@ int ie_match_stat(const struct index_state *istate,
        return changed;
 }
 
        return changed;
 }
 
-int ie_modified(const struct index_state *istate,
+int ie_modified(struct index_state *istate,
                const struct cache_entry *ce,
                struct stat *st, unsigned int options)
 {
                const struct cache_entry *ce,
                struct stat *st, unsigned int options)
 {
@@ -777,6 +787,7 @@ int chmod_index_entry(struct index_state *istate, struct cache_entry *ce,
        }
        cache_tree_invalidate_path(istate, ce->name);
        ce->ce_flags |= CE_UPDATE_IN_BASE;
        }
        cache_tree_invalidate_path(istate, ce->name);
        ce->ce_flags |= CE_UPDATE_IN_BASE;
+       mark_fsmonitor_invalid(istate, ce);
        istate->cache_changed |= CE_ENTRY_CHANGED;
 
        return 0;
        istate->cache_changed |= CE_ENTRY_CHANGED;
 
        return 0;
@@ -1228,10 +1239,13 @@ static struct cache_entry *refresh_cache_ent(struct index_state *istate,
        int ignore_valid = options & CE_MATCH_IGNORE_VALID;
        int ignore_skip_worktree = options & CE_MATCH_IGNORE_SKIP_WORKTREE;
        int ignore_missing = options & CE_MATCH_IGNORE_MISSING;
        int ignore_valid = options & CE_MATCH_IGNORE_VALID;
        int ignore_skip_worktree = options & CE_MATCH_IGNORE_SKIP_WORKTREE;
        int ignore_missing = options & CE_MATCH_IGNORE_MISSING;
+       int ignore_fsmonitor = options & CE_MATCH_IGNORE_FSMONITOR;
 
        if (!refresh || ce_uptodate(ce))
                return ce;
 
 
        if (!refresh || ce_uptodate(ce))
                return ce;
 
+       if (!ignore_fsmonitor)
+               refresh_fsmonitor(istate);
        /*
         * CE_VALID or CE_SKIP_WORKTREE means the user promised us
         * that the change to the work tree does not matter and told
        /*
         * CE_VALID or CE_SKIP_WORKTREE means the user promised us
         * that the change to the work tree does not matter and told
@@ -1245,6 +1259,10 @@ static struct cache_entry *refresh_cache_ent(struct index_state *istate,
                ce_mark_uptodate(ce);
                return ce;
        }
                ce_mark_uptodate(ce);
                return ce;
        }
+       if (!ignore_fsmonitor && (ce->ce_flags & CE_FSMONITOR_VALID)) {
+               ce_mark_uptodate(ce);
+               return ce;
+       }
 
        if (has_symlink_leading_path(ce->name, ce_namelen(ce))) {
                if (ignore_missing)
 
        if (has_symlink_leading_path(ce->name, ce_namelen(ce))) {
                if (ignore_missing)
@@ -1282,8 +1300,10 @@ static struct cache_entry *refresh_cache_ent(struct index_state *istate,
                         * because CE_UPTODATE flag is in-core only;
                         * we are not going to write this change out.
                         */
                         * because CE_UPTODATE flag is in-core only;
                         * we are not going to write this change out.
                         */
-                       if (!S_ISGITLINK(ce->ce_mode))
+                       if (!S_ISGITLINK(ce->ce_mode)) {
                                ce_mark_uptodate(ce);
                                ce_mark_uptodate(ce);
+                               mark_fsmonitor_valid(ce);
+                       }
                        return ce;
                }
        }
                        return ce;
                }
        }
@@ -1391,6 +1411,7 @@ int refresh_index(struct index_state *istate, unsigned int flags,
                                 */
                                ce->ce_flags &= ~CE_VALID;
                                ce->ce_flags |= CE_UPDATE_IN_BASE;
                                 */
                                ce->ce_flags &= ~CE_VALID;
                                ce->ce_flags |= CE_UPDATE_IN_BASE;
+                               mark_fsmonitor_invalid(istate, ce);
                                istate->cache_changed |= CE_ENTRY_CHANGED;
                        }
                        if (quiet)
                                istate->cache_changed |= CE_ENTRY_CHANGED;
                        }
                        if (quiet)
@@ -1550,6 +1571,9 @@ static int read_index_extension(struct index_state *istate,
        case CACHE_EXT_UNTRACKED:
                istate->untracked = read_untracked_extension(data, sz);
                break;
        case CACHE_EXT_UNTRACKED:
                istate->untracked = read_untracked_extension(data, sz);
                break;
+       case CACHE_EXT_FSMONITOR:
+               read_fsmonitor_extension(istate, data, sz);
+               break;
        default:
                if (*ext < 'A' || 'Z' < *ext)
                        return error("index uses %.4s extension, which we do not understand",
        default:
                if (*ext < 'A' || 'Z' < *ext)
                        return error("index uses %.4s extension, which we do not understand",
@@ -1722,6 +1746,7 @@ static void post_read_index_from(struct index_state *istate)
        check_ce_order(istate);
        tweak_untracked_cache(istate);
        tweak_split_index(istate);
        check_ce_order(istate);
        tweak_untracked_cache(istate);
        tweak_split_index(istate);
+       tweak_fsmonitor(istate);
 }
 
 /* remember to discard_cache() before reading a different cache! */
 }
 
 /* remember to discard_cache() before reading a different cache! */
@@ -2306,6 +2331,16 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile,
                if (err)
                        return -1;
        }
                if (err)
                        return -1;
        }
+       if (!strip_extensions && istate->fsmonitor_last_update) {
+               struct strbuf sb = STRBUF_INIT;
+
+               write_fsmonitor_extension(&sb, istate);
+               err = write_index_ext_header(&c, newfd, CACHE_EXT_FSMONITOR, sb.len) < 0
+                       || ce_write(&c, newfd, sb.buf, sb.len) < 0;
+               strbuf_release(&sb);
+               if (err)
+                       return -1;
+       }
 
        if (ce_flush(&c, newfd, istate->sha1))
                return -1;
 
        if (ce_flush(&c, newfd, istate->sha1))
                return -1;
index 3cea8221e0bc3dbe77b157384adcecdfc077c7dc..8a931a1aaa2200adff96d4006f3fb4971c558e67 100644 (file)
@@ -62,7 +62,7 @@ int is_staging_gitmodules_ok(const struct index_state *istate)
        if ((pos >= 0) && (pos < istate->cache_nr)) {
                struct stat st;
                if (lstat(GITMODULES_FILE, &st) == 0 &&
        if ((pos >= 0) && (pos < istate->cache_nr)) {
                struct stat st;
                if (lstat(GITMODULES_FILE, &st) == 0 &&
-                   ce_match_stat(istate->cache[pos], &st, 0) & DATA_CHANGED)
+                   ce_match_stat(istate->cache[pos], &st, CE_MATCH_IGNORE_FSMONITOR) & DATA_CHANGED)
                        return 0;
        }
 
                        return 0;
        }
 
index 71b70ccb12756cde1f1c5aeda952be6e2b91f5b8..1f5d3716360bd04bf882396be23bd798c209b7b0 100644 (file)
@@ -14,6 +14,7 @@
 #include "dir.h"
 #include "submodule.h"
 #include "submodule-config.h"
 #include "dir.h"
 #include "submodule.h"
 #include "submodule-config.h"
+#include "fsmonitor.h"
 
 /*
  * Error messages expected by scripts out of plumbing commands such as
 
 /*
  * Error messages expected by scripts out of plumbing commands such as
@@ -408,6 +409,7 @@ static int apply_sparse_checkout(struct index_state *istate,
                ce->ce_flags &= ~CE_SKIP_WORKTREE;
        if (was_skip_worktree != ce_skip_worktree(ce)) {
                ce->ce_flags |= CE_UPDATE_IN_BASE;
                ce->ce_flags &= ~CE_SKIP_WORKTREE;
        if (was_skip_worktree != ce_skip_worktree(ce)) {
                ce->ce_flags |= CE_UPDATE_IN_BASE;
+               mark_fsmonitor_invalid(istate, ce);
                istate->cache_changed |= CE_ENTRY_CHANGED;
        }
 
                istate->cache_changed |= CE_ENTRY_CHANGED;
        }