untracked cache: save to an index extension

Helped-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/technical/index-format.txt b/Documentation/technical/index-format.txt
index 35112e4966f9b021b0889d734d5cb945b8943003..db59a13600382004bbfc93d1b882afd0d3f47d61 100644 (file)
--- a/Documentation/technical/index-format.txt
+++ b/Documentation/technical/index-format.txt
@@ -233,3 +233,61 @@ Git index format
   The remaining index entries after replaced ones will be added to the
   final index. These added entries are also sorted by entry name then
   stage.
   The remaining index entries after replaced ones will be added to the
   final index. These added entries are also sorted by entry name then
   stage.

+
+== Untracked cache
+
+  Untracked cache saves the untracked file list and necessary data to
+  verify the cache. The signature for this extension is { 'U', 'N',
+  'T', 'R' }.
+
+  The extension starts with
+
+  - Stat data of $GIT_DIR/info/exclude. See "Index entry" section from
+    ctime field until "file size".
+
+  - Stat data of core.excludesfile
+
+  - 32-bit dir_flags (see struct dir_struct)
+
+  - 160-bit SHA-1 of $GIT_DIR/info/exclude. Null SHA-1 means the file
+    does not exist.
+
+  - 160-bit SHA-1 of core.excludesfile. Null SHA-1 means the file does
+    not exist.
+
+  - NUL-terminated string of per-dir exclude file name. This usually
+    is ".gitignore".
+
+  - The number of following directory blocks, variable width
+    encoding. If this number is zero, the extension ends here with a
+    following NUL.
+
+  - A number of directory blocks in depth-first-search order, each
+    consists of
+
+    - The number of untracked entries, variable width encoding.
+
+    - The number of sub-directory blocks, variable width encoding.
+
+    - The directory name terminated by NUL.
+
+    - A number of untrached file/dir names terminated by NUL.
+
+The remaining data of each directory block is grouped by type:
+
+  - An ewah bitmap, the n-th bit marks whether the n-th directory has
+    valid untracked cache entries.
+
+  - An ewah bitmap, the n-th bit records "check-only" bit of
+    read_directory_recursive() for the n-th directory.
+
+  - An ewah bitmap, the n-th bit indicates whether SHA-1 and stat data
+    is valid for the n-th directory and exists in the next data.
+
+  - An array of stat data. The n-th data corresponds with the n-th
+    "one" bit in the previous ewah bitmap.
+
+  - An array of SHA-1. The n-th SHA-1 corresponds with the n-th "one" bit
+    in the previous ewah bitmap.
+
+  - One NUL.

diff --git a/cache.h b/cache.h
index 761c5704b2e21f5dcd68fdcc9c3b1247ab0cfd18..811cc36547aa5b2b036f56e7d4a2c6dd100e913d 100644 (file)
--- a/cache.h
+++ b/cache.h
@@ -291,6 +291,8 @@ static inline unsigned int canon_mode(unsigned int mode)
 #define SPLIT_INDEX_ORDERED    (1 << 6)
 
 struct split_index;
 #define SPLIT_INDEX_ORDERED    (1 << 6)
 
 struct split_index;

+struct untracked_cache;
+

 struct index_state {
        struct cache_entry **cache;
        unsigned int version;
 struct index_state {
        struct cache_entry **cache;
        unsigned int version;


@@ -304,6 +306,7 @@ struct index_state {
        struct hashmap name_hash;
        struct hashmap dir_hash;
        unsigned char sha1[20];
        struct hashmap name_hash;
        struct hashmap dir_hash;
        unsigned char sha1[20];

+       struct untracked_cache *untracked;

 };
 
 extern struct index_state the_index;
 };
 
 extern struct index_state the_index;

diff --git a/dir.c b/dir.c
index f39024c6396bf456a9ecbfa8dc886e83aebeb3d4..f695e0c685862a6ba181b7b674e703e9add6d845 100644 (file)
--- a/dir.c
+++ b/dir.c
@@ -12,6 +12,8 @@
 #include "refs.h"
 #include "wildmatch.h"
 #include "pathspec.h"
 #include "refs.h"
 #include "wildmatch.h"
 #include "pathspec.h"

+#include "varint.h"
+#include "ewah/ewok.h"

 
 struct path_simplify {
        int len;
 
 struct path_simplify {
        int len;


@@ -2144,3 +2146,140 @@ void clear_directory(struct dir_struct *dir)
        }
        strbuf_release(&dir->basebuf);
 }
        }
        strbuf_release(&dir->basebuf);
 }

+
+struct ondisk_untracked_cache {
+       struct stat_data info_exclude_stat;
+       struct stat_data excludes_file_stat;
+       uint32_t dir_flags;
+       unsigned char info_exclude_sha1[20];
+       unsigned char excludes_file_sha1[20];
+       char exclude_per_dir[FLEX_ARRAY];
+};
+
+#define ouc_size(len) (offsetof(struct ondisk_untracked_cache, exclude_per_dir) + len + 1)
+
+struct write_data {
+       int index;         /* number of written untracked_cache_dir */
+       struct ewah_bitmap *check_only; /* from untracked_cache_dir */
+       struct ewah_bitmap *valid;      /* from untracked_cache_dir */
+       struct ewah_bitmap *sha1_valid; /* set if exclude_sha1 is not null */
+       struct strbuf out;
+       struct strbuf sb_stat;
+       struct strbuf sb_sha1;
+};
+
+static void stat_data_to_disk(struct stat_data *to, const struct stat_data *from)
+{
+       to->sd_ctime.sec  = htonl(from->sd_ctime.sec);
+       to->sd_ctime.nsec = htonl(from->sd_ctime.nsec);
+       to->sd_mtime.sec  = htonl(from->sd_mtime.sec);
+       to->sd_mtime.nsec = htonl(from->sd_mtime.nsec);
+       to->sd_dev        = htonl(from->sd_dev);
+       to->sd_ino        = htonl(from->sd_ino);
+       to->sd_uid        = htonl(from->sd_uid);
+       to->sd_gid        = htonl(from->sd_gid);
+       to->sd_size       = htonl(from->sd_size);
+}
+
+static void write_one_dir(struct untracked_cache_dir *untracked,
+                         struct write_data *wd)
+{
+       struct stat_data stat_data;
+       struct strbuf *out = &wd->out;
+       unsigned char intbuf[16];
+       unsigned int intlen, value;
+       int i = wd->index++;
+
+       /*
+        * untracked_nr should be reset whenever valid is clear, but
+        * for safety..
+        */
+       if (!untracked->valid) {
+               untracked->untracked_nr = 0;
+               untracked->check_only = 0;
+       }
+
+       if (untracked->check_only)
+               ewah_set(wd->check_only, i);
+       if (untracked->valid) {
+               ewah_set(wd->valid, i);
+               stat_data_to_disk(&stat_data, &untracked->stat_data);
+               strbuf_add(&wd->sb_stat, &stat_data, sizeof(stat_data));
+       }
+       if (!is_null_sha1(untracked->exclude_sha1)) {
+               ewah_set(wd->sha1_valid, i);
+               strbuf_add(&wd->sb_sha1, untracked->exclude_sha1, 20);
+       }
+
+       intlen = encode_varint(untracked->untracked_nr, intbuf);
+       strbuf_add(out, intbuf, intlen);
+
+       /* skip non-recurse directories */
+       for (i = 0, value = 0; i < untracked->dirs_nr; i++)
+               if (untracked->dirs[i]->recurse)
+                       value++;
+       intlen = encode_varint(value, intbuf);
+       strbuf_add(out, intbuf, intlen);
+
+       strbuf_add(out, untracked->name, strlen(untracked->name) + 1);
+
+       for (i = 0; i < untracked->untracked_nr; i++)
+               strbuf_add(out, untracked->untracked[i],
+                          strlen(untracked->untracked[i]) + 1);
+
+       for (i = 0; i < untracked->dirs_nr; i++)
+               if (untracked->dirs[i]->recurse)
+                       write_one_dir(untracked->dirs[i], wd);
+}
+
+void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked)
+{
+       struct ondisk_untracked_cache *ouc;
+       struct write_data wd;
+       unsigned char varbuf[16];
+       int len = 0, varint_len;
+       if (untracked->exclude_per_dir)
+               len = strlen(untracked->exclude_per_dir);
+       ouc = xmalloc(sizeof(*ouc) + len + 1);
+       stat_data_to_disk(&ouc->info_exclude_stat, &untracked->ss_info_exclude.stat);
+       stat_data_to_disk(&ouc->excludes_file_stat, &untracked->ss_excludes_file.stat);
+       hashcpy(ouc->info_exclude_sha1, untracked->ss_info_exclude.sha1);
+       hashcpy(ouc->excludes_file_sha1, untracked->ss_excludes_file.sha1);
+       ouc->dir_flags = htonl(untracked->dir_flags);
+       memcpy(ouc->exclude_per_dir, untracked->exclude_per_dir, len + 1);
+       strbuf_add(out, ouc, ouc_size(len));
+       free(ouc);
+       ouc = NULL;
+
+       if (!untracked->root) {
+               varint_len = encode_varint(0, varbuf);
+               strbuf_add(out, varbuf, varint_len);
+               return;
+       }
+
+       wd.index      = 0;
+       wd.check_only = ewah_new();
+       wd.valid      = ewah_new();
+       wd.sha1_valid = ewah_new();
+       strbuf_init(&wd.out, 1024);
+       strbuf_init(&wd.sb_stat, 1024);
+       strbuf_init(&wd.sb_sha1, 1024);
+       write_one_dir(untracked->root, &wd);
+
+       varint_len = encode_varint(wd.index, varbuf);
+       strbuf_add(out, varbuf, varint_len);
+       strbuf_addbuf(out, &wd.out);
+       ewah_serialize_strbuf(wd.valid, out);
+       ewah_serialize_strbuf(wd.check_only, out);
+       ewah_serialize_strbuf(wd.sha1_valid, out);
+       strbuf_addbuf(out, &wd.sb_stat);
+       strbuf_addbuf(out, &wd.sb_sha1);
+       strbuf_addch(out, '\0'); /* safe guard for string lists */
+
+       ewah_free(wd.valid);
+       ewah_free(wd.check_only);
+       ewah_free(wd.sha1_valid);
+       strbuf_release(&wd.out);
+       strbuf_release(&wd.sb_stat);
+       strbuf_release(&wd.sb_sha1);
+}

diff --git a/dir.h b/dir.h
index 95baf014ca6356039deeda6bc4259d15f54a3c20..dc3ee0b2e577be1fbc70a98645c44d8f6da5fab6 100644 (file)
--- a/dir.h
+++ b/dir.h
@@ -298,4 +298,5 @@ static inline int dir_path_match(const struct dir_entry *ent,
                              has_trailing_dir);
 }
 
                              has_trailing_dir);
 }
 

+void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked);

 #endif
 #endif

diff --git a/read-cache.c b/read-cache.c
index 8d71860f692ffa5deafe7f23774b1b9ab2f31378..3a058d008abce8894335b8c442f38b17d56f30a1 100644 (file)
--- a/read-cache.c
+++ b/read-cache.c
@@ -39,6 +39,7 @@ static struct cache_entry *refresh_cache_entry(struct cache_entry *ce,
 #define CACHE_EXT_TREE 0x54524545      /* "TREE" */
 #define CACHE_EXT_RESOLVE_UNDO 0x52455543 /* "REUC" */
 #define CACHE_EXT_LINK 0x6c696e6b        /* "link" */
 #define CACHE_EXT_TREE 0x54524545      /* "TREE" */
 #define CACHE_EXT_RESOLVE_UNDO 0x52455543 /* "REUC" */
 #define CACHE_EXT_LINK 0x6c696e6b        /* "link" */

+#define CACHE_EXT_UNTRACKED 0x554E5452   /* "UNTR" */

 
 /* changes that can be kept in $GIT_DIR/index (basically all extensions) */
 #define EXTMASK (RESOLVE_UNDO_CHANGED | CACHE_TREE_CHANGED | \
 
 /* changes that can be kept in $GIT_DIR/index (basically all extensions) */
 #define EXTMASK (RESOLVE_UNDO_CHANGED | CACHE_TREE_CHANGED | \


@@ -2047,6 +2048,17 @@ static int do_write_index(struct index_state *istate, int newfd,
                if (err)
                        return -1;
        }
                if (err)
                        return -1;
        }

+       if (!strip_extensions && istate->untracked) {
+               struct strbuf sb = STRBUF_INIT;
+
+               write_untracked_extension(&sb, istate->untracked);
+               err = write_index_ext_header(&c, newfd, CACHE_EXT_UNTRACKED,
+                                            sb.len) < 0 ||
+                       ce_write(&c, newfd, sb.buf, sb.len) < 0;
+               strbuf_release(&sb);
+               if (err)
+                       return -1;
+       }

 
        if (ce_flush(&c, newfd, istate->sha1) || fstat(newfd, &st))
                return -1;
 
        if (ce_flush(&c, newfd, istate->sha1) || fstat(newfd, &st))
                return -1;