#include "refs.h"
#include "wildmatch.h"
#include "pathspec.h"
+#include "varint.h"
+#include "ewah/ewok.h"
struct path_simplify {
int len;
path_untracked
};
+/*
+ * Support data structure for our opendir/readdir/closedir wrappers
+ */
+struct cached_dir {
+ DIR *fdir;
+ struct untracked_cache_dir *untracked;
+ int nr_files;
+ int nr_dirs;
+
+ struct dirent *de;
+ const char *file;
+ struct untracked_cache_dir *ucd;
+};
+
static enum path_treatment read_directory_recursive(struct dir_struct *dir,
const char *path, int len, struct untracked_cache_dir *untracked,
int check_only, const struct path_simplify *simplify);
do_invalidate_gitignore(dir);
}
+static void invalidate_directory(struct untracked_cache *uc,
+ struct untracked_cache_dir *dir)
+{
+ int i;
+ uc->dir_invalidated++;
+ dir->valid = 0;
+ dir->untracked_nr = 0;
+ for (i = 0; i < dir->dirs_nr; i++)
+ dir->dirs[i]->recurse = 0;
+}
+
/*
* Given a file with name "fname", read it (either from disk, or from
* the index if "check_index" is non-zero), parse it and store the
/* Try to read per-directory file */
hashclr(sha1_stat.sha1);
sha1_stat.valid = 0;
- if (dir->exclude_per_dir) {
+ if (dir->exclude_per_dir &&
+ /*
+ * If we know that no files have been added in
+ * this directory (i.e. valid_cached_dir() has
+ * been executed and set untracked->valid) ..
+ */
+ (!untracked || !untracked->valid ||
+ /*
+ * .. and .gitignore does not exist before
+ * (i.e. null exclude_sha1 and skip_worktree is
+ * not set). Then we can skip loading .gitignore,
+ * which would result in ENOENT anyway.
+ * skip_worktree is taken care in read_directory()
+ */
+ !is_null_sha1(untracked->exclude_sha1))) {
/*
* dir->basebuf gets reused by the traversal, but we
* need fname to remain unchanged to ensure the src
}
}
+static enum path_treatment treat_path_fast(struct dir_struct *dir,
+ struct untracked_cache_dir *untracked,
+ struct cached_dir *cdir,
+ struct strbuf *path,
+ int baselen,
+ const struct path_simplify *simplify)
+{
+ strbuf_setlen(path, baselen);
+ if (!cdir->ucd) {
+ strbuf_addstr(path, cdir->file);
+ return path_untracked;
+ }
+ strbuf_addstr(path, cdir->ucd->name);
+ /* treat_one_path() does this before it calls treat_directory() */
+ if (path->buf[path->len - 1] != '/')
+ strbuf_addch(path, '/');
+ if (cdir->ucd->check_only)
+ /*
+ * check_only is set as a result of treat_directory() getting
+ * to its bottom. Verify again the same set of directories
+ * with check_only set.
+ */
+ return read_directory_recursive(dir, path->buf, path->len,
+ cdir->ucd, 1, simplify);
+ /*
+ * We get path_recurse in the first run when
+ * directory_exists_in_index() returns index_nonexistent. We
+ * are sure that new changes in the index does not impact the
+ * outcome. Return now.
+ */
+ return path_recurse;
+}
+
static enum path_treatment treat_path(struct dir_struct *dir,
struct untracked_cache_dir *untracked,
- struct dirent *de,
+ struct cached_dir *cdir,
struct strbuf *path,
int baselen,
const struct path_simplify *simplify)
{
int dtype;
+ struct dirent *de = cdir->de;
+ if (!de)
+ return treat_path_fast(dir, untracked, cdir, path,
+ baselen, simplify);
if (is_dot_or_dotdot(de->d_name) || !strcmp(de->d_name, ".git"))
return path_none;
strbuf_setlen(path, baselen);
dir->untracked[dir->untracked_nr++] = xstrdup(name);
}
+static int valid_cached_dir(struct dir_struct *dir,
+ struct untracked_cache_dir *untracked,
+ struct strbuf *path,
+ int check_only)
+{
+ struct stat st;
+
+ if (!untracked)
+ return 0;
+
+ if (stat(path->len ? path->buf : ".", &st)) {
+ invalidate_directory(dir->untracked, untracked);
+ memset(&untracked->stat_data, 0, sizeof(untracked->stat_data));
+ return 0;
+ }
+ if (!untracked->valid ||
+ match_stat_data(&untracked->stat_data, &st)) {
+ if (untracked->valid)
+ invalidate_directory(dir->untracked, untracked);
+ fill_stat_data(&untracked->stat_data, &st);
+ return 0;
+ }
+
+ if (untracked->check_only != !!check_only) {
+ invalidate_directory(dir->untracked, untracked);
+ return 0;
+ }
+
+ /*
+ * prep_exclude will be called eventually on this directory,
+ * but it's called much later in last_exclude_matching(). We
+ * need it now to determine the validity of the cache for this
+ * path. The next calls will be nearly no-op, the way
+ * prep_exclude() is designed.
+ */
+ if (path->len && path->buf[path->len - 1] != '/') {
+ strbuf_addch(path, '/');
+ prep_exclude(dir, path->buf, path->len);
+ strbuf_setlen(path, path->len - 1);
+ } else
+ prep_exclude(dir, path->buf, path->len);
+
+ /* hopefully prep_exclude() haven't invalidated this entry... */
+ return untracked->valid;
+}
+
+static int open_cached_dir(struct cached_dir *cdir,
+ struct dir_struct *dir,
+ struct untracked_cache_dir *untracked,
+ struct strbuf *path,
+ int check_only)
+{
+ memset(cdir, 0, sizeof(*cdir));
+ cdir->untracked = untracked;
+ if (valid_cached_dir(dir, untracked, path, check_only))
+ return 0;
+ cdir->fdir = opendir(path->len ? path->buf : ".");
+ if (dir->untracked)
+ dir->untracked->dir_opened++;
+ if (!cdir->fdir)
+ return -1;
+ return 0;
+}
+
+static int read_cached_dir(struct cached_dir *cdir)
+{
+ if (cdir->fdir) {
+ cdir->de = readdir(cdir->fdir);
+ if (!cdir->de)
+ return -1;
+ return 0;
+ }
+ while (cdir->nr_dirs < cdir->untracked->dirs_nr) {
+ struct untracked_cache_dir *d = cdir->untracked->dirs[cdir->nr_dirs];
+ if (!d->recurse) {
+ cdir->nr_dirs++;
+ continue;
+ }
+ cdir->ucd = d;
+ cdir->nr_dirs++;
+ return 0;
+ }
+ cdir->ucd = NULL;
+ if (cdir->nr_files < cdir->untracked->untracked_nr) {
+ struct untracked_cache_dir *d = cdir->untracked;
+ cdir->file = d->untracked[cdir->nr_files++];
+ return 0;
+ }
+ return -1;
+}
+
+static void close_cached_dir(struct cached_dir *cdir)
+{
+ if (cdir->fdir)
+ closedir(cdir->fdir);
+ /*
+ * We have gone through this directory and found no untracked
+ * entries. Mark it valid.
+ */
+ if (cdir->untracked) {
+ cdir->untracked->valid = 1;
+ cdir->untracked->recurse = 1;
+ }
+}
+
/*
* Read a directory tree. We currently ignore anything but
* directories, regular files and symlinks. That's because git
struct untracked_cache_dir *untracked, int check_only,
const struct path_simplify *simplify)
{
- DIR *fdir;
+ struct cached_dir cdir;
enum path_treatment state, subdir_state, dir_state = path_none;
- struct dirent *de;
struct strbuf path = STRBUF_INIT;
strbuf_add(&path, base, baselen);
- fdir = opendir(path.len ? path.buf : ".");
- if (!fdir)
+ if (open_cached_dir(&cdir, dir, untracked, &path, check_only))
goto out;
if (untracked)
untracked->check_only = !!check_only;
- while ((de = readdir(fdir)) != NULL) {
+ while (!read_cached_dir(&cdir)) {
/* check how the file or directory should be treated */
- state = treat_path(dir, untracked, de, &path, baselen, simplify);
+ state = treat_path(dir, untracked, &cdir, &path, baselen, simplify);
if (state > dir_state)
dir_state = state;
if (check_only) {
/* abort early if maximum state has been reached */
if (dir_state == path_untracked) {
- if (untracked)
+ if (cdir.fdir)
add_untracked(untracked, path.buf + baselen);
break;
}
if (dir->flags & DIR_SHOW_IGNORED)
break;
dir_add_name(dir, path.buf, path.len);
- if (untracked)
+ if (cdir.fdir)
add_untracked(untracked, path.buf + baselen);
break;
break;
}
}
- closedir(fdir);
+ close_cached_dir(&cdir);
out:
strbuf_release(&path);
const struct pathspec *pathspec)
{
struct untracked_cache_dir *root;
+ int i;
if (!dir->untracked)
return NULL;
if (dir->exclude_list_group[EXC_CMDL].nr)
return NULL;
+ /*
+ * An optimization in prep_exclude() does not play well with
+ * CE_SKIP_WORKTREE. It's a rare case anyway, if a single
+ * entry has that bit set, disable the whole untracked cache.
+ */
+ for (i = 0; i < active_nr; i++)
+ if (ce_skip_worktree(active_cache[i]))
+ return NULL;
+
if (!dir->untracked->root) {
const int len = sizeof(*dir->untracked->root);
dir->untracked->root = xmalloc(len);
invalidate_gitignore(dir->untracked, root);
dir->untracked->ss_excludes_file = dir->ss_excludes_file;
}
+
+ /* Make sure this directory is not dropped out at saving phase */
+ root->recurse = 1;
return root;
}
}
strbuf_release(&dir->basebuf);
}
+
+struct ondisk_untracked_cache {
+ struct stat_data info_exclude_stat;
+ struct stat_data excludes_file_stat;
+ uint32_t dir_flags;
+ unsigned char info_exclude_sha1[20];
+ unsigned char excludes_file_sha1[20];
+ char exclude_per_dir[FLEX_ARRAY];
+};
+
+#define ouc_size(len) (offsetof(struct ondisk_untracked_cache, exclude_per_dir) + len + 1)
+
+struct write_data {
+ int index; /* number of written untracked_cache_dir */
+ struct ewah_bitmap *check_only; /* from untracked_cache_dir */
+ struct ewah_bitmap *valid; /* from untracked_cache_dir */
+ struct ewah_bitmap *sha1_valid; /* set if exclude_sha1 is not null */
+ struct strbuf out;
+ struct strbuf sb_stat;
+ struct strbuf sb_sha1;
+};
+
+static void stat_data_to_disk(struct stat_data *to, const struct stat_data *from)
+{
+ to->sd_ctime.sec = htonl(from->sd_ctime.sec);
+ to->sd_ctime.nsec = htonl(from->sd_ctime.nsec);
+ to->sd_mtime.sec = htonl(from->sd_mtime.sec);
+ to->sd_mtime.nsec = htonl(from->sd_mtime.nsec);
+ to->sd_dev = htonl(from->sd_dev);
+ to->sd_ino = htonl(from->sd_ino);
+ to->sd_uid = htonl(from->sd_uid);
+ to->sd_gid = htonl(from->sd_gid);
+ to->sd_size = htonl(from->sd_size);
+}
+
+static void write_one_dir(struct untracked_cache_dir *untracked,
+ struct write_data *wd)
+{
+ struct stat_data stat_data;
+ struct strbuf *out = &wd->out;
+ unsigned char intbuf[16];
+ unsigned int intlen, value;
+ int i = wd->index++;
+
+ /*
+ * untracked_nr should be reset whenever valid is clear, but
+ * for safety..
+ */
+ if (!untracked->valid) {
+ untracked->untracked_nr = 0;
+ untracked->check_only = 0;
+ }
+
+ if (untracked->check_only)
+ ewah_set(wd->check_only, i);
+ if (untracked->valid) {
+ ewah_set(wd->valid, i);
+ stat_data_to_disk(&stat_data, &untracked->stat_data);
+ strbuf_add(&wd->sb_stat, &stat_data, sizeof(stat_data));
+ }
+ if (!is_null_sha1(untracked->exclude_sha1)) {
+ ewah_set(wd->sha1_valid, i);
+ strbuf_add(&wd->sb_sha1, untracked->exclude_sha1, 20);
+ }
+
+ intlen = encode_varint(untracked->untracked_nr, intbuf);
+ strbuf_add(out, intbuf, intlen);
+
+ /* skip non-recurse directories */
+ for (i = 0, value = 0; i < untracked->dirs_nr; i++)
+ if (untracked->dirs[i]->recurse)
+ value++;
+ intlen = encode_varint(value, intbuf);
+ strbuf_add(out, intbuf, intlen);
+
+ strbuf_add(out, untracked->name, strlen(untracked->name) + 1);
+
+ for (i = 0; i < untracked->untracked_nr; i++)
+ strbuf_add(out, untracked->untracked[i],
+ strlen(untracked->untracked[i]) + 1);
+
+ for (i = 0; i < untracked->dirs_nr; i++)
+ if (untracked->dirs[i]->recurse)
+ write_one_dir(untracked->dirs[i], wd);
+}
+
+void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked)
+{
+ struct ondisk_untracked_cache *ouc;
+ struct write_data wd;
+ unsigned char varbuf[16];
+ int len = 0, varint_len;
+ if (untracked->exclude_per_dir)
+ len = strlen(untracked->exclude_per_dir);
+ ouc = xmalloc(sizeof(*ouc) + len + 1);
+ stat_data_to_disk(&ouc->info_exclude_stat, &untracked->ss_info_exclude.stat);
+ stat_data_to_disk(&ouc->excludes_file_stat, &untracked->ss_excludes_file.stat);
+ hashcpy(ouc->info_exclude_sha1, untracked->ss_info_exclude.sha1);
+ hashcpy(ouc->excludes_file_sha1, untracked->ss_excludes_file.sha1);
+ ouc->dir_flags = htonl(untracked->dir_flags);
+ memcpy(ouc->exclude_per_dir, untracked->exclude_per_dir, len + 1);
+ strbuf_add(out, ouc, ouc_size(len));
+ free(ouc);
+ ouc = NULL;
+
+ if (!untracked->root) {
+ varint_len = encode_varint(0, varbuf);
+ strbuf_add(out, varbuf, varint_len);
+ return;
+ }
+
+ wd.index = 0;
+ wd.check_only = ewah_new();
+ wd.valid = ewah_new();
+ wd.sha1_valid = ewah_new();
+ strbuf_init(&wd.out, 1024);
+ strbuf_init(&wd.sb_stat, 1024);
+ strbuf_init(&wd.sb_sha1, 1024);
+ write_one_dir(untracked->root, &wd);
+
+ varint_len = encode_varint(wd.index, varbuf);
+ strbuf_add(out, varbuf, varint_len);
+ strbuf_addbuf(out, &wd.out);
+ ewah_serialize_strbuf(wd.valid, out);
+ ewah_serialize_strbuf(wd.check_only, out);
+ ewah_serialize_strbuf(wd.sha1_valid, out);
+ strbuf_addbuf(out, &wd.sb_stat);
+ strbuf_addbuf(out, &wd.sb_sha1);
+ strbuf_addch(out, '\0'); /* safe guard for string lists */
+
+ ewah_free(wd.valid);
+ ewah_free(wd.check_only);
+ ewah_free(wd.sha1_valid);
+ strbuf_release(&wd.out);
+ strbuf_release(&wd.sb_stat);
+ strbuf_release(&wd.sb_sha1);
+}
+
+static void free_untracked(struct untracked_cache_dir *ucd)
+{
+ int i;
+ if (!ucd)
+ return;
+ for (i = 0; i < ucd->dirs_nr; i++)
+ free_untracked(ucd->dirs[i]);
+ for (i = 0; i < ucd->untracked_nr; i++)
+ free(ucd->untracked[i]);
+ free(ucd->untracked);
+ free(ucd->dirs);
+ free(ucd);
+}
+
+void free_untracked_cache(struct untracked_cache *uc)
+{
+ if (uc)
+ free_untracked(uc->root);
+ free(uc);
+}
+
+struct read_data {
+ int index;
+ struct untracked_cache_dir **ucd;
+ struct ewah_bitmap *check_only;
+ struct ewah_bitmap *valid;
+ struct ewah_bitmap *sha1_valid;
+ const unsigned char *data;
+ const unsigned char *end;
+};
+
+static void stat_data_from_disk(struct stat_data *to, const struct stat_data *from)
+{
+ to->sd_ctime.sec = get_be32(&from->sd_ctime.sec);
+ to->sd_ctime.nsec = get_be32(&from->sd_ctime.nsec);
+ to->sd_mtime.sec = get_be32(&from->sd_mtime.sec);
+ to->sd_mtime.nsec = get_be32(&from->sd_mtime.nsec);
+ to->sd_dev = get_be32(&from->sd_dev);
+ to->sd_ino = get_be32(&from->sd_ino);
+ to->sd_uid = get_be32(&from->sd_uid);
+ to->sd_gid = get_be32(&from->sd_gid);
+ to->sd_size = get_be32(&from->sd_size);
+}
+
+static int read_one_dir(struct untracked_cache_dir **untracked_,
+ struct read_data *rd)
+{
+ struct untracked_cache_dir ud, *untracked;
+ const unsigned char *next, *data = rd->data, *end = rd->end;
+ unsigned int value;
+ int i, len;
+
+ memset(&ud, 0, sizeof(ud));
+
+ next = data;
+ value = decode_varint(&next);
+ if (next > end)
+ return -1;
+ ud.recurse = 1;
+ ud.untracked_alloc = value;
+ ud.untracked_nr = value;
+ if (ud.untracked_nr)
+ ud.untracked = xmalloc(sizeof(*ud.untracked) * ud.untracked_nr);
+ data = next;
+
+ next = data;
+ ud.dirs_alloc = ud.dirs_nr = decode_varint(&next);
+ if (next > end)
+ return -1;
+ ud.dirs = xmalloc(sizeof(*ud.dirs) * ud.dirs_nr);
+ data = next;
+
+ len = strlen((const char *)data);
+ next = data + len + 1;
+ if (next > rd->end)
+ return -1;
+ *untracked_ = untracked = xmalloc(sizeof(*untracked) + len);
+ memcpy(untracked, &ud, sizeof(ud));
+ memcpy(untracked->name, data, len + 1);
+ data = next;
+
+ for (i = 0; i < untracked->untracked_nr; i++) {
+ len = strlen((const char *)data);
+ next = data + len + 1;
+ if (next > rd->end)
+ return -1;
+ untracked->untracked[i] = xstrdup((const char*)data);
+ data = next;
+ }
+
+ rd->ucd[rd->index++] = untracked;
+ rd->data = data;
+
+ for (i = 0; i < untracked->dirs_nr; i++) {
+ len = read_one_dir(untracked->dirs + i, rd);
+ if (len < 0)
+ return -1;
+ }
+ return 0;
+}
+
+static void set_check_only(size_t pos, void *cb)
+{
+ struct read_data *rd = cb;
+ struct untracked_cache_dir *ud = rd->ucd[pos];
+ ud->check_only = 1;
+}
+
+static void read_stat(size_t pos, void *cb)
+{
+ struct read_data *rd = cb;
+ struct untracked_cache_dir *ud = rd->ucd[pos];
+ if (rd->data + sizeof(struct stat_data) > rd->end) {
+ rd->data = rd->end + 1;
+ return;
+ }
+ stat_data_from_disk(&ud->stat_data, (struct stat_data *)rd->data);
+ rd->data += sizeof(struct stat_data);
+ ud->valid = 1;
+}
+
+static void read_sha1(size_t pos, void *cb)
+{
+ struct read_data *rd = cb;
+ struct untracked_cache_dir *ud = rd->ucd[pos];
+ if (rd->data + 20 > rd->end) {
+ rd->data = rd->end + 1;
+ return;
+ }
+ hashcpy(ud->exclude_sha1, rd->data);
+ rd->data += 20;
+}
+
+static void load_sha1_stat(struct sha1_stat *sha1_stat,
+ const struct stat_data *stat,
+ const unsigned char *sha1)
+{
+ stat_data_from_disk(&sha1_stat->stat, stat);
+ hashcpy(sha1_stat->sha1, sha1);
+ sha1_stat->valid = 1;
+}
+
+struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz)
+{
+ const struct ondisk_untracked_cache *ouc;
+ struct untracked_cache *uc;
+ struct read_data rd;
+ const unsigned char *next = data, *end = (const unsigned char *)data + sz;
+ int len;
+
+ if (sz <= 1 || end[-1] != '\0')
+ return NULL;
+ end--;
+
+ ouc = (const struct ondisk_untracked_cache *)next;
+ if (next + ouc_size(0) > end)
+ return NULL;
+
+ uc = xcalloc(1, sizeof(*uc));
+ load_sha1_stat(&uc->ss_info_exclude, &ouc->info_exclude_stat,
+ ouc->info_exclude_sha1);
+ load_sha1_stat(&uc->ss_excludes_file, &ouc->excludes_file_stat,
+ ouc->excludes_file_sha1);
+ uc->dir_flags = get_be32(&ouc->dir_flags);
+ uc->exclude_per_dir = xstrdup(ouc->exclude_per_dir);
+ /* NUL after exclude_per_dir is covered by sizeof(*ouc) */
+ next += ouc_size(strlen(ouc->exclude_per_dir));
+ if (next >= end)
+ goto done2;
+
+ len = decode_varint(&next);
+ if (next > end || len == 0)
+ goto done2;
+
+ rd.valid = ewah_new();
+ rd.check_only = ewah_new();
+ rd.sha1_valid = ewah_new();
+ rd.data = next;
+ rd.end = end;
+ rd.index = 0;
+ rd.ucd = xmalloc(sizeof(*rd.ucd) * len);
+
+ if (read_one_dir(&uc->root, &rd) || rd.index != len)
+ goto done;
+
+ next = rd.data;
+ len = ewah_read_mmap(rd.valid, next, end - next);
+ if (len < 0)
+ goto done;
+
+ next += len;
+ len = ewah_read_mmap(rd.check_only, next, end - next);
+ if (len < 0)
+ goto done;
+
+ next += len;
+ len = ewah_read_mmap(rd.sha1_valid, next, end - next);
+ if (len < 0)
+ goto done;
+
+ ewah_each_bit(rd.check_only, set_check_only, &rd);
+ rd.data = next + len;
+ ewah_each_bit(rd.valid, read_stat, &rd);
+ ewah_each_bit(rd.sha1_valid, read_sha1, &rd);
+ next = rd.data;
+
+done:
+ free(rd.ucd);
+ ewah_free(rd.valid);
+ ewah_free(rd.check_only);
+ ewah_free(rd.sha1_valid);
+done2:
+ if (next != end) {
+ free_untracked_cache(uc);
+ uc = NULL;
+ }
+ return uc;
+}