From: Junio C Hamano Date: Fri, 26 Oct 2018 05:22:10 +0000 (+0900) Subject: Merge branch 'sg/split-index-racefix' X-Git-Tag: v2.20.0-rc0~142 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/7a43ab6fb25924ca018073ef7ef46e16b3156bd0?ds=inline;hp=-c Merge branch 'sg/split-index-racefix' The codepath to support the experimental split-index mode had remaining "racily clean" issues fixed. * sg/split-index-racefix: split-index: BUG() when cache entry refers to non-existing shared entry split-index: smudge and add racily clean cache entries to split index split-index: don't compare cached data of entries already marked for split index split-index: count the number of deleted entries t1700-split-index: date back files to avoid racy situations split-index: add tests to demonstrate the racy split index problem t1700-split-index: document why FSMONITOR is disabled in this test script --- 7a43ab6fb25924ca018073ef7ef46e16b3156bd0 diff --combined cache.h index 59c8a93046,3f419b6c79..f7fabdde8f --- a/cache.h +++ b/cache.h @@@ -410,7 -410,7 +410,7 @@@ void validate_cache_entries(const struc #define read_cache() read_index(&the_index) #define read_cache_from(path) read_index_from(&the_index, (path), (get_git_dir())) -#define read_cache_preload(pathspec) read_index_preload(&the_index, (pathspec)) +#define read_cache_preload(pathspec) read_index_preload(&the_index, (pathspec), 0) #define is_cache_unborn() is_index_unborn(&the_index) #define read_cache_unmerged() read_index_unmerged(&the_index) #define discard_cache() discard_index(&the_index) @@@ -659,9 -659,7 +659,9 @@@ extern int daemonize(void) /* Initialize and use the cache information */ struct lock_file; extern int read_index(struct index_state *); -extern int read_index_preload(struct index_state *, const struct pathspec *pathspec); +extern int read_index_preload(struct index_state *, + const struct pathspec *pathspec, + unsigned int refresh_flags); extern int do_read_index(struct index_state *istate, const char *path, int must_exist); /* for testting only! */ extern int read_index_from(struct index_state *, const char *path, @@@ -705,7 -703,7 +705,7 @@@ extern int unmerged_index(const struct * provided, the space-separated list of files that differ will be appended * to it. */ -extern int index_has_changes(const struct index_state *istate, +extern int index_has_changes(struct index_state *istate, struct tree *tree, struct strbuf *sb); @@@ -783,14 -781,16 +783,16 @@@ extern void *read_blob_data_from_index( #define CE_MATCH_REFRESH 0x10 /* don't refresh_fsmonitor state or do stat comparison even if CE_FSMONITOR_VALID is true */ #define CE_MATCH_IGNORE_FSMONITOR 0X20 + extern int is_racy_timestamp(const struct index_state *istate, + const struct cache_entry *ce); extern int ie_match_stat(struct index_state *, const struct cache_entry *, struct stat *, unsigned int); extern int ie_modified(struct index_state *, const struct cache_entry *, struct stat *, unsigned int); #define HASH_WRITE_OBJECT 1 #define HASH_FORMAT_CHECK 2 #define HASH_RENORMALIZE 4 -extern int index_fd(struct object_id *oid, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags); -extern int index_path(struct object_id *oid, const char *path, struct stat *st, unsigned flags); +extern int index_fd(struct index_state *istate, struct object_id *oid, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags); +extern int index_path(struct index_state *istate, struct object_id *oid, const char *path, struct stat *st, unsigned flags); /* * Record to sd the data from st that we use to check whether a file @@@ -816,7 -816,6 +818,7 @@@ extern void fill_stat_cache_info(struc #define REFRESH_IGNORE_MISSING 0x0008 /* ignore non-existent */ #define REFRESH_IGNORE_SUBMODULES 0x0010 /* ignore submodules */ #define REFRESH_IN_PORCELAIN 0x0020 /* user friendly output, not "needs update" */ +#define REFRESH_PROGRESS 0x0040 /* show progress bar if stderr is tty */ extern int refresh_index(struct index_state *, unsigned int flags, const struct pathspec *pathspec, char *seen, const char *header_msg); extern struct cache_entry *refresh_cache_entry(struct index_state *, struct cache_entry *, unsigned int); @@@ -1044,24 -1043,14 +1046,24 @@@ static inline int oidcmp(const struct o return hashcmp(oid1->hash, oid2->hash); } +static inline int hasheq(const unsigned char *sha1, const unsigned char *sha2) +{ + return !hashcmp(sha1, sha2); +} + +static inline int oideq(const struct object_id *oid1, const struct object_id *oid2) +{ + return hasheq(oid1->hash, oid2->hash); +} + static inline int is_null_sha1(const unsigned char *sha1) { - return !hashcmp(sha1, null_sha1); + return hasheq(sha1, null_sha1); } static inline int is_null_oid(const struct object_id *oid) { - return !hashcmp(oid->hash, null_sha1); + return hasheq(oid->hash, null_sha1); } static inline void hashcpy(unsigned char *sha_dst, const unsigned char *sha_src) @@@ -1098,22 -1087,22 +1100,22 @@@ static inline void oidread(struct objec static inline int is_empty_blob_sha1(const unsigned char *sha1) { - return !hashcmp(sha1, the_hash_algo->empty_blob->hash); + return hasheq(sha1, the_hash_algo->empty_blob->hash); } static inline int is_empty_blob_oid(const struct object_id *oid) { - return !oidcmp(oid, the_hash_algo->empty_blob); + return oideq(oid, the_hash_algo->empty_blob); } static inline int is_empty_tree_sha1(const unsigned char *sha1) { - return !hashcmp(sha1, the_hash_algo->empty_tree->hash); + return hasheq(sha1, the_hash_algo->empty_tree->hash); } static inline int is_empty_tree_oid(const struct object_id *oid) { - return !oidcmp(oid, the_hash_algo->empty_tree); + return oideq(oid, the_hash_algo->empty_tree); } const char *empty_tree_oid_hex(void); @@@ -1531,7 -1520,6 +1533,7 @@@ struct checkout unsigned force:1, quiet:1, not_new:1, + clone:1, refresh_cache:1; }; #define CHECKOUT_INIT { NULL, "" } @@@ -1708,7 -1696,7 +1710,7 @@@ void shift_tree_by(const struct object_ /* All WS_* -- when extended, adapt diff.c emit_symbol */ #define WS_RULE_MASK 07777 extern unsigned whitespace_rule_cfg; -extern unsigned whitespace_rule(const char *); +extern unsigned whitespace_rule(struct index_state *, const char *); extern unsigned parse_whitespace_rule(const char *); extern unsigned ws_check(const char *line, int len, unsigned ws_rule); extern void ws_check_emit(const char *line, int len, unsigned ws_rule, FILE *stream, const char *set, const char *reset, const char *ws); @@@ -1730,12 -1718,10 +1732,12 @@@ extern struct startup_info *startup_inf /* merge.c */ struct commit_list; -int try_merge_command(const char *strategy, size_t xopts_nr, +int try_merge_command(struct repository *r, + const char *strategy, size_t xopts_nr, const char **xopts, struct commit_list *common, const char *head_arg, struct commit_list *remotes); -int checkout_fast_forward(const struct object_id *from, +int checkout_fast_forward(struct repository *r, + const struct object_id *from, const struct object_id *to, int overwrite_ignore); diff --combined read-cache.c index 1df5c16dbc,8f644f68b4..d57958233e --- a/read-cache.c +++ b/read-cache.c @@@ -23,8 -23,6 +23,8 @@@ #include "split-index.h" #include "utf8.h" #include "fsmonitor.h" +#include "thread-utils.h" +#include "progress.h" /* Mask for the name length in ce_flags in the on-disk index */ @@@ -45,8 -43,6 +45,8 @@@ #define CACHE_EXT_LINK 0x6c696e6b /* "link" */ #define CACHE_EXT_UNTRACKED 0x554E5452 /* "UNTR" */ #define CACHE_EXT_FSMONITOR 0x46534D4E /* "FSMN" */ +#define CACHE_EXT_ENDOFINDEXENTRIES 0x454F4945 /* "EOIE" */ +#define CACHE_EXT_INDEXENTRYOFFSETTABLE 0x49454F54 /* "IEOT" */ /* changes that can be kept in $GIT_DIR/index (basically all extensions) */ #define EXTMASK (RESOLVE_UNDO_CHANGED | CACHE_TREE_CHANGED | \ @@@ -209,17 -205,15 +209,17 @@@ void fill_stat_cache_info(struct cache_ } } -static int ce_compare_data(const struct cache_entry *ce, struct stat *st) +static int ce_compare_data(struct index_state *istate, + const struct cache_entry *ce, + struct stat *st) { int match = -1; int fd = git_open_cloexec(ce->name, O_RDONLY); if (fd >= 0) { struct object_id oid; - if (!index_fd(&oid, fd, st, OBJ_BLOB, ce->name, 0)) - match = oidcmp(&oid, &ce->oid); + if (!index_fd(istate, &oid, fd, st, OBJ_BLOB, ce->name, 0)) + match = !oideq(&oid, &ce->oid); /* index_fd() closed the file descriptor already */ } return match; @@@ -260,16 -254,14 +260,16 @@@ static int ce_compare_gitlink(const str */ if (resolve_gitlink_ref(ce->name, "HEAD", &oid) < 0) return 0; - return oidcmp(&oid, &ce->oid); + return !oideq(&oid, &ce->oid); } -static int ce_modified_check_fs(const struct cache_entry *ce, struct stat *st) +static int ce_modified_check_fs(struct index_state *istate, + const struct cache_entry *ce, + struct stat *st) { switch (st->st_mode & S_IFMT) { case S_IFREG: - if (ce_compare_data(ce, st)) + if (ce_compare_data(istate, ce, st)) return DATA_CHANGED; break; case S_IFLNK: @@@ -345,7 -337,7 +345,7 @@@ static int is_racy_stat(const struct in ); } - static int is_racy_timestamp(const struct index_state *istate, + int is_racy_timestamp(const struct index_state *istate, const struct cache_entry *ce) { return (!S_ISGITLINK(ce->ce_mode) && @@@ -415,7 -407,7 +415,7 @@@ int ie_match_stat(struct index_state *i if (assume_racy_is_modified) changed |= DATA_CHANGED; else - changed |= ce_modified_check_fs(ce, st); + changed |= ce_modified_check_fs(istate, ce, st); } return changed; @@@ -455,7 -447,7 +455,7 @@@ int ie_modified(struct index_state *ist (S_ISGITLINK(ce->ce_mode) || ce->ce_stat_data.sd_size != 0)) return changed; - changed_fs = ce_modified_check_fs(ce, st); + changed_fs = ce_modified_check_fs(istate, ce, st); if (changed_fs) return changed | changed_fs; return 0; @@@ -761,7 -753,7 +761,7 @@@ int add_to_index(struct index_state *is } } if (!intent_only) { - if (index_path(&ce->oid, path, st, newflags)) { + if (index_path(istate, &ce->oid, path, st, newflags)) { discard_cache_entry(ce); return error("unable to index file %s", path); } @@@ -775,7 -767,7 +775,7 @@@ /* It was suspected to be racily clean, but it turns out to be Ok */ was_same = (alias && !ce_stage(alias) && - !oidcmp(&alias->oid, &ce->oid) && + oideq(&alias->oid, &ce->oid) && ce->ce_mode == alias->ce_mode); if (pretend) @@@ -831,7 -823,7 +831,7 @@@ struct cache_entry *make_cache_entry(st ce->ce_namelen = len; ce->ce_mode = create_ce_mode(mode); - ret = refresh_cache_entry(&the_index, ce, refresh_options); + ret = refresh_cache_entry(istate, ce, refresh_options); if (ret != ce) discard_cache_entry(ce); return ret; @@@ -1484,13 -1476,8 +1484,13 @@@ int refresh_index(struct index_state *i const char *typechange_fmt; const char *added_fmt; const char *unmerged_fmt; - uint64_t start = getnanotime(); + struct progress *progress = NULL; + + if (flags & REFRESH_PROGRESS && isatty(2)) + progress = start_delayed_progress(_("Refresh index"), + istate->cache_nr); + trace_performance_enter(); modified_fmt = (in_porcelain ? "M\t%s\n" : "%s: needs update\n"); deleted_fmt = (in_porcelain ? "D\t%s\n" : "%s: needs update\n"); typechange_fmt = (in_porcelain ? "T\t%s\n" : "%s needs update\n"); @@@ -1506,7 -1493,7 +1506,7 @@@ if (ignore_submodules && S_ISGITLINK(ce->ce_mode)) continue; - if (pathspec && !ce_path_match(&the_index, ce, pathspec, seen)) + if (pathspec && !ce_path_match(istate, ce, pathspec, seen)) filtered = 1; if (ce_stage(ce)) { @@@ -1529,8 -1516,6 +1529,8 @@@ new_entry = refresh_cache_ent(istate, ce, options, &cache_errno, &changed); if (new_entry == ce) continue; + if (progress) + display_progress(progress, i); if (!new_entry) { const char *fmt; @@@ -1562,11 -1547,7 +1562,11 @@@ replace_index_entry(istate, i, new_entry); } - trace_performance_since(start, "refresh index"); + if (progress) { + display_progress(progress, istate->cache_nr); + stop_progress(&progress); + } + trace_performance_leave("refresh index"); return has_errors; } @@@ -1669,7 -1650,7 +1669,7 @@@ int verify_index_checksum /* Allow fsck to force verification of the cache entry order. */ int verify_ce_order; -static int verify_hdr(struct cache_header *hdr, unsigned long size) +static int verify_hdr(const struct cache_header *hdr, unsigned long size) { git_hash_ctx c; unsigned char hash[GIT_MAX_RAWSZ]; @@@ -1687,13 -1668,13 +1687,13 @@@ the_hash_algo->init_fn(&c); the_hash_algo->update_fn(&c, hdr, size - the_hash_algo->rawsz); the_hash_algo->final_fn(hash, &c); - if (hashcmp(hash, (unsigned char *)hdr + size - the_hash_algo->rawsz)) + if (!hasheq(hash, (unsigned char *)hdr + size - the_hash_algo->rawsz)) return error("bad index file sha1 signature"); return 0; } static int read_index_extension(struct index_state *istate, - const char *ext, void *data, unsigned long sz) + const char *ext, const char *data, unsigned long sz) { switch (CACHE_EXT(ext)) { case CACHE_EXT_TREE: @@@ -1712,10 -1693,6 +1712,10 @@@ case CACHE_EXT_FSMONITOR: read_fsmonitor_extension(istate, data, sz); break; + case CACHE_EXT_ENDOFINDEXENTRIES: + case CACHE_EXT_INDEXENTRYOFFSETTABLE: + /* already handled in do_read_index() */ + break; default: if (*ext < 'A' || 'Z' < *ext) return error("index uses %.4s extension, which we do not understand", @@@ -1736,25 -1713,63 +1736,25 @@@ int read_index(struct index_state *ista return read_index_from(istate, get_index_file(), get_git_dir()); } -static struct cache_entry *cache_entry_from_ondisk(struct mem_pool *mem_pool, - struct ondisk_cache_entry *ondisk, - unsigned int flags, - const char *name, - size_t len) -{ - struct cache_entry *ce = mem_pool__ce_alloc(mem_pool, len); - - ce->ce_stat_data.sd_ctime.sec = get_be32(&ondisk->ctime.sec); - ce->ce_stat_data.sd_mtime.sec = get_be32(&ondisk->mtime.sec); - ce->ce_stat_data.sd_ctime.nsec = get_be32(&ondisk->ctime.nsec); - ce->ce_stat_data.sd_mtime.nsec = get_be32(&ondisk->mtime.nsec); - ce->ce_stat_data.sd_dev = get_be32(&ondisk->dev); - ce->ce_stat_data.sd_ino = get_be32(&ondisk->ino); - ce->ce_mode = get_be32(&ondisk->mode); - ce->ce_stat_data.sd_uid = get_be32(&ondisk->uid); - ce->ce_stat_data.sd_gid = get_be32(&ondisk->gid); - ce->ce_stat_data.sd_size = get_be32(&ondisk->size); - ce->ce_flags = flags & ~CE_NAMEMASK; - ce->ce_namelen = len; - ce->index = 0; - hashcpy(ce->oid.hash, ondisk->sha1); - memcpy(ce->name, name, len); - ce->name[len] = '\0'; - return ce; -} - -/* - * Adjacent cache entries tend to share the leading paths, so it makes - * sense to only store the differences in later entries. In the v4 - * on-disk format of the index, each on-disk cache entry stores the - * number of bytes to be stripped from the end of the previous name, - * and the bytes to append to the result, to come up with its name. - */ -static unsigned long expand_name_field(struct strbuf *name, const char *cp_) -{ - const unsigned char *ep, *cp = (const unsigned char *)cp_; - size_t len = decode_varint(&cp); - - if (name->len < len) - die("malformed name field in the index"); - strbuf_remove(name, name->len - len, len); - for (ep = cp; *ep; ep++) - ; /* find the end */ - strbuf_add(name, cp, ep - cp); - return (const char *)ep + 1 - cp_; -} - -static struct cache_entry *create_from_disk(struct mem_pool *mem_pool, +static struct cache_entry *create_from_disk(struct mem_pool *ce_mem_pool, + unsigned int version, struct ondisk_cache_entry *ondisk, unsigned long *ent_size, - struct strbuf *previous_name) + const struct cache_entry *previous_ce) { struct cache_entry *ce; size_t len; const char *name; unsigned int flags; + size_t copy_len; + /* + * Adjacent cache entries tend to share the leading paths, so it makes + * sense to only store the differences in later entries. In the v4 + * on-disk format of the index, each on-disk cache entry stores the + * number of bytes to be stripped from the end of the previous name, + * and the bytes to append to the result, to come up with its name. + */ + int expand_name_field = version == 4; /* On-disk flags are just 16 bits */ flags = get_be16(&ondisk->flags); @@@ -1774,55 -1789,21 +1774,55 @@@ else name = ondisk->name; - if (!previous_name) { - /* v3 and earlier */ - if (len == CE_NAMEMASK) - len = strlen(name); - ce = cache_entry_from_ondisk(mem_pool, ondisk, flags, name, len); + if (expand_name_field) { + const unsigned char *cp = (const unsigned char *)name; + size_t strip_len, previous_len; + + /* If we're at the begining of a block, ignore the previous name */ + strip_len = decode_varint(&cp); + if (previous_ce) { + previous_len = previous_ce->ce_namelen; + if (previous_len < strip_len) + die(_("malformed name field in the index, near path '%s'"), + previous_ce->name); + copy_len = previous_len - strip_len; + } else { + copy_len = 0; + } + name = (const char *)cp; + } - *ent_size = ondisk_ce_size(ce); - } else { - unsigned long consumed; - consumed = expand_name_field(previous_name, name); - ce = cache_entry_from_ondisk(mem_pool, ondisk, flags, - previous_name->buf, - previous_name->len); + if (len == CE_NAMEMASK) { + len = strlen(name); + if (expand_name_field) + len += copy_len; + } - *ent_size = (name - ((char *)ondisk)) + consumed; + ce = mem_pool__ce_alloc(ce_mem_pool, len); + + ce->ce_stat_data.sd_ctime.sec = get_be32(&ondisk->ctime.sec); + ce->ce_stat_data.sd_mtime.sec = get_be32(&ondisk->mtime.sec); + ce->ce_stat_data.sd_ctime.nsec = get_be32(&ondisk->ctime.nsec); + ce->ce_stat_data.sd_mtime.nsec = get_be32(&ondisk->mtime.nsec); + ce->ce_stat_data.sd_dev = get_be32(&ondisk->dev); + ce->ce_stat_data.sd_ino = get_be32(&ondisk->ino); + ce->ce_mode = get_be32(&ondisk->mode); + ce->ce_stat_data.sd_uid = get_be32(&ondisk->uid); + ce->ce_stat_data.sd_gid = get_be32(&ondisk->gid); + ce->ce_stat_data.sd_size = get_be32(&ondisk->size); + ce->ce_flags = flags & ~CE_NAMEMASK; + ce->ce_namelen = len; + ce->index = 0; + hashcpy(ce->oid.hash, ondisk->sha1); + + if (expand_name_field) { + if (copy_len) + memcpy(ce->name, previous_ce->name, copy_len); + memcpy(ce->name + copy_len, name, len + 1 - copy_len); + *ent_size = (name - ((char *)ondisk)) + len + 1 - copy_len; + } else { + memcpy(ce->name, name, len + 1); + *ent_size = ondisk_ce_size(ce); } return ce; } @@@ -1908,237 -1889,16 +1908,237 @@@ static size_t estimate_cache_size(size_ return ondisk_size + entries * per_entry; } +struct index_entry_offset +{ + /* starting byte offset into index file, count of index entries in this block */ + int offset, nr; +}; + +struct index_entry_offset_table +{ + int nr; + struct index_entry_offset entries[FLEX_ARRAY]; +}; + +#ifndef NO_PTHREADS +static struct index_entry_offset_table *read_ieot_extension(const char *mmap, size_t mmap_size, size_t offset); +static void write_ieot_extension(struct strbuf *sb, struct index_entry_offset_table *ieot); +#endif + +static size_t read_eoie_extension(const char *mmap, size_t mmap_size); +static void write_eoie_extension(struct strbuf *sb, git_hash_ctx *eoie_context, size_t offset); + +struct load_index_extensions +{ +#ifndef NO_PTHREADS + pthread_t pthread; +#endif + struct index_state *istate; + const char *mmap; + size_t mmap_size; + unsigned long src_offset; +}; + +static void *load_index_extensions(void *_data) +{ + struct load_index_extensions *p = _data; + unsigned long src_offset = p->src_offset; + + while (src_offset <= p->mmap_size - the_hash_algo->rawsz - 8) { + /* After an array of active_nr index entries, + * there can be arbitrary number of extended + * sections, each of which is prefixed with + * extension name (4-byte) and section length + * in 4-byte network byte order. + */ + uint32_t extsize = get_be32(p->mmap + src_offset + 4); + if (read_index_extension(p->istate, + p->mmap + src_offset, + p->mmap + src_offset + 8, + extsize) < 0) { + munmap((void *)p->mmap, p->mmap_size); + die(_("index file corrupt")); + } + src_offset += 8; + src_offset += extsize; + } + + return NULL; +} + +/* + * A helper function that will load the specified range of cache entries + * from the memory mapped file and add them to the given index. + */ +static unsigned long load_cache_entry_block(struct index_state *istate, + struct mem_pool *ce_mem_pool, int offset, int nr, const char *mmap, + unsigned long start_offset, const struct cache_entry *previous_ce) +{ + int i; + unsigned long src_offset = start_offset; + + for (i = offset; i < offset + nr; i++) { + struct ondisk_cache_entry *disk_ce; + struct cache_entry *ce; + unsigned long consumed; + + disk_ce = (struct ondisk_cache_entry *)(mmap + src_offset); + ce = create_from_disk(ce_mem_pool, istate->version, disk_ce, &consumed, previous_ce); + set_index_entry(istate, i, ce); + + src_offset += consumed; + previous_ce = ce; + } + return src_offset - start_offset; +} + +static unsigned long load_all_cache_entries(struct index_state *istate, + const char *mmap, size_t mmap_size, unsigned long src_offset) +{ + unsigned long consumed; + + if (istate->version == 4) { + mem_pool_init(&istate->ce_mem_pool, + estimate_cache_size_from_compressed(istate->cache_nr)); + } else { + mem_pool_init(&istate->ce_mem_pool, + estimate_cache_size(mmap_size, istate->cache_nr)); + } + + consumed = load_cache_entry_block(istate, istate->ce_mem_pool, + 0, istate->cache_nr, mmap, src_offset, NULL); + return consumed; +} + +#ifndef NO_PTHREADS + +/* + * Mostly randomly chosen maximum thread counts: we + * cap the parallelism to online_cpus() threads, and we want + * to have at least 10000 cache entries per thread for it to + * be worth starting a thread. + */ + +#define THREAD_COST (10000) + +struct load_cache_entries_thread_data +{ + pthread_t pthread; + struct index_state *istate; + struct mem_pool *ce_mem_pool; + int offset; + const char *mmap; + struct index_entry_offset_table *ieot; + int ieot_start; /* starting index into the ieot array */ + int ieot_blocks; /* count of ieot entries to process */ + unsigned long consumed; /* return # of bytes in index file processed */ +}; + +/* + * A thread proc to run the load_cache_entries() computation + * across multiple background threads. + */ +static void *load_cache_entries_thread(void *_data) +{ + struct load_cache_entries_thread_data *p = _data; + int i; + + /* iterate across all ieot blocks assigned to this thread */ + for (i = p->ieot_start; i < p->ieot_start + p->ieot_blocks; i++) { + p->consumed += load_cache_entry_block(p->istate, p->ce_mem_pool, + p->offset, p->ieot->entries[i].nr, p->mmap, p->ieot->entries[i].offset, NULL); + p->offset += p->ieot->entries[i].nr; + } + return NULL; +} + +static unsigned long load_cache_entries_threaded(struct index_state *istate, const char *mmap, size_t mmap_size, + unsigned long src_offset, int nr_threads, struct index_entry_offset_table *ieot) +{ + int i, offset, ieot_blocks, ieot_start, err; + struct load_cache_entries_thread_data *data; + unsigned long consumed = 0; + + /* a little sanity checking */ + if (istate->name_hash_initialized) + BUG("the name hash isn't thread safe"); + + mem_pool_init(&istate->ce_mem_pool, 0); + + /* ensure we have no more threads than we have blocks to process */ + if (nr_threads > ieot->nr) + nr_threads = ieot->nr; + data = xcalloc(nr_threads, sizeof(*data)); + + offset = ieot_start = 0; + ieot_blocks = DIV_ROUND_UP(ieot->nr, nr_threads); + for (i = 0; i < nr_threads; i++) { + struct load_cache_entries_thread_data *p = &data[i]; + int nr, j; + + if (ieot_start + ieot_blocks > ieot->nr) + ieot_blocks = ieot->nr - ieot_start; + + p->istate = istate; + p->offset = offset; + p->mmap = mmap; + p->ieot = ieot; + p->ieot_start = ieot_start; + p->ieot_blocks = ieot_blocks; + + /* create a mem_pool for each thread */ + nr = 0; + for (j = p->ieot_start; j < p->ieot_start + p->ieot_blocks; j++) + nr += p->ieot->entries[j].nr; + if (istate->version == 4) { + mem_pool_init(&p->ce_mem_pool, + estimate_cache_size_from_compressed(nr)); + } else { + mem_pool_init(&p->ce_mem_pool, + estimate_cache_size(mmap_size, nr)); + } + + err = pthread_create(&p->pthread, NULL, load_cache_entries_thread, p); + if (err) + die(_("unable to create load_cache_entries thread: %s"), strerror(err)); + + /* increment by the number of cache entries in the ieot block being processed */ + for (j = 0; j < ieot_blocks; j++) + offset += ieot->entries[ieot_start + j].nr; + ieot_start += ieot_blocks; + } + + for (i = 0; i < nr_threads; i++) { + struct load_cache_entries_thread_data *p = &data[i]; + + err = pthread_join(p->pthread, NULL); + if (err) + die(_("unable to join load_cache_entries thread: %s"), strerror(err)); + mem_pool_combine(istate->ce_mem_pool, p->ce_mem_pool); + consumed += p->consumed; + } + + free(data); + + return consumed; +} +#endif + /* remember to discard_cache() before reading a different cache! */ int do_read_index(struct index_state *istate, const char *path, int must_exist) { - int fd, i; + int fd; struct stat st; unsigned long src_offset; - struct cache_header *hdr; - void *mmap; + const struct cache_header *hdr; + const char *mmap; size_t mmap_size; - struct strbuf previous_name_buf = STRBUF_INIT, *previous_name; + struct load_index_extensions p; + size_t extension_offset = 0; +#ifndef NO_PTHREADS + int nr_threads, cpus; + struct index_entry_offset_table *ieot = NULL; +#endif if (istate->initialized) return istate->cache_nr; @@@ -2164,7 -1924,7 +2164,7 @@@ die_errno("unable to map index file"); close(fd); - hdr = mmap; + hdr = (const struct cache_header *)mmap; if (verify_hdr(hdr, mmap_size) < 0) goto unmap; @@@ -2175,74 -1935,55 +2175,74 @@@ istate->cache = xcalloc(istate->cache_alloc, sizeof(*istate->cache)); istate->initialized = 1; - if (istate->version == 4) { - previous_name = &previous_name_buf; - mem_pool_init(&istate->ce_mem_pool, - estimate_cache_size_from_compressed(istate->cache_nr)); - } else { - previous_name = NULL; - mem_pool_init(&istate->ce_mem_pool, - estimate_cache_size(mmap_size, istate->cache_nr)); - } + p.istate = istate; + p.mmap = mmap; + p.mmap_size = mmap_size; src_offset = sizeof(*hdr); - for (i = 0; i < istate->cache_nr; i++) { - struct ondisk_cache_entry *disk_ce; - struct cache_entry *ce; - unsigned long consumed; - disk_ce = (struct ondisk_cache_entry *)((char *)mmap + src_offset); - ce = create_from_disk(istate->ce_mem_pool, disk_ce, &consumed, previous_name); - set_index_entry(istate, i, ce); +#ifndef NO_PTHREADS + nr_threads = git_config_get_index_threads(); - src_offset += consumed; + /* TODO: does creating more threads than cores help? */ + if (!nr_threads) { + nr_threads = istate->cache_nr / THREAD_COST; + cpus = online_cpus(); + if (nr_threads > cpus) + nr_threads = cpus; } - strbuf_release(&previous_name_buf); + + if (nr_threads > 1) { + extension_offset = read_eoie_extension(mmap, mmap_size); + if (extension_offset) { + int err; + + p.src_offset = extension_offset; + err = pthread_create(&p.pthread, NULL, load_index_extensions, &p); + if (err) + die(_("unable to create load_index_extensions thread: %s"), strerror(err)); + + nr_threads--; + } + } + + /* + * Locate and read the index entry offset table so that we can use it + * to multi-thread the reading of the cache entries. + */ + if (extension_offset && nr_threads > 1) + ieot = read_ieot_extension(mmap, mmap_size, extension_offset); + + if (ieot) { + src_offset += load_cache_entries_threaded(istate, mmap, mmap_size, src_offset, nr_threads, ieot); + free(ieot); + } else { + src_offset += load_all_cache_entries(istate, mmap, mmap_size, src_offset); + } +#else + src_offset += load_all_cache_entries(istate, mmap, mmap_size, src_offset); +#endif + istate->timestamp.sec = st.st_mtime; istate->timestamp.nsec = ST_MTIME_NSEC(st); - while (src_offset <= mmap_size - the_hash_algo->rawsz - 8) { - /* After an array of active_nr index entries, - * there can be arbitrary number of extended - * sections, each of which is prefixed with - * extension name (4-byte) and section length - * in 4-byte network byte order. - */ - uint32_t extsize; - memcpy(&extsize, (char *)mmap + src_offset + 4, 4); - extsize = ntohl(extsize); - if (read_index_extension(istate, - (const char *) mmap + src_offset, - (char *) mmap + src_offset + 8, - extsize) < 0) - goto unmap; - src_offset += 8; - src_offset += extsize; + /* if we created a thread, join it otherwise load the extensions on the primary thread */ +#ifndef NO_PTHREADS + if (extension_offset) { + int ret = pthread_join(p.pthread, NULL); + if (ret) + die(_("unable to join load_index_extensions thread: %s"), strerror(ret)); + } +#endif + if (!extension_offset) { + p.src_offset = src_offset; + load_index_extensions(&p); } - munmap(mmap, mmap_size); + munmap((void *)mmap, mmap_size); return istate->cache_nr; unmap: - munmap(mmap, mmap_size); + munmap((void *)mmap, mmap_size); die("index file corrupt"); } @@@ -2261,6 -2002,7 +2261,6 @@@ static void freshen_shared_index(const int read_index_from(struct index_state *istate, const char *path, const char *gitdir) { - uint64_t start = getnanotime(); struct split_index *split_index; int ret; char *base_oid_hex; @@@ -2270,9 -2012,8 +2270,9 @@@ if (istate->initialized) return istate->cache_nr; + trace_performance_enter(); ret = do_read_index(istate, path, 0); - trace_performance_since(start, "read cache %s", path); + trace_performance_leave("read cache %s", path); split_index = istate->split_index; if (!split_index || is_null_oid(&split_index->base_oid)) { @@@ -2280,7 -2021,6 +2280,7 @@@ return ret; } + trace_performance_enter(); if (split_index->base) discard_index(split_index->base); else @@@ -2289,7 -2029,7 +2289,7 @@@ base_oid_hex = oid_to_hex(&split_index->base_oid); base_path = xstrfmt("%s/sharedindex.%s", gitdir, base_oid_hex); ret = do_read_index(split_index->base, base_path, 1); - if (oidcmp(&split_index->base_oid, &split_index->base->oid)) + if (!oideq(&split_index->base_oid, &split_index->base->oid)) die("broken index, expect %s in %s, got %s", base_oid_hex, base_path, oid_to_hex(&split_index->base->oid)); @@@ -2297,8 -2037,8 +2297,8 @@@ freshen_shared_index(base_path, 0); merge_base_index(istate); post_read_index_from(istate); - trace_performance_since(start, "read cache %s", base_path); free(base_path); + trace_performance_leave("read cache %s", base_path); return ret; } @@@ -2382,7 -2122,7 +2382,7 @@@ int unmerged_index(const struct index_s return 0; } -int index_has_changes(const struct index_state *istate, +int index_has_changes(struct index_state *istate, struct tree *tree, struct strbuf *sb) { @@@ -2397,7 -2137,7 +2397,7 @@@ if (tree || !get_oid_tree("HEAD", &cmp)) { struct diff_options opt; - diff_setup(&opt); + repo_diff_setup(the_repository, &opt); opt.flags.exit_with_status = 1; if (!sb) opt.flags.quick = 1; @@@ -2458,15 -2198,11 +2458,15 @@@ static int ce_write(git_hash_ctx *conte return 0; } -static int write_index_ext_header(git_hash_ctx *context, int fd, - unsigned int ext, unsigned int sz) +static int write_index_ext_header(git_hash_ctx *context, git_hash_ctx *eoie_context, + int fd, unsigned int ext, unsigned int sz) { ext = htonl(ext); sz = htonl(sz); + if (eoie_context) { + the_hash_algo->update_fn(eoie_context, &ext, 4); + the_hash_algo->update_fn(eoie_context, &sz, 4); + } return ((ce_write(context, fd, &ext, 4) < 0) || (ce_write(context, fd, &sz, 4) < 0)) ? -1 : 0; } @@@ -2494,8 -2230,7 +2494,8 @@@ static int ce_flush(git_hash_ctx *conte return (write_in_full(fd, write_buffer, left) < 0) ? -1 : 0; } -static void ce_smudge_racily_clean_entry(struct cache_entry *ce) +static void ce_smudge_racily_clean_entry(struct index_state *istate, + struct cache_entry *ce) { /* * The only thing we care about in this function is to smudge the @@@ -2514,7 -2249,7 +2514,7 @@@ return; if (ce_match_stat_basic(ce, &st)) return; - if (ce_modified_check_fs(ce, &st)) { + if (ce_modified_check_fs(istate, ce, &st)) { /* This is "racily clean"; smudge it. Note that this * is a tricky code. At first glance, it may appear * that it can break with this sequence: @@@ -2660,7 -2395,7 +2660,7 @@@ static int verify_index_from(const stru if (n != the_hash_algo->rawsz) goto out; - if (hashcmp(istate->oid.hash, hash)) + if (!hasheq(istate->oid.hash, hash)) goto out; close(fd); @@@ -2710,7 -2445,7 +2710,7 @@@ static int do_write_index(struct index_ { uint64_t start = getnanotime(); int newfd = tempfile->fd; - git_hash_ctx c; + git_hash_ctx c, eoie_c; struct cache_header hdr; int i, err = 0, removed, extended, hdr_version; struct cache_entry **cache = istate->cache; @@@ -2719,10 -2454,6 +2719,10 @@@ struct ondisk_cache_entry_extended ondisk; struct strbuf previous_name_buf = STRBUF_INIT, *previous_name; int drop_cache_tree = istate->drop_cache_tree; + off_t offset; + int ieot_entries = 1; + struct index_entry_offset_table *ieot = NULL; + int nr, nr_threads; for (i = removed = extended = 0; i < entries; i++) { if (cache[i]->ce_flags & CE_REMOVE) @@@ -2756,46 -2487,6 +2756,46 @@@ if (ce_write(&c, newfd, &hdr, sizeof(hdr)) < 0) return -1; +#ifndef NO_PTHREADS + nr_threads = git_config_get_index_threads(); + if (nr_threads != 1) { + int ieot_blocks, cpus; + + /* + * ensure default number of ieot blocks maps evenly to the + * default number of threads that will process them leaving + * room for the thread to load the index extensions. + */ + if (!nr_threads) { + ieot_blocks = istate->cache_nr / THREAD_COST; + cpus = online_cpus(); + if (ieot_blocks > cpus - 1) + ieot_blocks = cpus - 1; + } else { + ieot_blocks = nr_threads; + if (ieot_blocks > istate->cache_nr) + ieot_blocks = istate->cache_nr; + } + + /* + * no reason to write out the IEOT extension if we don't + * have enough blocks to utilize multi-threading + */ + if (ieot_blocks > 1) { + ieot = xcalloc(1, sizeof(struct index_entry_offset_table) + + (ieot_blocks * sizeof(struct index_entry_offset))); + ieot_entries = DIV_ROUND_UP(entries, ieot_blocks); + } + } +#endif + + offset = lseek(newfd, 0, SEEK_CUR); + if (offset < 0) { + free(ieot); + return -1; + } + offset += write_buffer_len; + nr = 0; previous_name = (hdr_version == 4) ? &previous_name_buf : NULL; for (i = 0; i < entries; i++) { @@@ -2803,7 -2494,7 +2803,7 @@@ if (ce->ce_flags & CE_REMOVE) continue; if (!ce_uptodate(ce) && is_racy_timestamp(istate, ce)) - ce_smudge_racily_clean_entry(ce); + ce_smudge_racily_clean_entry(istate, ce); if (is_null_oid(&ce->oid)) { static const char msg[] = "cache entry has null sha1: %s"; static int allow = -1; @@@ -2817,79 -2508,23 +2817,79 @@@ drop_cache_tree = 1; } + if (ieot && i && (i % ieot_entries == 0)) { + ieot->entries[ieot->nr].nr = nr; + ieot->entries[ieot->nr].offset = offset; + ieot->nr++; + /* + * If we have a V4 index, set the first byte to an invalid + * character to ensure there is nothing common with the previous + * entry + */ + if (previous_name) + previous_name->buf[0] = 0; + nr = 0; + offset = lseek(newfd, 0, SEEK_CUR); + if (offset < 0) { + free(ieot); + return -1; + } + offset += write_buffer_len; + } if (ce_write_entry(&c, newfd, ce, previous_name, (struct ondisk_cache_entry *)&ondisk) < 0) err = -1; if (err) break; + nr++; + } + if (ieot && nr) { + ieot->entries[ieot->nr].nr = nr; + ieot->entries[ieot->nr].offset = offset; + ieot->nr++; } strbuf_release(&previous_name_buf); - if (err) + if (err) { + free(ieot); return err; + } /* Write extension data here */ + offset = lseek(newfd, 0, SEEK_CUR); + if (offset < 0) { + free(ieot); + return -1; + } + offset += write_buffer_len; + the_hash_algo->init_fn(&eoie_c); + + /* + * Lets write out CACHE_EXT_INDEXENTRYOFFSETTABLE first so that we + * can minimize the number of extensions we have to scan through to + * find it during load. Write it out regardless of the + * strip_extensions parameter as we need it when loading the shared + * index. + */ +#ifndef NO_PTHREADS + if (ieot) { + struct strbuf sb = STRBUF_INIT; + + write_ieot_extension(&sb, ieot); + err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_INDEXENTRYOFFSETTABLE, sb.len) < 0 + || ce_write(&c, newfd, sb.buf, sb.len) < 0; + strbuf_release(&sb); + free(ieot); + if (err) + return -1; + } +#endif + if (!strip_extensions && istate->split_index) { struct strbuf sb = STRBUF_INIT; err = write_link_extension(&sb, istate) < 0 || - write_index_ext_header(&c, newfd, CACHE_EXT_LINK, + write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_LINK, sb.len) < 0 || ce_write(&c, newfd, sb.buf, sb.len) < 0; strbuf_release(&sb); @@@ -2900,7 -2535,7 +2900,7 @@@ struct strbuf sb = STRBUF_INIT; cache_tree_write(&sb, istate->cache_tree); - err = write_index_ext_header(&c, newfd, CACHE_EXT_TREE, sb.len) < 0 + err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_TREE, sb.len) < 0 || ce_write(&c, newfd, sb.buf, sb.len) < 0; strbuf_release(&sb); if (err) @@@ -2910,7 -2545,7 +2910,7 @@@ struct strbuf sb = STRBUF_INIT; resolve_undo_write(&sb, istate->resolve_undo); - err = write_index_ext_header(&c, newfd, CACHE_EXT_RESOLVE_UNDO, + err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_RESOLVE_UNDO, sb.len) < 0 || ce_write(&c, newfd, sb.buf, sb.len) < 0; strbuf_release(&sb); @@@ -2921,7 -2556,7 +2921,7 @@@ struct strbuf sb = STRBUF_INIT; write_untracked_extension(&sb, istate->untracked); - err = write_index_ext_header(&c, newfd, CACHE_EXT_UNTRACKED, + err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_UNTRACKED, sb.len) < 0 || ce_write(&c, newfd, sb.buf, sb.len) < 0; strbuf_release(&sb); @@@ -2932,24 -2567,7 +2932,24 @@@ struct strbuf sb = STRBUF_INIT; write_fsmonitor_extension(&sb, istate); - err = write_index_ext_header(&c, newfd, CACHE_EXT_FSMONITOR, sb.len) < 0 + err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_FSMONITOR, sb.len) < 0 + || ce_write(&c, newfd, sb.buf, sb.len) < 0; + strbuf_release(&sb); + if (err) + return -1; + } + + /* + * CACHE_EXT_ENDOFINDEXENTRIES must be written as the last entry before the SHA1 + * so that it can be found and processed before all the index entries are + * read. Write it out regardless of the strip_extensions parameter as we need it + * when loading the shared index. + */ + if (offset) { + struct strbuf sb = STRBUF_INIT; + + write_eoie_extension(&sb, &eoie_c, offset); + err = write_index_ext_header(&c, NULL, newfd, CACHE_EXT_ENDOFINDEXENTRIES, sb.len) < 0 || ce_write(&c, newfd, sb.buf, sb.len) < 0; strbuf_release(&sb); if (err) @@@ -3125,9 -2743,6 +3125,9 @@@ int write_locked_index(struct index_sta int new_shared_index, ret; struct split_index *si = istate->split_index; + if (git_env_bool("GIT_TEST_CHECK_CACHE_TREE", 0)) + cache_tree_verify(istate); + if ((flags & SKIP_IF_UNCHANGED) && !istate->cache_changed) { if (flags & COMMIT_LOCK) rollback_lock_file(lock); @@@ -3324,8 -2939,6 +3324,8 @@@ void move_index_extensions(struct index { dst->untracked = src->untracked; src->untracked = NULL; + dst->cache_tree = src->cache_tree; + src->cache_tree = NULL; } struct cache_entry *dup_cache_entry(const struct cache_entry *ce, @@@ -3365,181 -2978,3 +3365,181 @@@ int should_validate_cache_entries(void return validate_index_cache_entries; } + +#define EOIE_SIZE (4 + GIT_SHA1_RAWSZ) /* <4-byte offset> + <20-byte hash> */ +#define EOIE_SIZE_WITH_HEADER (4 + 4 + EOIE_SIZE) /* <4-byte signature> + <4-byte length> + EOIE_SIZE */ + +static size_t read_eoie_extension(const char *mmap, size_t mmap_size) +{ + /* + * The end of index entries (EOIE) extension is guaranteed to be last + * so that it can be found by scanning backwards from the EOF. + * + * "EOIE" + * <4-byte length> + * <4-byte offset> + * <20-byte hash> + */ + const char *index, *eoie; + uint32_t extsize; + size_t offset, src_offset; + unsigned char hash[GIT_MAX_RAWSZ]; + git_hash_ctx c; + + /* ensure we have an index big enough to contain an EOIE extension */ + if (mmap_size < sizeof(struct cache_header) + EOIE_SIZE_WITH_HEADER + the_hash_algo->rawsz) + return 0; + + /* validate the extension signature */ + index = eoie = mmap + mmap_size - EOIE_SIZE_WITH_HEADER - the_hash_algo->rawsz; + if (CACHE_EXT(index) != CACHE_EXT_ENDOFINDEXENTRIES) + return 0; + index += sizeof(uint32_t); + + /* validate the extension size */ + extsize = get_be32(index); + if (extsize != EOIE_SIZE) + return 0; + index += sizeof(uint32_t); + + /* + * Validate the offset we're going to look for the first extension + * signature is after the index header and before the eoie extension. + */ + offset = get_be32(index); + if (mmap + offset < mmap + sizeof(struct cache_header)) + return 0; + if (mmap + offset >= eoie) + return 0; + index += sizeof(uint32_t); + + /* + * The hash is computed over extension types and their sizes (but not + * their contents). E.g. if we have "TREE" extension that is N-bytes + * long, "REUC" extension that is M-bytes long, followed by "EOIE", + * then the hash would be: + * + * SHA-1("TREE" + + + * "REUC" + ) + */ + src_offset = offset; + the_hash_algo->init_fn(&c); + while (src_offset < mmap_size - the_hash_algo->rawsz - EOIE_SIZE_WITH_HEADER) { + /* After an array of active_nr index entries, + * there can be arbitrary number of extended + * sections, each of which is prefixed with + * extension name (4-byte) and section length + * in 4-byte network byte order. + */ + uint32_t extsize; + memcpy(&extsize, mmap + src_offset + 4, 4); + extsize = ntohl(extsize); + + /* verify the extension size isn't so large it will wrap around */ + if (src_offset + 8 + extsize < src_offset) + return 0; + + the_hash_algo->update_fn(&c, mmap + src_offset, 8); + + src_offset += 8; + src_offset += extsize; + } + the_hash_algo->final_fn(hash, &c); + if (!hasheq(hash, (const unsigned char *)index)) + return 0; + + /* Validate that the extension offsets returned us back to the eoie extension. */ + if (src_offset != mmap_size - the_hash_algo->rawsz - EOIE_SIZE_WITH_HEADER) + return 0; + + return offset; +} + +static void write_eoie_extension(struct strbuf *sb, git_hash_ctx *eoie_context, size_t offset) +{ + uint32_t buffer; + unsigned char hash[GIT_MAX_RAWSZ]; + + /* offset */ + put_be32(&buffer, offset); + strbuf_add(sb, &buffer, sizeof(uint32_t)); + + /* hash */ + the_hash_algo->final_fn(hash, eoie_context); + strbuf_add(sb, hash, the_hash_algo->rawsz); +} + +#ifndef NO_PTHREADS +#define IEOT_VERSION (1) + +static struct index_entry_offset_table *read_ieot_extension(const char *mmap, size_t mmap_size, size_t offset) +{ + const char *index = NULL; + uint32_t extsize, ext_version; + struct index_entry_offset_table *ieot; + int i, nr; + + /* find the IEOT extension */ + if (!offset) + return NULL; + while (offset <= mmap_size - the_hash_algo->rawsz - 8) { + extsize = get_be32(mmap + offset + 4); + if (CACHE_EXT((mmap + offset)) == CACHE_EXT_INDEXENTRYOFFSETTABLE) { + index = mmap + offset + 4 + 4; + break; + } + offset += 8; + offset += extsize; + } + if (!index) + return NULL; + + /* validate the version is IEOT_VERSION */ + ext_version = get_be32(index); + if (ext_version != IEOT_VERSION) { + error("invalid IEOT version %d", ext_version); + return NULL; + } + index += sizeof(uint32_t); + + /* extension size - version bytes / bytes per entry */ + nr = (extsize - sizeof(uint32_t)) / (sizeof(uint32_t) + sizeof(uint32_t)); + if (!nr) { + error("invalid number of IEOT entries %d", nr); + return NULL; + } + ieot = xmalloc(sizeof(struct index_entry_offset_table) + + (nr * sizeof(struct index_entry_offset))); + ieot->nr = nr; + for (i = 0; i < nr; i++) { + ieot->entries[i].offset = get_be32(index); + index += sizeof(uint32_t); + ieot->entries[i].nr = get_be32(index); + index += sizeof(uint32_t); + } + + return ieot; +} + +static void write_ieot_extension(struct strbuf *sb, struct index_entry_offset_table *ieot) +{ + uint32_t buffer; + int i; + + /* version */ + put_be32(&buffer, IEOT_VERSION); + strbuf_add(sb, &buffer, sizeof(uint32_t)); + + /* ieot */ + for (i = 0; i < ieot->nr; i++) { + + /* offset */ + put_be32(&buffer, ieot->entries[i].offset); + strbuf_add(sb, &buffer, sizeof(uint32_t)); + + /* count */ + put_be32(&buffer, ieot->entries[i].nr); + strbuf_add(sb, &buffer, sizeof(uint32_t)); + } +} +#endif diff --combined t/t1700-split-index.sh index 3e618cad12,ae74d2664a..2ac47aa0e4 --- a/t/t1700-split-index.sh +++ b/t/t1700-split-index.sh @@@ -6,13 -6,18 +6,20 @@@ test_description='split index mode test # We need total control of index splitting here sane_unset GIT_TEST_SPLIT_INDEX -# A couple of tests expect the index to have a specific checksum, -# but the presence of the optional FSMN extension would interfere -# with those checks, so disable it in this test script. -sane_unset GIT_FSMONITOR_TEST + +# Testing a hard coded SHA against an index with an extension +# that can vary from run to run is problematic so we disable +# those extensions. +sane_unset GIT_TEST_FSMONITOR +sane_unset GIT_TEST_INDEX_THREADS + # Create a file named as $1 with content read from stdin. + # Set the file's mtime to a few seconds in the past to avoid racy situations. + create_non_racy_file () { + cat >"$1" && + test-tool chmtime =-5 "$1" + } + test_expect_success 'enable split index' ' git config splitIndex.maxPercentChange 100 && git update-index --split-index && @@@ -20,11 -25,11 +27,11 @@@ indexversion=$(test-tool index-version <.git/index) && if test "$indexversion" = "4" then - own=432ef4b63f32193984f339431fd50ca796493569 - base=508851a7f0dfa8691e9f69c7f055865389012491 + own=3527df833c6c100d3d1d921a9a782d62a8be4b58 + base=746f7ab2ed44fb839efdfbffcf399d0b113fb4cb else - own=8299b0bcd1ac364e5f1d7768efb62fa2da79a339 - base=39d890139ee5356c7ef572216cebcd27aa41f9df + own=5e9b60117ece18da410ddecc8b8d43766a0e4204 + base=4370042739b31cd17a5c5cd6043a77c9a00df113 fi && cat >expect <<-EOF && own $own @@@ -36,7 -41,7 +43,7 @@@ ' test_expect_success 'add one file' ' - : >one && + create_non_racy_file one && git update-index --add one && git ls-files --stage >ls-files.actual && cat >ls-files.expect <<-EOF && @@@ -62,7 -67,7 +69,7 @@@ test_expect_success 'disable split inde EOF test_cmp ls-files.expect ls-files.actual && - BASE=$(test-tool dump-split-index .git/index | grep "^own" | sed "s/own/base/") && + BASE=$(test-tool dump-split-index .git/index | sed -n "s/^own/base/p") && test-tool dump-split-index .git/index | sed "/^own/d" >actual && cat >expect <<-EOF && not a split index @@@ -88,7 -93,7 +95,7 @@@ test_expect_success 'enable split inde ' test_expect_success 'modify original file, base index untouched' ' - echo modified >one && + echo modified | create_non_racy_file one && git update-index one && git ls-files --stage >ls-files.actual && cat >ls-files.expect <<-EOF && @@@ -107,7 -112,7 +114,7 @@@ ' test_expect_success 'add another file, which stays index' ' - : >two && + create_non_racy_file two && git update-index --add two && git ls-files --stage >ls-files.actual && cat >ls-files.expect <<-EOF && @@@ -160,7 -165,7 +167,7 @@@ test_expect_success 'remove file in bas ' test_expect_success 'add original file back' ' - : >one && + create_non_racy_file one && git update-index --add one && git ls-files --stage >ls-files.actual && cat >ls-files.expect <<-EOF && @@@ -179,7 -184,7 +186,7 @@@ ' test_expect_success 'add new file' ' - : >two && + create_non_racy_file two && git update-index --add two && git ls-files --stage >actual && cat >expect <<-EOF && @@@ -223,7 -228,7 +230,7 @@@ test_expect_success 'rev-parse --shared test_expect_success 'set core.splitIndex config variable to true' ' git config core.splitIndex true && - : >three && + create_non_racy_file three && git update-index --add three && git ls-files --stage >ls-files.actual && cat >ls-files.expect <<-EOF && @@@ -258,9 -263,9 +265,9 @@@ test_expect_success 'set core.splitInde test_cmp expect actual ' - test_expect_success 'set core.splitIndex config variable to true' ' + test_expect_success 'set core.splitIndex config variable back to true' ' git config core.splitIndex true && - : >three && + create_non_racy_file three && git update-index --add three && BASE=$(test-tool dump-split-index .git/index | grep "^base") && test-tool dump-split-index .git/index | sed "/^own/d" >actual && @@@ -270,7 -275,7 +277,7 @@@ deletions: EOF test_cmp expect actual && - : >four && + create_non_racy_file four && git update-index --add four && test-tool dump-split-index .git/index | sed "/^own/d" >actual && cat >expect <<-EOF && @@@ -284,7 -289,7 +291,7 @@@ test_expect_success 'check behavior with splitIndex.maxPercentChange unset' ' git config --unset splitIndex.maxPercentChange && - : >five && + create_non_racy_file five && git update-index --add five && BASE=$(test-tool dump-split-index .git/index | grep "^base") && test-tool dump-split-index .git/index | sed "/^own/d" >actual && @@@ -294,7 -299,7 +301,7 @@@ deletions: EOF test_cmp expect actual && - : >six && + create_non_racy_file six && git update-index --add six && test-tool dump-split-index .git/index | sed "/^own/d" >actual && cat >expect <<-EOF && @@@ -308,7 -313,7 +315,7 @@@ test_expect_success 'check splitIndex.maxPercentChange set to 0' ' git config splitIndex.maxPercentChange 0 && - : >seven && + create_non_racy_file seven && git update-index --add seven && BASE=$(test-tool dump-split-index .git/index | grep "^base") && test-tool dump-split-index .git/index | sed "/^own/d" >actual && @@@ -318,7 -323,7 +325,7 @@@ deletions: EOF test_cmp expect actual && - : >eight && + create_non_racy_file eight && git update-index --add eight && BASE=$(test-tool dump-split-index .git/index | grep "^base") && test-tool dump-split-index .git/index | sed "/^own/d" >actual && @@@ -331,17 -336,17 +338,17 @@@ ' test_expect_success 'shared index files expire after 2 weeks by default' ' - : >ten && + create_non_racy_file ten && git update-index --add ten && test $(ls .git/sharedindex.* | wc -l) -gt 2 && just_under_2_weeks_ago=$((5-14*86400)) && test-tool chmtime =$just_under_2_weeks_ago .git/sharedindex.* && - : >eleven && + create_non_racy_file eleven && git update-index --add eleven && test $(ls .git/sharedindex.* | wc -l) -gt 2 && just_over_2_weeks_ago=$((-1-14*86400)) && test-tool chmtime =$just_over_2_weeks_ago .git/sharedindex.* && - : >twelve && + create_non_racy_file twelve && git update-index --add twelve && test $(ls .git/sharedindex.* | wc -l) -le 2 ' @@@ -349,12 -354,12 +356,12 @@@ test_expect_success 'check splitIndex.sharedIndexExpire set to 16 days' ' git config splitIndex.sharedIndexExpire "16.days.ago" && test-tool chmtime =$just_over_2_weeks_ago .git/sharedindex.* && - : >thirteen && + create_non_racy_file thirteen && git update-index --add thirteen && test $(ls .git/sharedindex.* | wc -l) -gt 2 && just_over_16_days_ago=$((-1-16*86400)) && test-tool chmtime =$just_over_16_days_ago .git/sharedindex.* && - : >fourteen && + create_non_racy_file fourteen && git update-index --add fourteen && test $(ls .git/sharedindex.* | wc -l) -le 2 ' @@@ -363,13 -368,13 +370,13 @@@ test_expect_success 'check splitIndex.s git config splitIndex.sharedIndexExpire never && just_10_years_ago=$((-365*10*86400)) && test-tool chmtime =$just_10_years_ago .git/sharedindex.* && - : >fifteen && + create_non_racy_file fifteen && git update-index --add fifteen && test $(ls .git/sharedindex.* | wc -l) -gt 2 && git config splitIndex.sharedIndexExpire now && just_1_second_ago=-1 && test-tool chmtime =$just_1_second_ago .git/sharedindex.* && - : >sixteen && + create_non_racy_file sixteen && git update-index --add sixteen && test $(ls .git/sharedindex.* | wc -l) -le 2 ' @@@ -384,7 -389,7 +391,7 @@@ d # Create one new shared index file git config core.sharedrepository "$mode" && git config core.splitIndex true && - : >one && + create_non_racy_file one && git update-index --add one && echo "$modebits" >expect && test_modebits .git/index >actual &&