From: Junio C Hamano Date: Thu, 15 Feb 2018 22:55:44 +0000 (-0800) Subject: Merge branch 'nd/trace-index-ops' X-Git-Tag: v2.17.0-rc0~93 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/090dbea6843f860de8c981d97d031621176fc2be?ds=inline;hp=-c Merge branch 'nd/trace-index-ops' * nd/trace-index-ops: trace: measure where the time is spent in the index-heavy operations --- 090dbea6843f860de8c981d97d031621176fc2be diff --combined dir.c index ff2188890e,4479a02a49..536416ff2d --- a/dir.c +++ b/dir.c @@@ -231,10 -231,12 +231,10 @@@ int within_depth(const char *name, int * 1 along with { data, size } of the (possibly augmented) buffer * when successful. * - * Optionally updates the given sha1_stat with the given OID (when valid). + * Optionally updates the given oid_stat with the given OID (when valid). */ -static int do_read_blob(const struct object_id *oid, - struct sha1_stat *sha1_stat, - size_t *size_out, - char **data_out) +static int do_read_blob(const struct object_id *oid, struct oid_stat *oid_stat, + size_t *size_out, char **data_out) { enum object_type type; unsigned long sz; @@@ -249,9 -251,9 +249,9 @@@ return -1; } - if (sha1_stat) { - memset(&sha1_stat->stat, 0, sizeof(sha1_stat->stat)); - hashcpy(sha1_stat->sha1, oid->hash); + if (oid_stat) { + memset(&oid_stat->stat, 0, sizeof(oid_stat->stat)); + oidcpy(&oid_stat->oid, oid); } if (sz == 0) { @@@ -652,8 -654,9 +652,8 @@@ void add_exclude(const char *string, co static int read_skip_worktree_file_from_index(const struct index_state *istate, const char *path, - size_t *size_out, - char **data_out, - struct sha1_stat *sha1_stat) + size_t *size_out, char **data_out, + struct oid_stat *oid_stat) { int pos, len; @@@ -664,7 -667,7 +664,7 @@@ if (!ce_skip_worktree(istate->cache[pos])) return -1; - return do_read_blob(&istate->cache[pos]->oid, sha1_stat, size_out, data_out); + return do_read_blob(&istate->cache[pos]->oid, oid_stat, size_out, data_out); } /* @@@ -744,8 -747,8 +744,8 @@@ static struct untracked_cache_dir *look FLEX_ALLOC_MEM(d, name, name, len); ALLOC_GROW(dir->dirs, dir->dirs_nr + 1, dir->dirs_alloc); - memmove(dir->dirs + first + 1, dir->dirs + first, - (dir->dirs_nr - first) * sizeof(*dir->dirs)); + MOVE_ARRAY(dir->dirs + first + 1, dir->dirs + first, + dir->dirs_nr - first); dir->dirs_nr++; dir->dirs[first] = d; return d; @@@ -792,8 -795,9 +792,8 @@@ static int add_excludes_from_buffer(cha * ss_valid is non-zero, "ss" must contain good value as input. */ static int add_excludes(const char *fname, const char *base, int baselen, - struct exclude_list *el, - struct index_state *istate, - struct sha1_stat *sha1_stat) + struct exclude_list *el, struct index_state *istate, + struct oid_stat *oid_stat) { struct stat st; int r; @@@ -811,16 -815,16 +811,16 @@@ return -1; r = read_skip_worktree_file_from_index(istate, fname, &size, &buf, - sha1_stat); + oid_stat); if (r != 1) return r; } else { size = xsize_t(st.st_size); if (size == 0) { - if (sha1_stat) { - fill_stat_data(&sha1_stat->stat, &st); - hashcpy(sha1_stat->sha1, EMPTY_BLOB_SHA1_BIN); - sha1_stat->valid = 1; + if (oid_stat) { + fill_stat_data(&oid_stat->stat, &st); + oidcpy(&oid_stat->oid, &empty_blob_oid); + oid_stat->valid = 1; } close(fd); return 0; @@@ -833,23 -837,22 +833,23 @@@ } buf[size++] = '\n'; close(fd); - if (sha1_stat) { + if (oid_stat) { int pos; - if (sha1_stat->valid && - !match_stat_data_racy(istate, &sha1_stat->stat, &st)) + if (oid_stat->valid && + !match_stat_data_racy(istate, &oid_stat->stat, &st)) ; /* no content change, ss->sha1 still good */ else if (istate && (pos = index_name_pos(istate, fname, strlen(fname))) >= 0 && !ce_stage(istate->cache[pos]) && ce_uptodate(istate->cache[pos]) && !would_convert_to_git(istate, fname)) - hashcpy(sha1_stat->sha1, - istate->cache[pos]->oid.hash); + oidcpy(&oid_stat->oid, + &istate->cache[pos]->oid); else - hash_sha1_file(buf, size, "blob", sha1_stat->sha1); - fill_stat_data(&sha1_stat->stat, &st); - sha1_stat->valid = 1; + hash_object_file(buf, size, "blob", + &oid_stat->oid); + fill_stat_data(&oid_stat->stat, &st); + oid_stat->valid = 1; } } @@@ -927,7 -930,7 +927,7 @@@ struct exclude_list *add_exclude_list(s * Used to set up core.excludesfile and .git/info/exclude lists. */ static void add_excludes_from_file_1(struct dir_struct *dir, const char *fname, - struct sha1_stat *sha1_stat) + struct oid_stat *oid_stat) { struct exclude_list *el; /* @@@ -938,7 -941,7 +938,7 @@@ if (!dir->untracked) dir->unmanaged_exclude_files++; el = add_exclude_list(dir, EXC_FILE, fname); - if (add_excludes(fname, "", 0, el, NULL, sha1_stat) < 0) + if (add_excludes(fname, "", 0, el, NULL, oid_stat) < 0) die("cannot use %s as an exclude file", fname); } @@@ -1177,7 -1180,7 +1177,7 @@@ static void prep_exclude(struct dir_str while (current < baselen) { const char *cp; - struct sha1_stat sha1_stat; + struct oid_stat oid_stat; stk = xcalloc(1, sizeof(*stk)); if (current < 0) { @@@ -1220,8 -1223,8 +1220,8 @@@ } /* Try to read per-directory file */ - hashclr(sha1_stat.sha1); - sha1_stat.valid = 0; + oidclr(&oid_stat.oid); + oid_stat.valid = 0; if (dir->exclude_per_dir && /* * If we know that no files have been added in @@@ -1249,7 -1252,7 +1249,7 @@@ strbuf_addstr(&sb, dir->exclude_per_dir); el->src = strbuf_detach(&sb, NULL); add_excludes(el->src, el->src, stk->baselen, el, istate, - untracked ? &sha1_stat : NULL); + untracked ? &oid_stat : NULL); } /* * NEEDSWORK: when untracked cache is enabled, prep_exclude() @@@ -1266,9 -1269,9 +1266,9 @@@ * order, though, if you do that. */ if (untracked && - hashcmp(sha1_stat.sha1, untracked->exclude_sha1)) { + hashcmp(oid_stat.oid.hash, untracked->exclude_sha1)) { invalidate_gitignore(dir->untracked, untracked); - hashcpy(untracked->exclude_sha1, sha1_stat.sha1); + hashcpy(untracked->exclude_sha1, oid_stat.oid.hash); } dir->exclude_stack = stk; current = stk->baselen; @@@ -2225,13 -2228,13 +2225,13 @@@ static struct untracked_cache_dir *vali /* Validate $GIT_DIR/info/exclude and core.excludesfile */ root = dir->untracked->root; - if (hashcmp(dir->ss_info_exclude.sha1, - dir->untracked->ss_info_exclude.sha1)) { + if (oidcmp(&dir->ss_info_exclude.oid, + &dir->untracked->ss_info_exclude.oid)) { invalidate_gitignore(dir->untracked, root); dir->untracked->ss_info_exclude = dir->ss_info_exclude; } - if (hashcmp(dir->ss_excludes_file.sha1, - dir->untracked->ss_excludes_file.sha1)) { + if (oidcmp(&dir->ss_excludes_file.oid, + &dir->untracked->ss_excludes_file.oid)) { invalidate_gitignore(dir->untracked, root); dir->untracked->ss_excludes_file = dir->ss_excludes_file; } @@@ -2245,6 -2248,7 +2245,7 @@@ int read_directory(struct dir_struct *d const char *path, int len, const struct pathspec *pathspec) { struct untracked_cache_dir *untracked; + uint64_t start = getnanotime(); if (has_symlink_leading_path(path, len)) return dir->nr; @@@ -2283,6 -2287,7 +2284,7 @@@ dir->nr = i; } + trace_performance_since(start, "read directory %.*s", len, path); if (dir->untracked) { static struct trace_key trace_untracked_stats = TRACE_KEY_INIT(UNTRACKED_STATS); trace_printf_key(&trace_untracked_stats, @@@ -2635,8 -2640,8 +2637,8 @@@ void write_untracked_extension(struct s FLEX_ALLOC_MEM(ouc, exclude_per_dir, untracked->exclude_per_dir, len); stat_data_to_disk(&ouc->info_exclude_stat, &untracked->ss_info_exclude.stat); stat_data_to_disk(&ouc->excludes_file_stat, &untracked->ss_excludes_file.stat); - hashcpy(ouc->info_exclude_sha1, untracked->ss_info_exclude.sha1); - hashcpy(ouc->excludes_file_sha1, untracked->ss_excludes_file.sha1); + hashcpy(ouc->info_exclude_sha1, untracked->ss_info_exclude.oid.hash); + hashcpy(ouc->excludes_file_sha1, untracked->ss_excludes_file.oid.hash); ouc->dir_flags = htonl(untracked->dir_flags); varint_len = encode_varint(untracked->ident.len, varbuf); @@@ -2813,12 -2818,13 +2815,12 @@@ static void read_sha1(size_t pos, void rd->data += 20; } -static void load_sha1_stat(struct sha1_stat *sha1_stat, - const unsigned char *data, - const unsigned char *sha1) +static void load_oid_stat(struct oid_stat *oid_stat, const unsigned char *data, + const unsigned char *sha1) { - stat_data_from_disk(&sha1_stat->stat, data); - hashcpy(sha1_stat->sha1, sha1); - sha1_stat->valid = 1; + stat_data_from_disk(&oid_stat->stat, data); + hashcpy(oid_stat->oid.hash, sha1); + oid_stat->valid = 1; } struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz) @@@ -2846,12 -2852,12 +2848,12 @@@ uc = xcalloc(1, sizeof(*uc)); strbuf_init(&uc->ident, ident_len); strbuf_add(&uc->ident, ident, ident_len); - load_sha1_stat(&uc->ss_info_exclude, - next + ouc_offset(info_exclude_stat), - next + ouc_offset(info_exclude_sha1)); - load_sha1_stat(&uc->ss_excludes_file, - next + ouc_offset(excludes_file_stat), - next + ouc_offset(excludes_file_sha1)); + load_oid_stat(&uc->ss_info_exclude, + next + ouc_offset(info_exclude_stat), + next + ouc_offset(info_exclude_sha1)); + load_oid_stat(&uc->ss_excludes_file, + next + ouc_offset(excludes_file_stat), + next + ouc_offset(excludes_file_sha1)); uc->dir_flags = get_be32(next + ouc_offset(dir_flags)); exclude_per_dir = (const char *)next + ouc_offset(exclude_per_dir); uc->exclude_per_dir = xstrdup(exclude_per_dir); diff --combined read-cache.c index f9871cde33,eac74bc9f1..82ba2287e1 --- a/read-cache.c +++ b/read-cache.c @@@ -631,10 -631,10 +631,10 @@@ static struct cache_entry *create_alias void set_object_name_for_intent_to_add_entry(struct cache_entry *ce) { - unsigned char sha1[20]; - if (write_sha1_file("", 0, blob_type, sha1)) + struct object_id oid; + if (write_object_file("", 0, blob_type, &oid)) die("cannot create an empty blob in the object database"); - hashcpy(ce->oid.hash, sha1); + oidcpy(&ce->oid, &oid); } int add_to_index(struct index_state *istate, const char *path, struct stat *st, int flags) @@@ -1217,8 -1217,9 +1217,8 @@@ int add_index_entry(struct index_state /* Add it in.. */ istate->cache_nr++; if (istate->cache_nr > pos + 1) - memmove(istate->cache + pos + 1, - istate->cache + pos, - (istate->cache_nr - pos - 1) * sizeof(ce)); + MOVE_ARRAY(istate->cache + pos + 1, istate->cache + pos, + istate->cache_nr - pos - 1); set_index_entry(istate, pos, ce); istate->cache_changed |= CE_ENTRY_ADDED; return 0; @@@ -1371,6 -1372,7 +1371,7 @@@ int refresh_index(struct index_state *i const char *typechange_fmt; const char *added_fmt; const char *unmerged_fmt; + uint64_t start = getnanotime(); modified_fmt = (in_porcelain ? "M\t%s\n" : "%s: needs update\n"); deleted_fmt = (in_porcelain ? "D\t%s\n" : "%s: needs update\n"); @@@ -1441,6 -1443,7 +1442,7 @@@ replace_index_entry(istate, i, new); } + trace_performance_since(start, "refresh index"); return has_errors; } @@@ -1602,7 -1605,7 +1604,7 @@@ int hold_locked_index(struct lock_file int read_index(struct index_state *istate) { - return read_index_from(istate, get_index_file()); + return read_index_from(istate, get_index_file(), get_git_dir()); } static struct cache_entry *cache_entry_from_ondisk(struct ondisk_cache_entry *ondisk, @@@ -1862,25 -1865,28 +1864,27 @@@ unmap * This way, shared index can be removed if they have not been used * for some time. */ -static void freshen_shared_index(char *base_sha1_hex, int warn) +static void freshen_shared_index(const char *shared_index, int warn) { - char *shared_index = git_pathdup("sharedindex.%s", base_sha1_hex); if (!check_and_freshen_file(shared_index, 1) && warn) warning("could not freshen shared index '%s'", shared_index); - free(shared_index); } -int read_index_from(struct index_state *istate, const char *path) +int read_index_from(struct index_state *istate, const char *path, + const char *gitdir) { + uint64_t start = getnanotime(); struct split_index *split_index; int ret; char *base_sha1_hex; - const char *base_path; + char *base_path; /* istate->initialized covers both .git/index and .git/sharedindex.xxx */ if (istate->initialized) return istate->cache_nr; ret = do_read_index(istate, path, 0); + trace_performance_since(start, "read cache %s", path); split_index = istate->split_index; if (!split_index || is_null_sha1(split_index->base_sha1)) { @@@ -1894,17 -1900,17 +1898,18 @@@ split_index->base = xcalloc(1, sizeof(*split_index->base)); base_sha1_hex = sha1_to_hex(split_index->base_sha1); - base_path = git_path("sharedindex.%s", base_sha1_hex); + base_path = xstrfmt("%s/sharedindex.%s", gitdir, base_sha1_hex); ret = do_read_index(split_index->base, base_path, 1); if (hashcmp(split_index->base_sha1, split_index->base->sha1)) die("broken index, expect %s in %s, got %s", base_sha1_hex, base_path, sha1_to_hex(split_index->base->sha1)); - freshen_shared_index(base_sha1_hex, 0); + freshen_shared_index(base_path, 0); merge_base_index(istate); post_read_index_from(istate); + trace_performance_since(start, "read cache %s", base_path); + free(base_path); return ret; } @@@ -2233,6 -2239,7 +2238,7 @@@ void update_index_if_able(struct index_ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, int strip_extensions) { + uint64_t start = getnanotime(); int newfd = tempfile->fd; git_SHA_CTX c; struct cache_header hdr; @@@ -2242,7 -2249,7 +2248,7 @@@ struct stat st; struct ondisk_cache_entry_extended ondisk; struct strbuf previous_name_buf = STRBUF_INIT, *previous_name; - int drop_cache_tree = 0; + int drop_cache_tree = istate->drop_cache_tree; for (i = removed = extended = 0; i < entries; i++) { if (cache[i]->ce_flags & CE_REMOVE) @@@ -2373,6 -2380,7 +2379,7 @@@ return -1; istate->timestamp.sec = (unsigned int)st.st_mtime; istate->timestamp.nsec = ST_MTIME_NSEC(st); + trace_performance_since(start, "write index, changed mask = %x", istate->cache_changed); return 0; } @@@ -2471,21 -2479,32 +2478,21 @@@ static int clean_shared_index_files(con } static int write_shared_index(struct index_state *istate, - struct lock_file *lock, unsigned flags) + struct tempfile **temp) { - struct tempfile *temp; struct split_index *si = istate->split_index; int ret; - temp = mks_tempfile(git_path("sharedindex_XXXXXX")); - if (!temp) { - hashclr(si->base_sha1); - return do_write_locked_index(istate, lock, flags); - } move_cache_to_base_index(istate); - ret = do_write_index(si->base, temp, 1); - if (ret) { - delete_tempfile(&temp); + ret = do_write_index(si->base, *temp, 1); + if (ret) return ret; - } - ret = adjust_shared_perm(get_tempfile_path(temp)); + ret = adjust_shared_perm(get_tempfile_path(*temp)); if (ret) { - int save_errno = errno; - error("cannot fix permission bits on %s", get_tempfile_path(temp)); - delete_tempfile(&temp); - errno = save_errno; + error("cannot fix permission bits on %s", get_tempfile_path(*temp)); return ret; } - ret = rename_tempfile(&temp, + ret = rename_tempfile(temp, git_path("sharedindex.%s", sha1_to_hex(si->base->sha1))); if (!ret) { hashcpy(si->base_sha1, si->base->sha1); @@@ -2553,22 -2572,7 +2560,22 @@@ int write_locked_index(struct index_sta new_shared_index = istate->cache_changed & SPLIT_INDEX_ORDERED; if (new_shared_index) { - ret = write_shared_index(istate, lock, flags); + struct tempfile *temp; + int saved_errno; + + temp = mks_tempfile(git_path("sharedindex_XXXXXX")); + if (!temp) { + hashclr(si->base_sha1); + ret = do_write_locked_index(istate, lock, flags); + goto out; + } + ret = write_shared_index(istate, &temp); + + saved_errno = errno; + if (is_tempfile_active(temp)) + delete_tempfile(&temp); + errno = saved_errno; + if (ret) goto out; } @@@ -2576,11 -2580,8 +2583,11 @@@ ret = write_split_index(istate, lock, flags); /* Freshen the shared index only if the split-index was written */ - if (!ret && !new_shared_index) - freshen_shared_index(sha1_to_hex(si->base_sha1), 1); + if (!ret && !new_shared_index) { + const char *shared_index = git_path("sharedindex.%s", + sha1_to_hex(si->base_sha1)); + freshen_shared_index(shared_index, 1); + } out: if (flags & COMMIT_LOCK)