Merge branch 'nd/trace-index-ops'
authorJunio C Hamano <gitster@pobox.com>
Thu, 15 Feb 2018 22:55:44 +0000 (14:55 -0800)
committerJunio C Hamano <gitster@pobox.com>
Thu, 15 Feb 2018 22:55:44 +0000 (14:55 -0800)
* nd/trace-index-ops:
trace: measure where the time is spent in the index-heavy operations

1  2 
dir.c
read-cache.c
diff --combined dir.c
index ff2188890e223ec8821ef43967425b3435c88c4c,4479a02a4935a57796600a76ba47392a803e50b1..536416ff2df5f7c475deba4739d4edaf340ae898
--- 1/dir.c
--- 2/dir.c
+++ b/dir.c
@@@ -231,10 -231,12 +231,10 @@@ int within_depth(const char *name, int 
   *     1 along with { data, size } of the (possibly augmented) buffer
   *       when successful.
   *
 - * Optionally updates the given sha1_stat with the given OID (when valid).
 + * Optionally updates the given oid_stat with the given OID (when valid).
   */
 -static int do_read_blob(const struct object_id *oid,
 -                      struct sha1_stat *sha1_stat,
 -                      size_t *size_out,
 -                      char **data_out)
 +static int do_read_blob(const struct object_id *oid, struct oid_stat *oid_stat,
 +                      size_t *size_out, char **data_out)
  {
        enum object_type type;
        unsigned long sz;
                return -1;
        }
  
 -      if (sha1_stat) {
 -              memset(&sha1_stat->stat, 0, sizeof(sha1_stat->stat));
 -              hashcpy(sha1_stat->sha1, oid->hash);
 +      if (oid_stat) {
 +              memset(&oid_stat->stat, 0, sizeof(oid_stat->stat));
 +              oidcpy(&oid_stat->oid, oid);
        }
  
        if (sz == 0) {
@@@ -652,8 -654,9 +652,8 @@@ void add_exclude(const char *string, co
  
  static int read_skip_worktree_file_from_index(const struct index_state *istate,
                                              const char *path,
 -                                            size_t *size_out,
 -                                            char **data_out,
 -                                            struct sha1_stat *sha1_stat)
 +                                            size_t *size_out, char **data_out,
 +                                            struct oid_stat *oid_stat)
  {
        int pos, len;
  
        if (!ce_skip_worktree(istate->cache[pos]))
                return -1;
  
 -      return do_read_blob(&istate->cache[pos]->oid, sha1_stat, size_out, data_out);
 +      return do_read_blob(&istate->cache[pos]->oid, oid_stat, size_out, data_out);
  }
  
  /*
@@@ -744,8 -747,8 +744,8 @@@ static struct untracked_cache_dir *look
        FLEX_ALLOC_MEM(d, name, name, len);
  
        ALLOC_GROW(dir->dirs, dir->dirs_nr + 1, dir->dirs_alloc);
 -      memmove(dir->dirs + first + 1, dir->dirs + first,
 -              (dir->dirs_nr - first) * sizeof(*dir->dirs));
 +      MOVE_ARRAY(dir->dirs + first + 1, dir->dirs + first,
 +                 dir->dirs_nr - first);
        dir->dirs_nr++;
        dir->dirs[first] = d;
        return d;
@@@ -792,8 -795,9 +792,8 @@@ static int add_excludes_from_buffer(cha
   * ss_valid is non-zero, "ss" must contain good value as input.
   */
  static int add_excludes(const char *fname, const char *base, int baselen,
 -                      struct exclude_list *el,
 -                      struct index_state *istate,
 -                      struct sha1_stat *sha1_stat)
 +                      struct exclude_list *el, struct index_state *istate,
 +                      struct oid_stat *oid_stat)
  {
        struct stat st;
        int r;
                        return -1;
                r = read_skip_worktree_file_from_index(istate, fname,
                                                       &size, &buf,
 -                                                     sha1_stat);
 +                                                     oid_stat);
                if (r != 1)
                        return r;
        } else {
                size = xsize_t(st.st_size);
                if (size == 0) {
 -                      if (sha1_stat) {
 -                              fill_stat_data(&sha1_stat->stat, &st);
 -                              hashcpy(sha1_stat->sha1, EMPTY_BLOB_SHA1_BIN);
 -                              sha1_stat->valid = 1;
 +                      if (oid_stat) {
 +                              fill_stat_data(&oid_stat->stat, &st);
 +                              oidcpy(&oid_stat->oid, &empty_blob_oid);
 +                              oid_stat->valid = 1;
                        }
                        close(fd);
                        return 0;
                }
                buf[size++] = '\n';
                close(fd);
 -              if (sha1_stat) {
 +              if (oid_stat) {
                        int pos;
 -                      if (sha1_stat->valid &&
 -                          !match_stat_data_racy(istate, &sha1_stat->stat, &st))
 +                      if (oid_stat->valid &&
 +                          !match_stat_data_racy(istate, &oid_stat->stat, &st))
                                ; /* no content change, ss->sha1 still good */
                        else if (istate &&
                                 (pos = index_name_pos(istate, fname, strlen(fname))) >= 0 &&
                                 !ce_stage(istate->cache[pos]) &&
                                 ce_uptodate(istate->cache[pos]) &&
                                 !would_convert_to_git(istate, fname))
 -                              hashcpy(sha1_stat->sha1,
 -                                      istate->cache[pos]->oid.hash);
 +                              oidcpy(&oid_stat->oid,
 +                                     &istate->cache[pos]->oid);
                        else
 -                              hash_sha1_file(buf, size, "blob", sha1_stat->sha1);
 -                      fill_stat_data(&sha1_stat->stat, &st);
 -                      sha1_stat->valid = 1;
 +                              hash_object_file(buf, size, "blob",
 +                                               &oid_stat->oid);
 +                      fill_stat_data(&oid_stat->stat, &st);
 +                      oid_stat->valid = 1;
                }
        }
  
@@@ -927,7 -930,7 +927,7 @@@ struct exclude_list *add_exclude_list(s
   * Used to set up core.excludesfile and .git/info/exclude lists.
   */
  static void add_excludes_from_file_1(struct dir_struct *dir, const char *fname,
 -                                   struct sha1_stat *sha1_stat)
 +                                   struct oid_stat *oid_stat)
  {
        struct exclude_list *el;
        /*
        if (!dir->untracked)
                dir->unmanaged_exclude_files++;
        el = add_exclude_list(dir, EXC_FILE, fname);
 -      if (add_excludes(fname, "", 0, el, NULL, sha1_stat) < 0)
 +      if (add_excludes(fname, "", 0, el, NULL, oid_stat) < 0)
                die("cannot use %s as an exclude file", fname);
  }
  
@@@ -1177,7 -1180,7 +1177,7 @@@ static void prep_exclude(struct dir_str
  
        while (current < baselen) {
                const char *cp;
 -              struct sha1_stat sha1_stat;
 +              struct oid_stat oid_stat;
  
                stk = xcalloc(1, sizeof(*stk));
                if (current < 0) {
                }
  
                /* Try to read per-directory file */
 -              hashclr(sha1_stat.sha1);
 -              sha1_stat.valid = 0;
 +              oidclr(&oid_stat.oid);
 +              oid_stat.valid = 0;
                if (dir->exclude_per_dir &&
                    /*
                     * If we know that no files have been added in
                        strbuf_addstr(&sb, dir->exclude_per_dir);
                        el->src = strbuf_detach(&sb, NULL);
                        add_excludes(el->src, el->src, stk->baselen, el, istate,
 -                                   untracked ? &sha1_stat : NULL);
 +                                   untracked ? &oid_stat : NULL);
                }
                /*
                 * NEEDSWORK: when untracked cache is enabled, prep_exclude()
                 * order, though, if you do that.
                 */
                if (untracked &&
 -                  hashcmp(sha1_stat.sha1, untracked->exclude_sha1)) {
 +                  hashcmp(oid_stat.oid.hash, untracked->exclude_sha1)) {
                        invalidate_gitignore(dir->untracked, untracked);
 -                      hashcpy(untracked->exclude_sha1, sha1_stat.sha1);
 +                      hashcpy(untracked->exclude_sha1, oid_stat.oid.hash);
                }
                dir->exclude_stack = stk;
                current = stk->baselen;
@@@ -2225,13 -2228,13 +2225,13 @@@ static struct untracked_cache_dir *vali
  
        /* Validate $GIT_DIR/info/exclude and core.excludesfile */
        root = dir->untracked->root;
 -      if (hashcmp(dir->ss_info_exclude.sha1,
 -                  dir->untracked->ss_info_exclude.sha1)) {
 +      if (oidcmp(&dir->ss_info_exclude.oid,
 +                 &dir->untracked->ss_info_exclude.oid)) {
                invalidate_gitignore(dir->untracked, root);
                dir->untracked->ss_info_exclude = dir->ss_info_exclude;
        }
 -      if (hashcmp(dir->ss_excludes_file.sha1,
 -                  dir->untracked->ss_excludes_file.sha1)) {
 +      if (oidcmp(&dir->ss_excludes_file.oid,
 +                 &dir->untracked->ss_excludes_file.oid)) {
                invalidate_gitignore(dir->untracked, root);
                dir->untracked->ss_excludes_file = dir->ss_excludes_file;
        }
@@@ -2245,6 -2248,7 +2245,7 @@@ int read_directory(struct dir_struct *d
                   const char *path, int len, const struct pathspec *pathspec)
  {
        struct untracked_cache_dir *untracked;
+       uint64_t start = getnanotime();
  
        if (has_symlink_leading_path(path, len))
                return dir->nr;
                dir->nr = i;
        }
  
+       trace_performance_since(start, "read directory %.*s", len, path);
        if (dir->untracked) {
                static struct trace_key trace_untracked_stats = TRACE_KEY_INIT(UNTRACKED_STATS);
                trace_printf_key(&trace_untracked_stats,
@@@ -2635,8 -2640,8 +2637,8 @@@ void write_untracked_extension(struct s
        FLEX_ALLOC_MEM(ouc, exclude_per_dir, untracked->exclude_per_dir, len);
        stat_data_to_disk(&ouc->info_exclude_stat, &untracked->ss_info_exclude.stat);
        stat_data_to_disk(&ouc->excludes_file_stat, &untracked->ss_excludes_file.stat);
 -      hashcpy(ouc->info_exclude_sha1, untracked->ss_info_exclude.sha1);
 -      hashcpy(ouc->excludes_file_sha1, untracked->ss_excludes_file.sha1);
 +      hashcpy(ouc->info_exclude_sha1, untracked->ss_info_exclude.oid.hash);
 +      hashcpy(ouc->excludes_file_sha1, untracked->ss_excludes_file.oid.hash);
        ouc->dir_flags = htonl(untracked->dir_flags);
  
        varint_len = encode_varint(untracked->ident.len, varbuf);
@@@ -2813,12 -2818,13 +2815,12 @@@ static void read_sha1(size_t pos, void 
        rd->data += 20;
  }
  
 -static void load_sha1_stat(struct sha1_stat *sha1_stat,
 -                         const unsigned char *data,
 -                         const unsigned char *sha1)
 +static void load_oid_stat(struct oid_stat *oid_stat, const unsigned char *data,
 +                        const unsigned char *sha1)
  {
 -      stat_data_from_disk(&sha1_stat->stat, data);
 -      hashcpy(sha1_stat->sha1, sha1);
 -      sha1_stat->valid = 1;
 +      stat_data_from_disk(&oid_stat->stat, data);
 +      hashcpy(oid_stat->oid.hash, sha1);
 +      oid_stat->valid = 1;
  }
  
  struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz)
        uc = xcalloc(1, sizeof(*uc));
        strbuf_init(&uc->ident, ident_len);
        strbuf_add(&uc->ident, ident, ident_len);
 -      load_sha1_stat(&uc->ss_info_exclude,
 -                     next + ouc_offset(info_exclude_stat),
 -                     next + ouc_offset(info_exclude_sha1));
 -      load_sha1_stat(&uc->ss_excludes_file,
 -                     next + ouc_offset(excludes_file_stat),
 -                     next + ouc_offset(excludes_file_sha1));
 +      load_oid_stat(&uc->ss_info_exclude,
 +                    next + ouc_offset(info_exclude_stat),
 +                    next + ouc_offset(info_exclude_sha1));
 +      load_oid_stat(&uc->ss_excludes_file,
 +                    next + ouc_offset(excludes_file_stat),
 +                    next + ouc_offset(excludes_file_sha1));
        uc->dir_flags = get_be32(next + ouc_offset(dir_flags));
        exclude_per_dir = (const char *)next + ouc_offset(exclude_per_dir);
        uc->exclude_per_dir = xstrdup(exclude_per_dir);
diff --combined read-cache.c
index f9871cde33ef5423f6e7e4e9d41ea7775bdb10f2,eac74bc9f17d69c304461a3174e3ee0888ea1be9..82ba2287e1bbe6121a9c1c5e9b20a9b352886f95
@@@ -631,10 -631,10 +631,10 @@@ static struct cache_entry *create_alias
  
  void set_object_name_for_intent_to_add_entry(struct cache_entry *ce)
  {
 -      unsigned char sha1[20];
 -      if (write_sha1_file("", 0, blob_type, sha1))
 +      struct object_id oid;
 +      if (write_object_file("", 0, blob_type, &oid))
                die("cannot create an empty blob in the object database");
 -      hashcpy(ce->oid.hash, sha1);
 +      oidcpy(&ce->oid, &oid);
  }
  
  int add_to_index(struct index_state *istate, const char *path, struct stat *st, int flags)
@@@ -1217,8 -1217,9 +1217,8 @@@ int add_index_entry(struct index_state 
        /* Add it in.. */
        istate->cache_nr++;
        if (istate->cache_nr > pos + 1)
 -              memmove(istate->cache + pos + 1,
 -                      istate->cache + pos,
 -                      (istate->cache_nr - pos - 1) * sizeof(ce));
 +              MOVE_ARRAY(istate->cache + pos + 1, istate->cache + pos,
 +                         istate->cache_nr - pos - 1);
        set_index_entry(istate, pos, ce);
        istate->cache_changed |= CE_ENTRY_ADDED;
        return 0;
@@@ -1371,6 -1372,7 +1371,7 @@@ int refresh_index(struct index_state *i
        const char *typechange_fmt;
        const char *added_fmt;
        const char *unmerged_fmt;
+       uint64_t start = getnanotime();
  
        modified_fmt = (in_porcelain ? "M\t%s\n" : "%s: needs update\n");
        deleted_fmt = (in_porcelain ? "D\t%s\n" : "%s: needs update\n");
  
                replace_index_entry(istate, i, new);
        }
+       trace_performance_since(start, "refresh index");
        return has_errors;
  }
  
@@@ -1602,7 -1605,7 +1604,7 @@@ int hold_locked_index(struct lock_file 
  
  int read_index(struct index_state *istate)
  {
 -      return read_index_from(istate, get_index_file());
 +      return read_index_from(istate, get_index_file(), get_git_dir());
  }
  
  static struct cache_entry *cache_entry_from_ondisk(struct ondisk_cache_entry *ondisk,
@@@ -1862,25 -1865,28 +1864,27 @@@ unmap
   * This way, shared index can be removed if they have not been used
   * for some time.
   */
 -static void freshen_shared_index(char *base_sha1_hex, int warn)
 +static void freshen_shared_index(const char *shared_index, int warn)
  {
 -      char *shared_index = git_pathdup("sharedindex.%s", base_sha1_hex);
        if (!check_and_freshen_file(shared_index, 1) && warn)
                warning("could not freshen shared index '%s'", shared_index);
 -      free(shared_index);
  }
  
 -int read_index_from(struct index_state *istate, const char *path)
 +int read_index_from(struct index_state *istate, const char *path,
 +                  const char *gitdir)
  {
+       uint64_t start = getnanotime();
        struct split_index *split_index;
        int ret;
        char *base_sha1_hex;
 -      const char *base_path;
 +      char *base_path;
  
        /* istate->initialized covers both .git/index and .git/sharedindex.xxx */
        if (istate->initialized)
                return istate->cache_nr;
  
        ret = do_read_index(istate, path, 0);
+       trace_performance_since(start, "read cache %s", path);
  
        split_index = istate->split_index;
        if (!split_index || is_null_sha1(split_index->base_sha1)) {
                split_index->base = xcalloc(1, sizeof(*split_index->base));
  
        base_sha1_hex = sha1_to_hex(split_index->base_sha1);
 -      base_path = git_path("sharedindex.%s", base_sha1_hex);
 +      base_path = xstrfmt("%s/sharedindex.%s", gitdir, base_sha1_hex);
        ret = do_read_index(split_index->base, base_path, 1);
        if (hashcmp(split_index->base_sha1, split_index->base->sha1))
                die("broken index, expect %s in %s, got %s",
                    base_sha1_hex, base_path,
                    sha1_to_hex(split_index->base->sha1));
  
 -      freshen_shared_index(base_sha1_hex, 0);
 +      freshen_shared_index(base_path, 0);
        merge_base_index(istate);
        post_read_index_from(istate);
+       trace_performance_since(start, "read cache %s", base_path);
 +      free(base_path);
        return ret;
  }
  
@@@ -2233,6 -2239,7 +2238,7 @@@ void update_index_if_able(struct index_
  static int do_write_index(struct index_state *istate, struct tempfile *tempfile,
                          int strip_extensions)
  {
+       uint64_t start = getnanotime();
        int newfd = tempfile->fd;
        git_SHA_CTX c;
        struct cache_header hdr;
        struct stat st;
        struct ondisk_cache_entry_extended ondisk;
        struct strbuf previous_name_buf = STRBUF_INIT, *previous_name;
 -      int drop_cache_tree = 0;
 +      int drop_cache_tree = istate->drop_cache_tree;
  
        for (i = removed = extended = 0; i < entries; i++) {
                if (cache[i]->ce_flags & CE_REMOVE)
                return -1;
        istate->timestamp.sec = (unsigned int)st.st_mtime;
        istate->timestamp.nsec = ST_MTIME_NSEC(st);
+       trace_performance_since(start, "write index, changed mask = %x", istate->cache_changed);
        return 0;
  }
  
@@@ -2471,21 -2479,32 +2478,21 @@@ static int clean_shared_index_files(con
  }
  
  static int write_shared_index(struct index_state *istate,
 -                            struct lock_file *lock, unsigned flags)
 +                            struct tempfile **temp)
  {
 -      struct tempfile *temp;
        struct split_index *si = istate->split_index;
        int ret;
  
 -      temp = mks_tempfile(git_path("sharedindex_XXXXXX"));
 -      if (!temp) {
 -              hashclr(si->base_sha1);
 -              return do_write_locked_index(istate, lock, flags);
 -      }
        move_cache_to_base_index(istate);
 -      ret = do_write_index(si->base, temp, 1);
 -      if (ret) {
 -              delete_tempfile(&temp);
 +      ret = do_write_index(si->base, *temp, 1);
 +      if (ret)
                return ret;
 -      }
 -      ret = adjust_shared_perm(get_tempfile_path(temp));
 +      ret = adjust_shared_perm(get_tempfile_path(*temp));
        if (ret) {
 -              int save_errno = errno;
 -              error("cannot fix permission bits on %s", get_tempfile_path(temp));
 -              delete_tempfile(&temp);
 -              errno = save_errno;
 +              error("cannot fix permission bits on %s", get_tempfile_path(*temp));
                return ret;
        }
 -      ret = rename_tempfile(&temp,
 +      ret = rename_tempfile(temp,
                              git_path("sharedindex.%s", sha1_to_hex(si->base->sha1)));
        if (!ret) {
                hashcpy(si->base_sha1, si->base->sha1);
@@@ -2553,22 -2572,7 +2560,22 @@@ int write_locked_index(struct index_sta
        new_shared_index = istate->cache_changed & SPLIT_INDEX_ORDERED;
  
        if (new_shared_index) {
 -              ret = write_shared_index(istate, lock, flags);
 +              struct tempfile *temp;
 +              int saved_errno;
 +
 +              temp = mks_tempfile(git_path("sharedindex_XXXXXX"));
 +              if (!temp) {
 +                      hashclr(si->base_sha1);
 +                      ret = do_write_locked_index(istate, lock, flags);
 +                      goto out;
 +              }
 +              ret = write_shared_index(istate, &temp);
 +
 +              saved_errno = errno;
 +              if (is_tempfile_active(temp))
 +                      delete_tempfile(&temp);
 +              errno = saved_errno;
 +
                if (ret)
                        goto out;
        }
        ret = write_split_index(istate, lock, flags);
  
        /* Freshen the shared index only if the split-index was written */
 -      if (!ret && !new_shared_index)
 -              freshen_shared_index(sha1_to_hex(si->base_sha1), 1);
 +      if (!ret && !new_shared_index) {
 +              const char *shared_index = git_path("sharedindex.%s",
 +                                                  sha1_to_hex(si->base_sha1));
 +              freshen_shared_index(shared_index, 1);
 +      }
  
  out:
        if (flags & COMMIT_LOCK)