sha1_name: cache readdir(3) results in find_short_object_filename()

Read each loose object subdirectory at most once when looking for unique
abbreviated hashes.  This speeds up commands like "git log --pretty=%h"
considerably, which previously caused one readdir(3) call for each
candidate, even for subdirectories that were visited before.

The new cache is kept until the program ends and never invalidated.  The
same is already true for pack indexes.  The inherent racy nature of
finding unique short hashes makes it still fit for this purpose -- a
conflicting new object may be added at any time.  Tasks with higher
consistency requirements should not use it, though.

The cached object names are stored in an oid_array, which is quite
compact.  The bitmap for remembering which subdir was already read is
stored as a char array, with one char per directory -- that's not quite
as compact, but really simple and incurs only an overhead equivalent to
11 hashes after all.

Suggested-by: Jeff King <peff@peff.net>
Helped-by: Jeff King <peff@peff.net>
Signed-off-by: Rene Scharfe <l.s.r@web.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/cache.h b/cache.h
index e1f0e182ad011d5a3b6912f48b597dcc6d1070b0..1ad914084b9981ff8abdd03d51712546b69950d8 100644 (file)
--- a/cache.h
+++ b/cache.h
@@ -11,6 +11,7 @@
 #include "string-list.h"
 #include "pack-revindex.h"
 #include "hash.h"
 #include "string-list.h"
 #include "pack-revindex.h"
 #include "hash.h"

+#include "sha1-array.h"

 
 #ifndef platform_SHA_CTX
 /*
 
 #ifndef platform_SHA_CTX
 /*


@@ -1579,6 +1580,16 @@ extern struct alternate_object_database {
        struct strbuf scratch;
        size_t base_len;
 
        struct strbuf scratch;
        size_t base_len;
 

+       /*
+        * Used to store the results of readdir(3) calls when searching
+        * for unique abbreviated hashes.  This cache is never
+        * invalidated, thus it's racy and not necessarily accurate.
+        * That's fine for its purpose; don't use it for tasks requiring
+        * greater accuracy!
+        */
+       char loose_objects_subdir_seen[256];
+       struct oid_array loose_objects_cache;
+

        char path[FLEX_ARRAY];
 } *alt_odb_list;
 extern void prepare_alt_odb(void);
        char path[FLEX_ARRAY];
 } *alt_odb_list;
 extern void prepare_alt_odb(void);


@@ -1797,6 +1808,12 @@ typedef int each_loose_cruft_fn(const char *basename,
 typedef int each_loose_subdir_fn(int nr,
                                 const char *path,
                                 void *data);
 typedef int each_loose_subdir_fn(int nr,
                                 const char *path,
                                 void *data);

+int for_each_file_in_obj_subdir(int subdir_nr,
+                               struct strbuf *path,
+                               each_loose_object_fn obj_cb,
+                               each_loose_cruft_fn cruft_cb,
+                               each_loose_subdir_fn subdir_cb,
+                               void *data);

 int for_each_loose_file_in_objdir(const char *path,
                                  each_loose_object_fn obj_cb,
                                  each_loose_cruft_fn cruft_cb,
 int for_each_loose_file_in_objdir(const char *path,
                                  each_loose_object_fn obj_cb,
                                  each_loose_cruft_fn cruft_cb,

diff --git a/sha1_file.c b/sha1_file.c
index 59a4ed2ed32336b41ab8f3b1d4aca30f045aa084..5e0ee2b68bcf50611a36edaf218bc89446303278 100644 (file)
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -3735,12 +3735,12 @@ void assert_sha1_type(const unsigned char *sha1, enum object_type expect)
                    typename(expect));
 }
 
                    typename(expect));
 }
 

-static int for_each_file_in_obj_subdir(int subdir_nr,
-                                      struct strbuf *path,
-                                      each_loose_object_fn obj_cb,
-                                      each_loose_cruft_fn cruft_cb,
-                                      each_loose_subdir_fn subdir_cb,
-                                      void *data)
+int for_each_file_in_obj_subdir(int subdir_nr,
+                               struct strbuf *path,
+                               each_loose_object_fn obj_cb,
+                               each_loose_cruft_fn cruft_cb,
+                               each_loose_subdir_fn subdir_cb,
+                               void *data)

 {
        size_t baselen = path->len;
        DIR *dir = opendir(path->buf);
 {
        size_t baselen = path->len;
        DIR *dir = opendir(path->buf);

diff --git a/sha1_name.c b/sha1_name.c
index 8eec9f7c1bb59b8124e098367b7bff552bcafcf4..76cb76a844ff2c916c55314d2447e12a5573e2f1 100644 (file)
--- a/sha1_name.c
+++ b/sha1_name.c
@@ -77,10 +77,19 @@ static void update_candidates(struct disambiguate_state *ds, const struct object
        /* otherwise, current can be discarded and candidate is still good */
 }
 
        /* otherwise, current can be discarded and candidate is still good */
 }
 

+static int append_loose_object(const struct object_id *oid, const char *path,
+                              void *data)
+{
+       oid_array_append(data, oid);
+       return 0;
+}
+
+static int match_sha(unsigned, const unsigned char *, const unsigned char *);
+

 static void find_short_object_filename(struct disambiguate_state *ds)
 {
 static void find_short_object_filename(struct disambiguate_state *ds)
 {

+       int subdir_nr = ds->bin_pfx.hash[0];

        struct alternate_object_database *alt;
        struct alternate_object_database *alt;

-       char hex[GIT_MAX_HEXSZ];

        static struct alternate_object_database *fakeent;
 
        if (!fakeent) {
        static struct alternate_object_database *fakeent;
 
        if (!fakeent) {


@@ -95,29 +104,30 @@ static void find_short_object_filename(struct disambiguate_state *ds)
        }
        fakeent->next = alt_odb_list;
 
        }
        fakeent->next = alt_odb_list;
 

-       xsnprintf(hex, sizeof(hex), "%.2s", ds->hex_pfx);

        for (alt = fakeent; alt && !ds->ambiguous; alt = alt->next) {
        for (alt = fakeent; alt && !ds->ambiguous; alt = alt->next) {

-               struct strbuf *buf = alt_scratch_buf(alt);
-               struct dirent *de;
-               DIR *dir;
-
-               strbuf_addf(buf, "%.2s/", ds->hex_pfx);
-               dir = opendir(buf->buf);
-               if (!dir)
-                       continue;
+               int pos;

 
 

-               while (!ds->ambiguous && (de = readdir(dir)) != NULL) {
-                       struct object_id oid;
+               if (!alt->loose_objects_subdir_seen[subdir_nr]) {
+                       struct strbuf *buf = alt_scratch_buf(alt);
+                       strbuf_addf(buf, "%02x/", subdir_nr);
+                       for_each_file_in_obj_subdir(subdir_nr, buf,
+                                                   append_loose_object,
+                                                   NULL, NULL,
+                                                   &alt->loose_objects_cache);
+                       alt->loose_objects_subdir_seen[subdir_nr] = 1;
+               }

 
 

-                       if (strlen(de->d_name) != GIT_SHA1_HEXSZ - 2)
-                               continue;
-                       if (memcmp(de->d_name, ds->hex_pfx + 2, ds->len - 2))
-                               continue;
-                       memcpy(hex + 2, de->d_name, GIT_SHA1_HEXSZ - 2);
-                       if (!get_oid_hex(hex, &oid))
-                               update_candidates(ds, &oid);
+               pos = oid_array_lookup(&alt->loose_objects_cache, &ds->bin_pfx);
+               if (pos < 0)
+                       pos = -1 - pos;
+               while (!ds->ambiguous && pos < alt->loose_objects_cache.nr) {
+                       const struct object_id *oid;
+                       oid = alt->loose_objects_cache.oid + pos;
+                       if (!match_sha(ds->len, ds->bin_pfx.hash, oid->hash))
+                               break;
+                       update_candidates(ds, oid);
+                       pos++;

                }
                }

-               closedir(dir);

        }
 }
 
        }
 }