sha1_file.c: Don't retain open fds on small packs

If a pack file is small enough that its entire contents fits within
one mmap window, mmap the file and then immediately close its file
descriptor.  This reduces the number of file descriptors that are
needed to read from repositories with many tiny pack files, such
as one that has received 1000 pushes (and created 1000 small pack
files) since its last repack.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/cache.h b/cache.h
index 3abf8950bdd9f950b65ee4544e0cfe6517f1912a..663ab5804bb0ed28f51f94f4fc4f8f38377fa27e 100644 (file)
--- a/cache.h
+++ b/cache.h
@@ -899,7 +899,8 @@ extern struct packed_git {
        time_t mtime;
        int pack_fd;
        unsigned pack_local:1,
        time_t mtime;
        int pack_fd;
        unsigned pack_local:1,

-                pack_keep:1;
+                pack_keep:1,
+                do_not_close:1;

        unsigned char sha1[20];
        /* something like ".git/objects/pack/xxxxx.pack" */
        char pack_name[FLEX_ARRAY]; /* more */
        unsigned char sha1[20];
        /* something like ".git/objects/pack/xxxxx.pack" */
        char pack_name[FLEX_ARRAY]; /* more */

diff --git a/fast-import.c b/fast-import.c
index 970d8470ed7c350337f28570fd2c111186a7941d..2369a7b30ae9247da06392e4ea7fa1a9e0292022 100644 (file)
--- a/fast-import.c
+++ b/fast-import.c
@@ -871,6 +871,7 @@ static void start_packfile(void)
        p = xcalloc(1, sizeof(*p) + strlen(tmpfile) + 2);
        strcpy(p->pack_name, tmpfile);
        p->pack_fd = pack_fd;
        p = xcalloc(1, sizeof(*p) + strlen(tmpfile) + 2);
        strcpy(p->pack_name, tmpfile);
        p->pack_fd = pack_fd;

+       p->do_not_close = 1;

        pack_file = sha1fd(pack_fd, p->pack_name);
 
        hdr.hdr_signature = htonl(PACK_SIGNATURE);
        pack_file = sha1fd(pack_fd, p->pack_name);
 
        hdr.hdr_signature = htonl(PACK_SIGNATURE);

diff --git a/sha1_file.c b/sha1_file.c
index c6beed2d6b727a72c94a14e5fc63b217946c75f8..e194f6a12862a4a4479fd714b550d426f8ee1313 100644 (file)
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -596,7 +596,8 @@ static int unuse_one_window(struct packed_git *current, int keep_fd)
                        lru_l->next = lru_w->next;
                else {
                        lru_p->windows = lru_w->next;
                        lru_l->next = lru_w->next;
                else {
                        lru_p->windows = lru_w->next;

-                       if (!lru_p->windows && lru_p->pack_fd != keep_fd) {
+                       if (!lru_p->windows && lru_p->pack_fd != -1
+                               && lru_p->pack_fd != keep_fd) {

                                close(lru_p->pack_fd);
                                pack_open_fds--;
                                lru_p->pack_fd = -1;
                                close(lru_p->pack_fd);
                                pack_open_fds--;
                                lru_p->pack_fd = -1;


@@ -812,14 +813,13 @@ unsigned char *use_pack(struct packed_git *p,
 {
        struct pack_window *win = *w_cursor;
 
 {
        struct pack_window *win = *w_cursor;
 

-       if (p->pack_fd == -1 && open_packed_git(p))
-               die("packfile %s cannot be accessed", p->pack_name);
-

        /* Since packfiles end in a hash of their content and it's
         * pointless to ask for an offset into the middle of that
         * hash, and the in_window function above wouldn't match
         * don't allow an offset too close to the end of the file.
         */
        /* Since packfiles end in a hash of their content and it's
         * pointless to ask for an offset into the middle of that
         * hash, and the in_window function above wouldn't match
         * don't allow an offset too close to the end of the file.
         */

+       if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p))
+               die("packfile %s cannot be accessed", p->pack_name);

        if (offset > (p->pack_size - 20))
                die("offset beyond end of packfile (truncated pack?)");
 
        if (offset > (p->pack_size - 20))
                die("offset beyond end of packfile (truncated pack?)");
 


@@ -833,6 +833,10 @@ unsigned char *use_pack(struct packed_git *p,
                if (!win) {
                        size_t window_align = packed_git_window_size / 2;
                        off_t len;
                if (!win) {
                        size_t window_align = packed_git_window_size / 2;
                        off_t len;

+
+                       if (p->pack_fd == -1 && open_packed_git(p))
+                               die("packfile %s cannot be accessed", p->pack_name);
+

                        win = xcalloc(1, sizeof(*win));
                        win->offset = (offset / window_align) * window_align;
                        len = p->pack_size - win->offset;
                        win = xcalloc(1, sizeof(*win));
                        win->offset = (offset / window_align) * window_align;
                        len = p->pack_size - win->offset;


@@ -850,6 +854,12 @@ unsigned char *use_pack(struct packed_git *p,
                                die("packfile %s cannot be mapped: %s",
                                        p->pack_name,
                                        strerror(errno));
                                die("packfile %s cannot be mapped: %s",
                                        p->pack_name,
                                        strerror(errno));

+                       if (!win->offset && win->len == p->pack_size
+                               && !p->do_not_close) {
+                               close(p->pack_fd);
+                               pack_open_fds--;
+                               p->pack_fd = -1;
+                       }

                        pack_mmap_calls++;
                        pack_open_windows++;
                        if (pack_mapped > peak_pack_mapped)
                        pack_mmap_calls++;
                        pack_open_windows++;
                        if (pack_mapped > peak_pack_mapped)


@@ -1950,6 +1960,27 @@ off_t find_pack_entry_one(const unsigned char *sha1,
        return 0;
 }
 
        return 0;
 }
 

+static int is_pack_valid(struct packed_git *p)
+{
+       /* An already open pack is known to be valid. */
+       if (p->pack_fd != -1)
+               return 1;
+
+       /* If the pack has one window completely covering the
+        * file size, the pack is known to be valid even if
+        * the descriptor is not currently open.
+        */
+       if (p->windows) {
+               struct pack_window *w = p->windows;
+
+               if (!w->offset && w->len == p->pack_size)
+                       return 1;
+       }
+
+       /* Force the pack to open to prove its valid. */
+       return !open_packed_git(p);
+}
+

 static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
 {
        static struct packed_git *last_found = (void *)1;
 static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
 {
        static struct packed_git *last_found = (void *)1;


@@ -1979,7 +2010,7 @@ static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
                         * it may have been deleted since the index
                         * was loaded!
                         */
                         * it may have been deleted since the index
                         * was loaded!
                         */

-                       if (p->pack_fd == -1 && open_packed_git(p)) {
+                       if (!is_pack_valid(p)) {

                                error("packfile %s cannot be accessed", p->pack_name);
                                goto next;
                        }
                                error("packfile %s cannot be accessed", p->pack_name);
                                goto next;
                        }