repository: introduce parsed objects field

Convert the existing global cache for parsed objects (obj_hash) into
repository-specific parsed object caches. Existing code that uses
obj_hash are modified to use the parsed object cache of
the_repository; future patches will use the parsed object caches of
other repositories.

Another future use case for a pool of objects is ease of memory management
in revision walking: If we can free the rev-list related memory early in
pack-objects (e.g. part of repack operation) then it could lower memory
pressure significantly when running on large repos. While this has been
discussed on the mailing list lately, this series doesn't implement this.

Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/object.c b/object.c
index 5044d08e96c287b722fcf300dfdf960fe27d2442..f7c624a7ba66280ac2609b61e06feec49072da03 100644 (file)
--- a/object.c
+++ b/object.c
@@ -8,17 +8,14 @@
 #include "object-store.h"
 #include "packfile.h"
 
 #include "object-store.h"
 #include "packfile.h"
 

-static struct object **obj_hash;
-static int nr_objs, obj_hash_size;
-

 unsigned int get_max_object_index(void)
 {
 unsigned int get_max_object_index(void)
 {

-       return obj_hash_size;
+       return the_repository->parsed_objects->obj_hash_size;

 }
 
 struct object *get_indexed_object(unsigned int idx)
 {
 }
 
 struct object *get_indexed_object(unsigned int idx)
 {

-       return obj_hash[idx];
+       return the_repository->parsed_objects->obj_hash[idx];

 }
 
 static const char *object_type_strings[] = {
 }
 
 static const char *object_type_strings[] = {


@@ -90,15 +87,16 @@ struct object *lookup_object(const unsigned char *sha1)
        unsigned int i, first;
        struct object *obj;
 
        unsigned int i, first;
        struct object *obj;
 

-       if (!obj_hash)
+       if (!the_repository->parsed_objects->obj_hash)

                return NULL;
 
                return NULL;
 

-       first = i = hash_obj(sha1, obj_hash_size);
-       while ((obj = obj_hash[i]) != NULL) {
+       first = i = hash_obj(sha1,
+                            the_repository->parsed_objects->obj_hash_size);
+       while ((obj = the_repository->parsed_objects->obj_hash[i]) != NULL) {

                if (!hashcmp(sha1, obj->oid.hash))
                        break;
                i++;
                if (!hashcmp(sha1, obj->oid.hash))
                        break;
                i++;

-               if (i == obj_hash_size)
+               if (i == the_repository->parsed_objects->obj_hash_size)

                        i = 0;
        }
        if (obj && i != first) {
                        i = 0;
        }
        if (obj && i != first) {


@@ -107,7 +105,8 @@ struct object *lookup_object(const unsigned char *sha1)
                 * that we do not need to walk the hash table the next
                 * time we look for it.
                 */
                 * that we do not need to walk the hash table the next
                 * time we look for it.
                 */

-               SWAP(obj_hash[i], obj_hash[first]);
+               SWAP(the_repository->parsed_objects->obj_hash[i],
+                    the_repository->parsed_objects->obj_hash[first]);

        }
        return obj;
 }
        }
        return obj;
 }


@@ -124,19 +123,19 @@ static void grow_object_hash(void)
         * Note that this size must always be power-of-2 to match hash_obj
         * above.
         */
         * Note that this size must always be power-of-2 to match hash_obj
         * above.
         */

-       int new_hash_size = obj_hash_size < 32 ? 32 : 2 * obj_hash_size;
+       int new_hash_size = the_repository->parsed_objects->obj_hash_size < 32 ? 32 : 2 * the_repository->parsed_objects->obj_hash_size;

        struct object **new_hash;
 
        new_hash = xcalloc(new_hash_size, sizeof(struct object *));
        struct object **new_hash;
 
        new_hash = xcalloc(new_hash_size, sizeof(struct object *));

-       for (i = 0; i < obj_hash_size; i++) {
-               struct object *obj = obj_hash[i];
+       for (i = 0; i < the_repository->parsed_objects->obj_hash_size; i++) {
+               struct object *obj = the_repository->parsed_objects->obj_hash[i];

                if (!obj)
                        continue;
                insert_obj_hash(obj, new_hash, new_hash_size);
        }
                if (!obj)
                        continue;
                insert_obj_hash(obj, new_hash, new_hash_size);
        }

-       free(obj_hash);
-       obj_hash = new_hash;
-       obj_hash_size = new_hash_size;
+       free(the_repository->parsed_objects->obj_hash);
+       the_repository->parsed_objects->obj_hash = new_hash;
+       the_repository->parsed_objects->obj_hash_size = new_hash_size;

 }
 
 void *create_object(const unsigned char *sha1, void *o)
 }
 
 void *create_object(const unsigned char *sha1, void *o)


@@ -147,11 +146,12 @@ void *create_object(const unsigned char *sha1, void *o)
        obj->flags = 0;
        hashcpy(obj->oid.hash, sha1);
 
        obj->flags = 0;
        hashcpy(obj->oid.hash, sha1);
 

-       if (obj_hash_size - 1 <= nr_objs * 2)
+       if (the_repository->parsed_objects->obj_hash_size - 1 <= the_repository->parsed_objects->nr_objs * 2)

                grow_object_hash();
 
                grow_object_hash();
 

-       insert_obj_hash(obj, obj_hash, obj_hash_size);
-       nr_objs++;
+       insert_obj_hash(obj, the_repository->parsed_objects->obj_hash,
+                       the_repository->parsed_objects->obj_hash_size);
+       the_repository->parsed_objects->nr_objs++;

        return obj;
 }
 
        return obj;
 }
 


@@ -431,8 +431,8 @@ void clear_object_flags(unsigned flags)
 {
        int i;
 
 {
        int i;
 

-       for (i=0; i < obj_hash_size; i++) {
-               struct object *obj = obj_hash[i];
+       for (i=0; i < the_repository->parsed_objects->obj_hash_size; i++) {
+               struct object *obj = the_repository->parsed_objects->obj_hash[i];

                if (obj)
                        obj->flags &= ~flags;
        }
                if (obj)
                        obj->flags &= ~flags;
        }


@@ -442,13 +442,20 @@ void clear_commit_marks_all(unsigned int flags)
 {
        int i;
 
 {
        int i;
 

-       for (i = 0; i < obj_hash_size; i++) {
-               struct object *obj = obj_hash[i];
+       for (i = 0; i < the_repository->parsed_objects->obj_hash_size; i++) {
+               struct object *obj = the_repository->parsed_objects->obj_hash[i];

                if (obj && obj->type == OBJ_COMMIT)
                        obj->flags &= ~flags;
        }
 }
 
                if (obj && obj->type == OBJ_COMMIT)
                        obj->flags &= ~flags;
        }
 }
 

+struct parsed_object_pool *parsed_object_pool_new(void)
+{
+       struct parsed_object_pool *o = xmalloc(sizeof(*o));
+       memset(o, 0, sizeof(*o));
+       return o;
+}
+

 struct raw_object_store *raw_object_store_new(void)
 {
        struct raw_object_store *o = xmalloc(sizeof(*o));
 struct raw_object_store *raw_object_store_new(void)
 {
        struct raw_object_store *o = xmalloc(sizeof(*o));


@@ -488,3 +495,13 @@ void raw_object_store_clear(struct raw_object_store *o)
        close_all_packs(o);
        o->packed_git = NULL;
 }
        close_all_packs(o);
        o->packed_git = NULL;
 }

+
+void parsed_object_pool_clear(struct parsed_object_pool *o)
+{
+       /*
+        * TOOD free objects in o->obj_hash.
+        *
+        * As objects are allocated in slabs (see alloc.c), we do
+        * not need to free each object, but each slab instead.
+        */
+}

diff --git a/object.h b/object.h
index f13f85b2a94e3afc15debfbaf89416b5cda45acb..cecda7da370e28e008b8f4fd02c26b9d63c5e849 100644 (file)
--- a/object.h
+++ b/object.h
@@ -1,6 +1,14 @@
 #ifndef OBJECT_H
 #define OBJECT_H
 
 #ifndef OBJECT_H
 #define OBJECT_H
 

+struct parsed_object_pool {
+       struct object **obj_hash;
+       int nr_objs, obj_hash_size;
+};
+
+struct parsed_object_pool *parsed_object_pool_new(void);
+void parsed_object_pool_clear(struct parsed_object_pool *o);
+

 struct object_list {
        struct object *item;
        struct object_list *next;
 struct object_list {
        struct object *item;
        struct object_list *next;

diff --git a/repository.c b/repository.c
index a4848c1bd05e94efbf7aadf155c2cce504523c83..c23404677eb0bcd16564bdb93c438894ad37eca8 100644 (file)
--- a/repository.c
+++ b/repository.c
@@ -2,6 +2,7 @@
 #include "repository.h"
 #include "object-store.h"
 #include "config.h"
 #include "repository.h"
 #include "object-store.h"
 #include "config.h"

+#include "object.h"

 #include "submodule-config.h"
 
 /* The main repository */
 #include "submodule-config.h"
 
 /* The main repository */


@@ -14,6 +15,8 @@ void initialize_the_repository(void)
 
        the_repo.index = &the_index;
        the_repo.objects = raw_object_store_new();
 
        the_repo.index = &the_index;
        the_repo.objects = raw_object_store_new();

+       the_repo.parsed_objects = parsed_object_pool_new();
+

        repo_set_hash_algo(&the_repo, GIT_HASH_SHA1);
 }
 
        repo_set_hash_algo(&the_repo, GIT_HASH_SHA1);
 }
 


@@ -143,6 +146,7 @@ static int repo_init(struct repository *repo,
        memset(repo, 0, sizeof(*repo));
 
        repo->objects = raw_object_store_new();
        memset(repo, 0, sizeof(*repo));
 
        repo->objects = raw_object_store_new();

+       repo->parsed_objects = parsed_object_pool_new();

 
        if (repo_init_gitdir(repo, gitdir))
                goto error;
 
        if (repo_init_gitdir(repo, gitdir))
                goto error;


@@ -226,6 +230,9 @@ void repo_clear(struct repository *repo)
        raw_object_store_clear(repo->objects);
        FREE_AND_NULL(repo->objects);
 
        raw_object_store_clear(repo->objects);
        FREE_AND_NULL(repo->objects);
 

+       parsed_object_pool_clear(repo->parsed_objects);
+       FREE_AND_NULL(repo->parsed_objects);
+

        if (repo->config) {
                git_configset_clear(repo->config);
                FREE_AND_NULL(repo->config);
        if (repo->config) {
                git_configset_clear(repo->config);
                FREE_AND_NULL(repo->config);

diff --git a/repository.h b/repository.h
index e6e00f541bdc7f61b08ad881b8d66047cc0f1d02..6d199819905f2a9f8334e496f2caf35a3546420c 100644 (file)
--- a/repository.h
+++ b/repository.h
@@ -26,6 +26,15 @@ struct repository {
         */
        struct raw_object_store *objects;
 
         */
        struct raw_object_store *objects;
 

+       /*
+        * All objects in this repository that have been parsed. This structure
+        * owns all objects it references, so users of "struct object *"
+        * generally do not need to free them; instead, when a repository is no
+        * longer used, call parsed_object_pool_clear() on this structure, which
+        * is called by the repositories repo_clear on its desconstruction.
+        */
+       struct parsed_object_pool *parsed_objects;
+

        /* The store in which the refs are held. */
        struct ref_store *refs;
 
        /* The store in which the refs are held. */
        struct ref_store *refs;