config: create core.multiPackIndex setting

The core.multiPackIndex config setting controls the multi-pack-
index (MIDX) feature. If false, the setting will disable all reads
from the multi-pack-index file.

Read this config setting in the new prepare_multi_pack_index_one()
which is called during prepare_packed_git(). This check is run once
per repository.

Add comparison commands in t5319-multi-pack-index.sh to check
typical Git behavior remains the same as the config setting is turned
on and off. This currently includes 'git rev-list' and 'git log'
commands to trigger several object database reads. Currently, these
would only catch an error in the prepare_multi_pack_index_one(), but
with later commits will catch errors in object lookups, abbreviations,
and approximate object counts.

Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/config.txt b/Documentation/config.txt
index ab641bf5a9984b7ab2dfea787a0db5704f79a448..25f817ca427c4506b941419dfcf8a12f915b32ca 100644 (file)
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -908,6 +908,11 @@ core.commitGraph::
        Enable git commit graph feature. Allows reading from the
        commit-graph file.
 
        Enable git commit graph feature. Allows reading from the
        commit-graph file.
 

+core.multiPackIndex::
+       Use the multi-pack-index file to track multiple packfiles using a
+       single index. See link:technical/multi-pack-index.html[the
+       multi-pack-index design document].
+

 core.sparseCheckout::
        Enable "sparse checkout" feature. See section "Sparse checkout" in
        linkgit:git-read-tree[1] for more information.
 core.sparseCheckout::
        Enable "sparse checkout" feature. See section "Sparse checkout" in
        linkgit:git-read-tree[1] for more information.

diff --git a/midx.c b/midx.c
index e83110ae92b4768802b6b70cb6b220ce91fc74b9..4090cf4ca42fe5b384a5b70bb79bc405d15d4e80 100644 (file)
--- a/midx.c
+++ b/midx.c
@@ -1,4 +1,5 @@
 #include "cache.h"
 #include "cache.h"

+#include "config.h"

 #include "csum-file.h"
 #include "dir.h"
 #include "lockfile.h"
 #include "csum-file.h"
 #include "dir.h"
 #include "lockfile.h"


@@ -177,6 +178,30 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir)
        return NULL;
 }
 
        return NULL;
 }
 

+int prepare_multi_pack_index_one(struct repository *r, const char *object_dir)
+{
+       struct multi_pack_index *m = r->objects->multi_pack_index;
+       struct multi_pack_index *m_search;
+       int config_value;
+
+       if (repo_config_get_bool(r, "core.multipackindex", &config_value) ||
+           !config_value)
+               return 0;
+
+       for (m_search = m; m_search; m_search = m_search->next)
+               if (!strcmp(object_dir, m_search->object_dir))
+                       return 1;
+
+       r->objects->multi_pack_index = load_multi_pack_index(object_dir);
+
+       if (r->objects->multi_pack_index) {
+               r->objects->multi_pack_index->next = m;
+               return 1;
+       }
+
+       return 0;
+}
+

 static size_t write_midx_header(struct hashfile *f,
                                unsigned char num_chunks,
                                uint32_t num_packs)
 static size_t write_midx_header(struct hashfile *f,
                                unsigned char num_chunks,
                                uint32_t num_packs)

diff --git a/midx.h b/midx.h
index e15966272ff4a5aab1edfa5e4751beb5ca47654b..9bcfc82d2ec3b275e1b23f01fd6c1dd0ebba858c 100644 (file)
--- a/midx.h
+++ b/midx.h
@@ -1,7 +1,11 @@
 #ifndef __MIDX_H__
 #define __MIDX_H__
 
 #ifndef __MIDX_H__
 #define __MIDX_H__
 

+#include "repository.h"
+

 struct multi_pack_index {
 struct multi_pack_index {

+       struct multi_pack_index *next;
+

        int fd;
 
        const unsigned char *data;
        int fd;
 
        const unsigned char *data;


@@ -25,6 +29,7 @@ struct multi_pack_index {
 };
 
 struct multi_pack_index *load_multi_pack_index(const char *object_dir);
 };
 
 struct multi_pack_index *load_multi_pack_index(const char *object_dir);

+int prepare_multi_pack_index_one(struct repository *r, const char *object_dir);

 
 int write_midx_file(const char *object_dir);
 
 
 int write_midx_file(const char *object_dir);
 

diff --git a/object-store.h b/object-store.h
index 13a766aea8ece947f2b266b4e376abf488fe3a80..c2b162489a161f6e3e01aa97dd3b0b81b82ea26d 100644 (file)
--- a/object-store.h
+++ b/object-store.h
@@ -105,6 +105,13 @@ struct raw_object_store {
         */
        struct oidmap *replace_map;
 
         */
        struct oidmap *replace_map;
 

+       /*
+        * private data
+        *
+        * should only be accessed directly by packfile.c and midx.c
+        */
+       struct multi_pack_index *multi_pack_index;
+

        /*
         * private data
         *
        /*
         * private data
         *

diff --git a/packfile.c b/packfile.c
index 3d652212c60633147c7df0c9d1acc135fc58d916..5d4493dbf4f9a4f047af54631541bd7138fa42af 100644 (file)
--- a/packfile.c
+++ b/packfile.c
@@ -15,6 +15,7 @@
 #include "tree-walk.h"
 #include "tree.h"
 #include "object-store.h"
 #include "tree-walk.h"
 #include "tree.h"
 #include "object-store.h"

+#include "midx.h"

 
 char *odb_pack_name(struct strbuf *buf,
                    const unsigned char *sha1,
 
 char *odb_pack_name(struct strbuf *buf,
                    const unsigned char *sha1,


@@ -935,10 +936,13 @@ static void prepare_packed_git(struct repository *r)
 
        if (r->objects->packed_git_initialized)
                return;
 
        if (r->objects->packed_git_initialized)
                return;

+       prepare_multi_pack_index_one(r, r->objects->objectdir);

        prepare_packed_git_one(r, r->objects->objectdir, 1);
        prepare_alt_odb(r);
        prepare_packed_git_one(r, r->objects->objectdir, 1);
        prepare_alt_odb(r);

-       for (alt = r->objects->alt_odb_list; alt; alt = alt->next)
+       for (alt = r->objects->alt_odb_list; alt; alt = alt->next) {
+               prepare_multi_pack_index_one(r, alt->path);

                prepare_packed_git_one(r, alt->path, 0);
                prepare_packed_git_one(r, alt->path, 0);

+       }

        rearrange_packed_git(r);
        prepare_packed_git_mru(r);
        r->objects->packed_git_initialized = 1;
        rearrange_packed_git(r);
        prepare_packed_git_mru(r);
        r->objects->packed_git_initialized = 1;

diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh
index 4a4fa26f7a5225752d63d699e9a01d1e5c6e02d6..b9661c7c660e206e864228e15f276e285fa05bb2 100755 (executable)
--- a/t/t5319-multi-pack-index.sh
+++ b/t/t5319-multi-pack-index.sh
@@ -3,6 +3,8 @@
 test_description='multi-pack-indexes'
 . ./test-lib.sh
 
 test_description='multi-pack-indexes'
 . ./test-lib.sh
 

+objdir=.git/objects
+

 midx_read_expect () {
        NUM_PACKS=$1
        NUM_OBJECTS=$2
 midx_read_expect () {
        NUM_PACKS=$1
        NUM_OBJECTS=$2


@@ -76,18 +78,35 @@ test_expect_success 'create objects' '
 '
 
 test_expect_success 'write midx with one v1 pack' '
 '
 
 test_expect_success 'write midx with one v1 pack' '

-       pack=$(git pack-objects --index-version=1 pack/test <obj-list) &&
-       test_when_finished rm pack/test-$pack.pack pack/test-$pack.idx pack/multi-pack-index &&
-       git multi-pack-index --object-dir=. write &&
-       midx_read_expect 1 18 4 .
+       pack=$(git pack-objects --index-version=1 $objdir/pack/test <obj-list) &&
+       test_when_finished rm $objdir/pack/test-$pack.pack \
+               $objdir/pack/test-$pack.idx $objdir/pack/multi-pack-index &&
+       git multi-pack-index --object-dir=$objdir write &&
+       midx_read_expect 1 18 4 $objdir

 '
 
 '
 

+midx_git_two_modes () {
+       git -c core.multiPackIndex=false $1 >expect &&
+       git -c core.multiPackIndex=true $1 >actual &&
+       test_cmp expect actual
+}
+
+compare_results_with_midx () {
+       MSG=$1
+       test_expect_success "check normal git operations: $MSG" '
+               midx_git_two_modes "rev-list --objects --all" &&
+               midx_git_two_modes "log --raw"
+       '
+}
+

 test_expect_success 'write midx with one v2 pack' '
 test_expect_success 'write midx with one v2 pack' '

-       git pack-objects --index-version=2,0x40 pack/test <obj-list &&
-       git multi-pack-index --object-dir=. write &&
-       midx_read_expect 1 18 4 .
+       git pack-objects --index-version=2,0x40 $objdir/pack/test <obj-list &&
+       git multi-pack-index --object-dir=$objdir write &&
+       midx_read_expect 1 18 4 $objdir

 '
 
 '
 

+compare_results_with_midx "one v2 pack"
+

 test_expect_success 'add more objects' '
        for i in $(test_seq 6 10)
        do
 test_expect_success 'add more objects' '
        for i in $(test_seq 6 10)
        do


@@ -97,25 +116,31 @@ test_expect_success 'add more objects' '
 '
 
 test_expect_success 'write midx with two packs' '
 '
 
 test_expect_success 'write midx with two packs' '

-       git pack-objects --index-version=1 pack/test-2 <obj-list &&
-       git multi-pack-index --object-dir=. write &&
-       midx_read_expect 2 34 4 .
+       git pack-objects --index-version=1 $objdir/pack/test-2 <obj-list &&
+       git multi-pack-index --object-dir=$objdir write &&
+       midx_read_expect 2 34 4 $objdir

 '
 
 '
 

+compare_results_with_midx "two packs"
+

 test_expect_success 'add more packs' '
        for j in $(test_seq 11 20)
        do
                generate_objects $j &&
                commit_and_list_objects &&
 test_expect_success 'add more packs' '
        for j in $(test_seq 11 20)
        do
                generate_objects $j &&
                commit_and_list_objects &&

-               git pack-objects --index-version=2 pack/test-pack <obj-list
+               git pack-objects --index-version=2 $objdir/pack/test-pack <obj-list

        done
 '
 
        done
 '
 

+compare_results_with_midx "mixed mode (two packs + extra)"
+

 test_expect_success 'write midx with twelve packs' '
 test_expect_success 'write midx with twelve packs' '

-       git multi-pack-index --object-dir=. write &&
-       midx_read_expect 12 74 4 .
+       git multi-pack-index --object-dir=$objdir write &&
+       midx_read_expect 12 74 4 $objdir

 '
 
 '
 

+compare_results_with_midx "twelve packs"
+

 # usage: corrupt_data <file> <pos> [<data>]
 corrupt_data () {
        file=$1
 # usage: corrupt_data <file> <pos> [<data>]
 corrupt_data () {
        file=$1