fast-import: implement unpack limit

With many incremental imports, small packs become highly
inefficient due to the need to readdir scan and load many
indices to locate even a single object.  Frequent repacking and
consolidation may be prohibitively expensive in terms of disk
I/O, especially in large repositories where the initial packs
were aggressively optimized and marked with .keep files.

In those cases, users may be better served with loose objects
and relying on "git gc --auto".

This changes the default behavior of fast-import for small
imports found in test cases, so adjustments to t9300 were
necessary.

Signed-off-by: Eric Wong <normalperson@yhbt.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/config.txt b/Documentation/config.txt
index 2cd6bdd7d2bc2816c1a9aed1c6a26bbd3285777c..283bf04091c1d365acb5576c607fb307cba689f8 100644 (file)
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -1153,6 +1153,15 @@ difftool.<tool>.cmd::
 difftool.prompt::
        Prompt before each invocation of the diff tool.
 
 difftool.prompt::
        Prompt before each invocation of the diff tool.
 

+fastimport.unpackLimit::
+       If the number of objects imported by linkgit:git-fast-import[1]
+       is below this limit, then the objects will be unpacked into
+       loose object files.  However if the number of imported objects
+       equals or exceeds this limit then the pack will be stored as a
+       pack.  Storing the pack from a fast-import can make the import
+       operation complete faster, especially on slow filesystems.  If
+       not set, the value of `transfer.unpackLimit` is used instead.
+

 fetch.recurseSubmodules::
        This option can be either set to a boolean value or to 'on-demand'.
        Setting it to a boolean changes the behavior of fetch and pull to
 fetch.recurseSubmodules::
        This option can be either set to a boolean value or to 'on-demand'.
        Setting it to a boolean changes the behavior of fetch and pull to

diff --git a/Documentation/git-fast-import.txt b/Documentation/git-fast-import.txt
index 66910aa2faff1d78b71c5c552550a4f8e9d51832..644df993f924d3265db10af6e6c29e33daf95482 100644 (file)
--- a/Documentation/git-fast-import.txt
+++ b/Documentation/git-fast-import.txt
@@ -136,6 +136,8 @@ Performance and Compression Tuning
        Maximum size of each output packfile.
        The default is unlimited.
 
        Maximum size of each output packfile.
        The default is unlimited.
 

+fastimport.unpackLimit::
+       See linkgit:git-config[1]

 
 Performance
 -----------
 
 Performance
 -----------

diff --git a/fast-import.c b/fast-import.c
index 9fc7093406b1e2085a3b2180a8506bcd6d486e06..4fb464c1ef553528021b981132919551517412ac 100644 (file)
--- a/fast-import.c
+++ b/fast-import.c
@@ -166,6 +166,7 @@ Format of STDIN stream:
 #include "quote.h"
 #include "exec_cmd.h"
 #include "dir.h"
 #include "quote.h"
 #include "exec_cmd.h"
 #include "dir.h"

+#include "run-command.h"

 
 #define PACK_ID_BITS 16
 #define MAX_PACK_ID ((1<<PACK_ID_BITS)-1)
 
 #define PACK_ID_BITS 16
 #define MAX_PACK_ID ((1<<PACK_ID_BITS)-1)


@@ -282,6 +283,7 @@ struct recent_command {
 /* Configured limits on output */
 static unsigned long max_depth = 10;
 static off_t max_packsize;
 /* Configured limits on output */
 static unsigned long max_depth = 10;
 static off_t max_packsize;

+static int unpack_limit = 100;

 static int force_update;
 static int pack_compression_level = Z_DEFAULT_COMPRESSION;
 static int pack_compression_seen;
 static int force_update;
 static int pack_compression_level = Z_DEFAULT_COMPRESSION;
 static int pack_compression_seen;


@@ -950,6 +952,23 @@ static void unkeep_all_packs(void)
        }
 }
 
        }
 }
 

+static int loosen_small_pack(const struct packed_git *p)
+{
+       struct child_process unpack = CHILD_PROCESS_INIT;
+
+       if (lseek(p->pack_fd, 0, SEEK_SET) < 0)
+               die_errno("Failed seeking to start of '%s'", p->pack_name);
+
+       unpack.in = p->pack_fd;
+       unpack.git_cmd = 1;
+       unpack.stdout_to_stderr = 1;
+       argv_array_push(&unpack.args, "unpack-objects");
+       if (!show_stats)
+               argv_array_push(&unpack.args, "-q");
+
+       return run_command(&unpack);
+}
+

 static void end_packfile(void)
 {
        static int running;
 static void end_packfile(void)
 {
        static int running;


@@ -972,6 +991,12 @@ static void end_packfile(void)
                fixup_pack_header_footer(pack_data->pack_fd, pack_data->sha1,
                                    pack_data->pack_name, object_count,
                                    cur_pack_sha1, pack_size);
                fixup_pack_header_footer(pack_data->pack_fd, pack_data->sha1,
                                    pack_data->pack_name, object_count,
                                    cur_pack_sha1, pack_size);

+
+               if (object_count <= unpack_limit) {
+                       if (!loosen_small_pack(pack_data))
+                               goto discard_pack;
+               }
+

                close(pack_data->pack_fd);
                idx_name = keep_pack(create_index());
 
                close(pack_data->pack_fd);
                idx_name = keep_pack(create_index());
 


@@ -1002,6 +1027,7 @@ static void end_packfile(void)
                pack_id++;
        }
        else {
                pack_id++;
        }
        else {

+discard_pack:

                close(pack_data->pack_fd);
                unlink_or_warn(pack_data->pack_name);
        }
                close(pack_data->pack_fd);
                unlink_or_warn(pack_data->pack_name);
        }


@@ -3317,6 +3343,7 @@ static void parse_option(const char *option)
 static void git_pack_config(void)
 {
        int indexversion_value;
 static void git_pack_config(void)
 {
        int indexversion_value;

+       int limit;

        unsigned long packsizelimit_value;
 
        if (!git_config_get_ulong("pack.depth", &max_depth)) {
        unsigned long packsizelimit_value;
 
        if (!git_config_get_ulong("pack.depth", &max_depth)) {


@@ -3341,6 +3368,11 @@ static void git_pack_config(void)
        if (!git_config_get_ulong("pack.packsizelimit", &packsizelimit_value))
                max_packsize = packsizelimit_value;
 
        if (!git_config_get_ulong("pack.packsizelimit", &packsizelimit_value))
                max_packsize = packsizelimit_value;
 

+       if (!git_config_get_int("fastimport.unpacklimit", &limit))
+               unpack_limit = limit;
+       else if (!git_config_get_int("transfer.unpacklimit", &limit))
+               unpack_limit = limit;
+

        git_config(git_default_config, NULL);
 }
 
        git_config(git_default_config, NULL);
 }
 

diff --git a/t/t9300-fast-import.sh b/t/t9300-fast-import.sh
index 25bb60b2814320b628d3d12af88ee1d394a94217..e6a2b8a4d7939eb268eea799687c2b3b4b94e8ef 100755 (executable)
--- a/t/t9300-fast-import.sh
+++ b/t/t9300-fast-import.sh
@@ -52,6 +52,7 @@ echo "$@"'
 ###
 
 test_expect_success 'empty stream succeeds' '
 ###
 
 test_expect_success 'empty stream succeeds' '

+       git config fastimport.unpackLimit 0 &&

        git fast-import </dev/null
 '
 
        git fast-import </dev/null
 '
 


@@ -2675,6 +2676,7 @@ test_expect_success 'R: blob bigger than threshold' '
        echo >>input &&
 
        test_create_repo R &&
        echo >>input &&
 
        test_create_repo R &&

+       git --git-dir=R/.git config fastimport.unpackLimit 0 &&

        git --git-dir=R/.git fast-import --big-file-threshold=1 <input
 '
 
        git --git-dir=R/.git fast-import --big-file-threshold=1 <input
 '
 

diff --git a/t/t9302-fast-import-unpack-limit.sh b/t/t9302-fast-import-unpack-limit.sh
new file mode 100755 (executable)
index 0000000..0f686d2
--- /dev/null
+++ b/t/t9302-fast-import-unpack-limit.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+test_description='test git fast-import unpack limit'
+. ./test-lib.sh
+
+test_expect_success 'create loose objects on import' '
+       test_tick &&
+       cat >input <<-INPUT_END &&
+       commit refs/heads/master
+       committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+       data <<COMMIT
+       initial
+       COMMIT
+
+       done
+       INPUT_END
+
+       git -c fastimport.unpackLimit=2 fast-import --done <input &&
+       git fsck --no-progress &&
+       test $(find .git/objects/?? -type f | wc -l) -eq 2 &&
+       test $(find .git/objects/pack -type f | wc -l) -eq 0
+'
+
+test_expect_success 'bigger packs are preserved' '
+       test_tick &&
+       cat >input <<-INPUT_END &&
+       commit refs/heads/master
+       committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+       data <<COMMIT
+       incremental should create a pack
+       COMMIT
+       from refs/heads/master^0
+
+       commit refs/heads/branch
+       committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+       data <<COMMIT
+       branch
+       COMMIT
+
+       done
+       INPUT_END
+
+       git -c fastimport.unpackLimit=2 fast-import --done <input &&
+       git fsck --no-progress &&
+       test $(find .git/objects/?? -type f | wc -l) -eq 2 &&
+       test $(find .git/objects/pack -type f | wc -l) -eq 2
+'
+
+test_done