git-svnimport: support for incremental import

This adds ability to do import "in chunks" (default 1000 revisions),
after each chunk git repo will be repacked. The option -R is used to
change default value of chunk size (or how often repository will
repacked).

Signed-off-by: Sasha Khapyorsky <sashak@voltaire.com>
Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/Documentation/git-svnimport.txt b/Documentation/git-svnimport.txt
index 2c7c7dad54b5d2c773194b494d98ea521bc54d2f..b166cf3327380a8f386825a7fbce92f2447bb54e 100644 (file)
--- a/Documentation/git-svnimport.txt
+++ b/Documentation/git-svnimport.txt
@@ -15,7 +15,7 @@ SYNOPSIS
                [ -b branch_subdir ] [ -T trunk_subdir ] [ -t tag_subdir ]
                [ -s start_chg ] [ -m ] [ -r ] [ -M regex ]
                [ -I <ignorefile_name> ] [ -A <author_file> ]
                [ -b branch_subdir ] [ -T trunk_subdir ] [ -t tag_subdir ]
                [ -s start_chg ] [ -m ] [ -r ] [ -M regex ]
                [ -I <ignorefile_name> ] [ -A <author_file> ]

-               [ -P <path_from_trunk> ]
+               [ -R <repack_each_revs>] [ -P <path_from_trunk> ]

                <SVN_repository_URL> [ <path> ]
 
 
                <SVN_repository_URL> [ <path> ]
 
 


@@ -108,6 +108,14 @@ repository without -A.
 Formerly, this option controlled how many revisions to pull,
 due to SVN memory leaks. (These have been worked around.)
 
 Formerly, this option controlled how many revisions to pull,
 due to SVN memory leaks. (These have been worked around.)
 

+-R <repack_each_revs>::
+       Specify how often git repository should be repacked.
++
+The default value is 1000. git-svnimport will do import in chunks of 1000
+revisions, after each chunk git repository will be repacked. To disable
+this behavior specify some big value here which is mote than number of
+revisions to import.
+

 -P <path_from_trunk>::
        Partial import of the SVN tree.
 +
 -P <path_from_trunk>::
        Partial import of the SVN tree.
 +

diff --git a/git-svnimport.perl b/git-svnimport.perl
index cbaa8ab37c78769002e10947d7ca08fc440e552f..f31fcf84ed3150a1d53676cd1e01ba276b122cba 100755 (executable)
--- a/git-svnimport.perl
+++ b/git-svnimport.perl
@@ -31,12 +31,13 @@
 $ENV{'TZ'}="UTC";
 
 our($opt_h,$opt_o,$opt_v,$opt_u,$opt_C,$opt_i,$opt_m,$opt_M,$opt_t,$opt_T,
 $ENV{'TZ'}="UTC";
 
 our($opt_h,$opt_o,$opt_v,$opt_u,$opt_C,$opt_i,$opt_m,$opt_M,$opt_t,$opt_T,

-    $opt_b,$opt_r,$opt_I,$opt_A,$opt_s,$opt_l,$opt_d,$opt_D,$opt_S,$opt_F,$opt_P);
+    $opt_b,$opt_r,$opt_I,$opt_A,$opt_s,$opt_l,$opt_d,$opt_D,$opt_S,$opt_F,
+    $opt_P,$opt_R);

 
 sub usage() {
        print STDERR <<END;
 Usage: ${\basename $0}     # fetch/update GIT from SVN
 
 sub usage() {
        print STDERR <<END;
 Usage: ${\basename $0}     # fetch/update GIT from SVN

-       [-o branch-for-HEAD] [-h] [-v] [-l max_rev]
+       [-o branch-for-HEAD] [-h] [-v] [-l max_rev] [-R repack_each_revs]

        [-C GIT_repository] [-t tagname] [-T trunkname] [-b branchname]
        [-d|-D] [-i] [-u] [-r] [-I ignorefilename] [-s start_chg]
        [-m] [-M regex] [-A author_file] [-S] [-F] [-P project_name] [SVN_URL]
        [-C GIT_repository] [-t tagname] [-T trunkname] [-b branchname]
        [-d|-D] [-i] [-u] [-r] [-I ignorefilename] [-s start_chg]
        [-m] [-M regex] [-A author_file] [-S] [-F] [-P project_name] [SVN_URL]


@@ -44,7 +45,7 @@ END
        exit(1);
 }
 
        exit(1);
 }
 

-getopts("A:b:C:dDFhiI:l:mM:o:rs:t:T:SP:uv") or usage();
+getopts("A:b:C:dDFhiI:l:mM:o:rs:t:T:SP:R:uv") or usage();

 usage if $opt_h;
 
 my $tag_name = $opt_t || "tags";
 usage if $opt_h;
 
 my $tag_name = $opt_t || "tags";


@@ -52,6 +53,7 @@ END
 my $branch_name = $opt_b || "branches";
 my $project_name = $opt_P || "";
 $project_name = "/" . $project_name if ($project_name);
 my $branch_name = $opt_b || "branches";
 my $project_name = $opt_P || "";
 $project_name = "/" . $project_name if ($project_name);

+my $repack_after = $opt_R || 1000;

 
 @ARGV == 1 or @ARGV == 2 or usage();
 
 
 @ARGV == 1 or @ARGV == 2 or usage();
 


@@ -934,11 +936,27 @@ sub commit_all {
     exit;
 }
 
     exit;
 }
 

-print "Fetching from $current_rev to $opt_l ...\n" if $opt_v;
+print "Processing from $current_rev to $opt_l ...\n" if $opt_v;

 
 

-my $pool=SVN::Pool->new;
-$svn->{'svn'}->get_log("/",$current_rev,$opt_l,0,1,1,\&commit_all,$pool);
-$pool->clear;
+my $from_rev;
+my $to_rev = $current_rev;
+
+while ($to_rev < $opt_l) {
+       $from_rev = $to_rev;
+       $to_rev = $from_rev + $repack_after;
+       $to_rev = $opt_l if $opt_l < $to_rev;
+       print "Fetching from $from_rev to $to_rev ...\n" if $opt_v;
+       my $pool=SVN::Pool->new;
+       $svn->{'svn'}->get_log("/",$from_rev,$to_rev,0,1,1,\&commit_all,$pool);
+       $pool->clear;
+       my $pid = fork();
+       die "Fork: $!\n" unless defined $pid;
+       unless($pid) {
+               exec("git-repack", "-d")
+                       or die "Cannot repack: $!\n";
+       }
+       waitpid($pid, 0);
+}

 
 
 unlink($git_index);
 
 
 unlink($git_index);