Cumulative update of merge-recursive in C
[gitweb.git] / git-cvsimport.perl
index 8c707f2c664ccfc9cad4fca443a5dee1d75604ba..e5a00a12857036d01e2e0dd18510df511378854f 100755 (executable)
 use warnings;
 use Getopt::Std;
 use File::Spec;
-use File::Temp qw(tempfile);
+use File::Temp qw(tempfile tmpnam);
 use File::Path qw(mkpath);
 use File::Basename qw(basename dirname);
 use Time::Local;
 use IO::Socket;
 use IO::Pipe;
-use POSIX qw(strftime dup2);
+use POSIX qw(strftime dup2 ENOENT);
 use IPC::Open2;
 
 $SIG{'PIPE'}="IGNORE";
 $ENV{'TZ'}="UTC";
 
-our($opt_h,$opt_o,$opt_v,$opt_k,$opt_u,$opt_d,$opt_p,$opt_C,$opt_z,$opt_i,$opt_P, $opt_s,$opt_m,$opt_M,$opt_A,$opt_S);
+our($opt_h,$opt_o,$opt_v,$opt_k,$opt_u,$opt_d,$opt_p,$opt_C,$opt_z,$opt_i,$opt_P, $opt_s,$opt_m,$opt_M,$opt_A,$opt_S,$opt_L);
 my (%conv_author_name, %conv_author_email);
 
 sub usage() {
@@ -85,7 +85,7 @@ ($)
        close ($f);
 }
 
-getopts("hivmkuo:d:p:C:z:s:M:P:A:S:") or usage();
+getopts("hivmkuo:d:p:C:z:s:M:P:A:S:L:") or usage();
 usage if $opt_h;
 
 @ARGV <= 1 or usage();
@@ -315,15 +315,7 @@ sub _line {
                        chomp $cnt;
                        die "Duh: Filesize $cnt" if $cnt !~ /^\d+$/;
                        $line="";
-                       $res=0;
-                       while($cnt) {
-                               my $buf;
-                               my $num = $self->{'socketi'}->read($buf,$cnt);
-                               die "Server: Filesize $cnt: $num: $!\n" if not defined $num or $num<=0;
-                               print $fh $buf;
-                               $res += $num;
-                               $cnt -= $num;
-                       }
+                       $res = $self->_fetchfile($fh, $cnt);
                } elsif($line =~ s/^ //) {
                        print $fh $line;
                        $res += length($line);
@@ -335,14 +327,7 @@ sub _line {
                        chomp $cnt;
                        die "Duh: Mbinary $cnt" if $cnt !~ /^\d+$/ or $cnt<1;
                        $line="";
-                       while($cnt) {
-                               my $buf;
-                               my $num = $self->{'socketi'}->read($buf,$cnt);
-                               die "S: Mbinary $cnt: $num: $!\n" if not defined $num or $num<=0;
-                               print $fh $buf;
-                               $res += $num;
-                               $cnt -= $num;
-                       }
+                       $res += $self->_fetchfile($fh, $cnt);
                } else {
                        chomp $line;
                        if($line eq "ok") {
@@ -384,6 +369,23 @@ sub file {
 
        return ($name, $res);
 }
+sub _fetchfile {
+       my ($self, $fh, $cnt) = @_;
+       my $res = 0;
+       my $bufsize = 1024 * 1024;
+       while($cnt) {
+           if ($bufsize > $cnt) {
+               $bufsize = $cnt;
+           }
+           my $buf;
+           my $num = $self->{'socketi'}->read($buf,$bufsize);
+           die "Server: Filesize $cnt: $num: $!\n" if not defined $num or $num<=0;
+           print $fh $buf;
+           $res += $num;
+           $cnt -= $num;
+       }
+       return $res;
+}
 
 
 package main;
@@ -429,22 +431,25 @@ ()
        return $pwd;
 }
 
+sub is_sha1 {
+       my $s = shift;
+       return $s =~ /^[a-f0-9]{40}$/;
+}
 
-sub get_headref($$) {
+sub get_headref ($$) {
     my $name    = shift;
     my $git_dir = shift; 
-    my $sha;
     
-    if (open(C,"$git_dir/refs/heads/$name")) {
-       chomp($sha = <C>);
-       close(C);
-       length($sha) == 40
-           or die "Cannot get head id for $name ($sha): $!\n";
+    my $f = "$git_dir/refs/heads/$name";
+    if(open(my $fh, $f)) {
+           chomp(my $r = <$fh>);
+           is_sha1($r) or die "Cannot get head id for $name ($r): $!";
+           return $r;
     }
-    return $sha;
+    die "unable to open $f: $!" unless $! == POSIX::ENOENT;
+    return undef;
 }
 
-
 -d $git_tree
        or mkdir($git_tree,0777)
        or die "Could not create $git_tree: $!";
@@ -460,10 +465,9 @@ ($$)
 $ENV{"GIT_DIR"} = $git_dir;
 my $orig_git_index;
 $orig_git_index = $ENV{GIT_INDEX_FILE} if exists $ENV{GIT_INDEX_FILE};
-my ($git_ih, $git_index) = tempfile('gitXXXXXX', SUFFIX => '.idx',
-                                   DIR => File::Spec->tmpdir());
-close ($git_ih);
-$ENV{GIT_INDEX_FILE} = $git_index;
+
+my %index; # holds filenames of one index per branch
+
 unless(-d $git_dir) {
        system("git-init-db");
        die "Cannot init the GIT db at $git_tree: $?\n" if $?;
@@ -490,10 +494,6 @@ ($$)
        $orig_branch = $last_branch;
        $tip_at_start = `git-rev-parse --verify HEAD`;
 
-       # populate index
-       system('git-read-tree', $last_branch);
-       die "read-tree failed: $?\n" if $?;
-
        # Get the last import timestamps
        opendir(D,"$git_dir/refs/heads");
        while(defined(my $head = readdir(D))) {
@@ -524,25 +524,39 @@ ($$)
        write_author_info("$git_dir/cvs-authors");
 }
 
-my $pid = open(CVS,"-|");
-die "Cannot fork: $!\n" unless defined $pid;
-unless($pid) {
-       my @opt;
-       @opt = split(/,/,$opt_p) if defined $opt_p;
-       unshift @opt, '-z', $opt_z if defined $opt_z;
-       unshift @opt, '-q'         unless defined $opt_v;
-       unless (defined($opt_p) && $opt_p =~ m/--no-cvs-direct/) {
-               push @opt, '--cvs-direct';
+
+#
+# run cvsps into a file unless we are getting
+# it passed as a file via $opt_P
+#
+unless ($opt_P) {
+       print "Running cvsps...\n" if $opt_v;
+       my $pid = open(CVSPS,"-|");
+       die "Cannot fork: $!\n" unless defined $pid;
+       unless($pid) {
+               my @opt;
+               @opt = split(/,/,$opt_p) if defined $opt_p;
+               unshift @opt, '-z', $opt_z if defined $opt_z;
+               unshift @opt, '-q'         unless defined $opt_v;
+               unless (defined($opt_p) && $opt_p =~ m/--no-cvs-direct/) {
+                       push @opt, '--cvs-direct';
+               }
+               exec("cvsps","--norc",@opt,"-u","-A",'--root',$opt_d,$cvs_tree);
+               die "Could not start cvsps: $!\n";
        }
-       if ($opt_P) {
-           exec("cat", $opt_P);
-       } else {
-           exec("cvsps","--norc",@opt,"-u","-A",'--root',$opt_d,$cvs_tree);
-           die "Could not start cvsps: $!\n";
+       my ($cvspsfh, $cvspsfile) = tempfile('gitXXXXXX', SUFFIX => '.cvsps',
+                                            DIR => File::Spec->tmpdir());
+       while (<CVSPS>) {
+           print $cvspsfh $_;
        }
+       close CVSPS;
+       close $cvspsfh;
+       $opt_P = $cvspsfile;
 }
 
 
+open(CVS, "<$opt_P") or die $!;
+
 ## cvsps output:
 #---------------------
 #PatchSet 314
@@ -561,98 +575,91 @@ ($$)
 
 my $state = 0;
 
-my($patchset,$date,$author_name,$author_email,$branch,$ancestor,$tag,$logmsg);
-my(@old,@new,@skipped);
-my $commit = sub {
-       my $pid;
-       while(@old) {
-               my @o2;
-               if(@old > 55) {
-                       @o2 = splice(@old,0,50);
-               } else {
-                       @o2 = @old;
-                       @old = ();
-               }
-               system("git-update-index","--force-remove","--",@o2);
-               die "Cannot remove files: $?\n" if $?;
-       }
-       while(@new) {
-               my @n2;
-               if(@new > 12) {
-                       @n2 = splice(@new,0,10);
-               } else {
-                       @n2 = @new;
-                       @new = ();
-               }
-               system("git-update-index","--add",
-                       (map { ('--cacheinfo', @$_) } @n2));
-               die "Cannot add files: $?\n" if $?;
-       }
+sub update_index (\@\@) {
+       my $old = shift;
+       my $new = shift;
+       open(my $fh, '|-', qw(git-update-index -z --index-info))
+               or die "unable to open git-update-index: $!";
+       print $fh
+               (map { "0 0000000000000000000000000000000000000000\t$_\0" }
+                       @$old),
+               (map { '100' . sprintf('%o', $_->[0]) . " $_->[1]\t$_->[2]\0" }
+                       @$new)
+               or die "unable to write to git-update-index: $!";
+       close $fh
+               or die "unable to write to git-update-index: $!";
+       $? and die "git-update-index reported error: $?";
+}
 
-       $pid = open(C,"-|");
-       die "Cannot fork: $!" unless defined $pid;
-       unless($pid) {
-               exec("git-write-tree");
-               die "Cannot exec git-write-tree: $!\n";
-       }
-       chomp(my $tree = <C>);
-       length($tree) == 40
-               or die "Cannot get tree id ($tree): $!\n";
-       close(C)
+sub write_tree () {
+       open(my $fh, '-|', qw(git-write-tree))
+               or die "unable to open git-write-tree: $!";
+       chomp(my $tree = <$fh>);
+       is_sha1($tree)
+               or die "Cannot get tree id ($tree): $!";
+       close($fh)
                or die "Error running git-write-tree: $?\n";
        print "Tree ID $tree\n" if $opt_v;
+       return $tree;
+}
 
-       my $parent = "";
-       if(open(C,"$git_dir/refs/heads/$last_branch")) {
-               chomp($parent = <C>);
-               close(C);
-               length($parent) == 40
-                       or die "Cannot get parent id ($parent): $!\n";
-               print "Parent ID $parent\n" if $opt_v;
-       }
+my($patchset,$date,$author_name,$author_email,$branch,$ancestor,$tag,$logmsg);
+my(@old,@new,@skipped,%ignorebranch);
 
-       my $pr = IO::Pipe->new() or die "Cannot open pipe: $!\n";
-       my $pw = IO::Pipe->new() or die "Cannot open pipe: $!\n";
-       $pid = fork();
-       die "Fork: $!\n" unless defined $pid;
-       unless($pid) {
-               $pr->writer();
-               $pw->reader();
-               open(OUT,">&STDOUT");
-               dup2($pw->fileno(),0);
-               dup2($pr->fileno(),1);
-               $pr->close();
-               $pw->close();
-
-               my @par = ();
-               @par = ("-p",$parent) if $parent;
-
-               # loose detection of merges
-               # based on the commit msg
-               foreach my $rx (@mergerx) {
-                       if ($logmsg =~ $rx) {
-                               my $mparent = $1;
-                               if ($mparent eq 'HEAD') { $mparent = $opt_o };
-                               if ( -e "$git_dir/refs/heads/$mparent") {
-                                       $mparent = get_headref($mparent, $git_dir);
-                                       push @par, '-p', $mparent;
-                                       print OUT "Merge parent branch: $mparent\n" if $opt_v;
-                               }
-                       }
-               }
+# commits that cvsps cannot place anywhere...
+$ignorebranch{'#CVSPS_NO_BRANCH'} = 1;
 
-               exec("env",
-                       "GIT_AUTHOR_NAME=$author_name",
-                       "GIT_AUTHOR_EMAIL=$author_email",
-                       "GIT_AUTHOR_DATE=".strftime("+0000 %Y-%m-%d %H:%M:%S",gmtime($date)),
-                       "GIT_COMMITTER_NAME=$author_name",
-                       "GIT_COMMITTER_EMAIL=$author_email",
-                       "GIT_COMMITTER_DATE=".strftime("+0000 %Y-%m-%d %H:%M:%S",gmtime($date)),
-                       "git-commit-tree", $tree,@par);
-               die "Cannot exec git-commit-tree: $!\n";
+sub commit {
+       if ($branch eq $opt_o && !$index{branch} && !get_headref($branch, $git_dir)) {
+           # looks like an initial commit
+           # use the index primed by git-init-db
+           $ENV{GIT_INDEX_FILE} = '.git/index';
+           $index{$branch} = '.git/index';
+       } else {
+           # use an index per branch to speed up
+           # imports of projects with many branches
+           unless ($index{$branch}) {
+               $index{$branch} = tmpnam();
+               $ENV{GIT_INDEX_FILE} = $index{$branch};
+               if ($ancestor) {
+                   system("git-read-tree", $ancestor);
+               } else {
+                   system("git-read-tree", $branch);
+               }
+               die "read-tree failed: $?\n" if $?;
+           }
        }
-       $pw->writer();
-       $pr->reader();
+        $ENV{GIT_INDEX_FILE} = $index{$branch};
+
+       update_index(@old, @new);
+       @old = @new = ();
+       my $tree = write_tree();
+       my $parent = get_headref($last_branch, $git_dir);
+       print "Parent ID " . ($parent ? $parent : "(empty)") . "\n" if $opt_v;
+
+       my @commit_args;
+       push @commit_args, ("-p", $parent) if $parent;
+
+       # loose detection of merges
+       # based on the commit msg
+       foreach my $rx (@mergerx) {
+               next unless $logmsg =~ $rx && $1;
+               my $mparent = $1 eq 'HEAD' ? $opt_o : $1;
+               if(my $sha1 = get_headref($mparent, $git_dir)) {
+                       push @commit_args, '-p', $mparent;
+                       print "Merge parent branch: $mparent\n" if $opt_v;
+               }
+       }
+
+       my $commit_date = strftime("+0000 %Y-%m-%d %H:%M:%S",gmtime($date));
+       $ENV{GIT_AUTHOR_NAME} = $author_name;
+       $ENV{GIT_AUTHOR_EMAIL} = $author_email;
+       $ENV{GIT_AUTHOR_DATE} = $commit_date;
+       $ENV{GIT_COMMITTER_NAME} = $author_name;
+       $ENV{GIT_COMMITTER_EMAIL} = $author_email;
+       $ENV{GIT_COMMITTER_DATE} = $commit_date;
+       my $pid = open2(my $commit_read, my $commit_write,
+               'git-commit-tree', $tree, @commit_args);
 
        # compatibility with git2cvs
        substr($logmsg,32767) = "" if length($logmsg) > 32767;
@@ -661,18 +668,17 @@ ($$)
        if (@skipped) {
            $logmsg .= "\n\n\nSKIPPED:\n\t";
            $logmsg .= join("\n\t", @skipped) . "\n";
+           @skipped = ();
        }
 
-       print $pw "$logmsg\n"
+       print($commit_write "$logmsg\n") && close($commit_write)
                or die "Error writing to git-commit-tree: $!\n";
-       $pw->close();
 
-       print "Committed patch $patchset ($branch ".strftime("%Y-%m-%d %H:%M:%S",gmtime($date)).")\n" if $opt_v;
-       chomp(my $cid = <$pr>);
-       length($cid) == 40
-               or die "Cannot get commit id ($cid): $!\n";
+       print "Committed patch $patchset ($branch $commit_date)\n" if $opt_v;
+       chomp(my $cid = <$commit_read>);
+       is_sha1($cid) or die "Cannot get commit id ($cid): $!\n";
        print "Commit ID $cid\n" if $opt_v;
-       $pr->close();
+       close($commit_read);
 
        waitpid($pid,0);
        die "Error running git-commit-tree: $?\n" if $?;
@@ -716,6 +722,7 @@ ($$)
        }
 };
 
+my $commitcount = 1;
 while(<CVS>) {
        chomp;
        if($state == 0 and /^-+$/) {
@@ -778,7 +785,16 @@ ($$)
                        $state = 11;
                        next;
                }
+               if (exists $ignorebranch{$branch}) {
+                       print STDERR "Skipping $branch\n";
+                       $state = 11;
+                       next;
+               }
                if($ancestor) {
+                       if($ancestor eq $branch) {
+                               print STDERR "Branch $branch erroneously stems from itself -- changed ancestor to $opt_o\n";
+                               $ancestor = $opt_o;
+                       }
                        if(-f "$git_dir/refs/heads/$branch") {
                                print STDERR "Branch $branch already exists!\n";
                                $state=11;
@@ -786,6 +802,7 @@ ($$)
                        }
                        unless(open(H,"$git_dir/refs/heads/$ancestor")) {
                                print STDERR "Branch $ancestor does not exist!\n";
+                               $ignorebranch{$branch} = 1;
                                $state=11;
                                next;
                        }
@@ -793,6 +810,7 @@ ($$)
                        close(H);
                        unless(open(H,"> $git_dir/refs/heads/$branch")) {
                                print STDERR "Could not create branch $branch: $!\n";
+                               $ignorebranch{$branch} = 1;
                                $state=11;
                                next;
                        }
@@ -801,11 +819,6 @@ ($$)
                        close(H)
                                or die "Could not write branch $branch: $!";
                }
-               if(($ancestor || $branch) ne $last_branch) {
-                       print "Switching from $last_branch to $branch\n" if $opt_v;
-                       system("git-read-tree", $branch);
-                       die "read-tree failed: $?\n" if $?;
-               }
                $last_branch = $branch if $branch ne $last_branch;
                $state = 9;
        } elsif($state == 8) {
@@ -849,7 +862,14 @@ ($$)
        } elsif($state == 9 and /^\s*$/) {
                $state = 10;
        } elsif(($state == 9 or $state == 10) and /^-+$/) {
-               &$commit();
+               $commitcount++;
+               if ($opt_L && $commitcount > $opt_L) {
+                       last;
+               }
+               commit();
+               if (($commitcount & 1023) == 0) {
+                       system("git repack -a -d");
+               }
                $state = 1;
        } elsif($state == 11 and /^-+$/) {
                $state = 1;
@@ -859,9 +879,13 @@ ($$)
                print "* UNKNOWN LINE * $_\n";
        }
 }
-&$commit() if $branch and $state != 11;
+commit() if $branch and $state != 11;
 
-unlink($git_index);
+foreach my $git_index (values %index) {
+    if ($git_index ne '.git/index') {
+       unlink($git_index);
+    }
+}
 
 if (defined $orig_git_index) {
        $ENV{GIT_INDEX_FILE} = $orig_git_index;