cvsimport: introduce _fetchfile() method and used a 1M buffer to read()

File retrieval from the socket is now moved to _fetchfile() and we now
cap reads at 1MB. This should limit the memory growth of the cvsimport
process.

Signed-off-by: Martin Langhoff <martin@catalyst.net.nz>
Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/git-cvsimport.perl b/git-cvsimport.perl
index f0e4d2422bd4e814aee6b7d77394acd1f306967e..41ee9a608d762718c33594374dfa4cb3205dd4f9 100755 (executable)
--- a/git-cvsimport.perl
+++ b/git-cvsimport.perl
@@ -315,15 +315,7 @@ sub _line {
                        chomp $cnt;
                        die "Duh: Filesize $cnt" if $cnt !~ /^\d+$/;
                        $line="";
                        chomp $cnt;
                        die "Duh: Filesize $cnt" if $cnt !~ /^\d+$/;
                        $line="";

-                       $res=0;
-                       while($cnt) {
-                               my $buf;
-                               my $num = $self->{'socketi'}->read($buf,$cnt);
-                               die "Server: Filesize $cnt: $num: $!\n" if not defined $num or $num<=0;
-                               print $fh $buf;
-                               $res += $num;
-                               $cnt -= $num;
-                       }
+                       $res = $self->_fetchfile($fh, $cnt);

                } elsif($line =~ s/^ //) {
                        print $fh $line;
                        $res += length($line);
                } elsif($line =~ s/^ //) {
                        print $fh $line;
                        $res += length($line);


@@ -335,14 +327,7 @@ sub _line {
                        chomp $cnt;
                        die "Duh: Mbinary $cnt" if $cnt !~ /^\d+$/ or $cnt<1;
                        $line="";
                        chomp $cnt;
                        die "Duh: Mbinary $cnt" if $cnt !~ /^\d+$/ or $cnt<1;
                        $line="";

-                       while($cnt) {
-                               my $buf;
-                               my $num = $self->{'socketi'}->read($buf,$cnt);
-                               die "S: Mbinary $cnt: $num: $!\n" if not defined $num or $num<=0;
-                               print $fh $buf;
-                               $res += $num;
-                               $cnt -= $num;
-                       }
+                       $res += $self->_fetchfile($fh, $cnt);

                } else {
                        chomp $line;
                        if($line eq "ok") {
                } else {
                        chomp $line;
                        if($line eq "ok") {


@@ -384,6 +369,23 @@ sub file {
 
        return ($name, $res);
 }
 
        return ($name, $res);
 }

+sub _fetchfile {
+       my ($self, $fh, $cnt) = @_;
+       my $res;
+       my $bufsize = 1024 * 1024;
+       while($cnt) {
+           if ($bufsize > $cnt) {
+               $bufsize = $cnt;
+           }
+           my $buf;
+           my $num = $self->{'socketi'}->read($buf,$bufsize);
+           die "Server: Filesize $cnt: $num: $!\n" if not defined $num or $num<=0;
+           print $fh $buf;
+           $res += $num;
+           $cnt -= $num;
+       }
+       return $res;
+}

 
 
 package main;
 
 
 package main;