git-svn: reload RA every log-window-size

Despite attempting to use local memory pools everywhere we can,
(including our call to SVN::Ra::do_update and all subsequent reporter
calls), there does not appear to be a way to force the Git::SVN::Fetcher
callbacks to use a pool other than the per-SVN::Ra pool.
Git::SVN::Fetcher ends up using the main RA pool which grows
monotonically in size for the lifetime of the RA object.

Thus the only way to free that memory appears to be to destroy and
recreate the RA connection for at every --log-window-size interval.

This reduces memory usage over the course of fetching 10K revisions
using a test repository created with the script at the end of this
commit message.

As reported by time(1) on my x86-64 system:

before: 54024k
after: 28680k

Unfortunately, there remains some yet-to-be-tracked-down slow memory
growth which would be evident as the `nr' parameter increases in
the repository generation script:
-----------------------------8<------------------------------
set -e
tmp=$(mktemp -d svntestrepo-XXXXXXXX)
svnadmin create "$tmp"
repo=file://"$(cd $tmp && pwd)"
svn co "$repo" "$tmp/wd"
cd "$tmp/wd"
if ! test -f a
then
> a
svn add a
svn commit -m 'A'
fi

nr=10000
while test $nr -gt 0
do
echo $nr > a
svn commit -q -m A
nr=$((nr - 1))
done
echo "repository created in $repo"
-----------------------------8<------------------------------

Signed-off-by: Eric Wong <normalperson@yhbt.net>

diff --git a/perl/Git/SVN/Ra.pm b/perl/Git/SVN/Ra.pm
index a7b0119ee5945e062aee27a8bb21bf7c818fb8c6..1828519b4a02c6f5b6eb1974c60cfc2ef2c0d0f7 100644 (file)
--- a/perl/Git/SVN/Ra.pm
+++ b/perl/Git/SVN/Ra.pm
@@ -376,10 +376,19 @@ sub longest_common_path {
 sub gs_fetch_loop_common {
        my ($self, $base, $head, $gsv, $globs) = @_;
        return if ($base > $head);
 sub gs_fetch_loop_common {
        my ($self, $base, $head, $gsv, $globs) = @_;
        return if ($base > $head);

+       my $gpool = SVN::Pool->new_default;
+       my $ra_url = $self->url;
+       my $reload_ra = sub {
+               $_[0] = undef;
+               $self = undef;
+               $RA = undef;
+               $gpool->clear;
+               $self = Git::SVN::Ra->new($ra_url);
+               $ra_invalid = undef;
+       };

        my $inc = $_log_window_size;
        my ($min, $max) = ($base, $head < $base + $inc ? $head : $base + $inc);
        my $longest_path = longest_common_path($gsv, $globs);
        my $inc = $_log_window_size;
        my ($min, $max) = ($base, $head < $base + $inc ? $head : $base + $inc);
        my $longest_path = longest_common_path($gsv, $globs);

-       my $ra_url = $self->url;

        my $find_trailing_edge;
        while (1) {
                my %revs;
        my $find_trailing_edge;
        while (1) {
                my %revs;


@@ -449,13 +458,7 @@ sub gs_fetch_loop_common {
                                        "$g->{t}-maxRev";
                                Git::SVN::tmp_config($k, $r);
                        }
                                        "$g->{t}-maxRev";
                                Git::SVN::tmp_config($k, $r);
                        }

-                       if ($ra_invalid) {
-                               $_[0] = undef;
-                               $self = undef;
-                               $RA = undef;
-                               $self = Git::SVN::Ra->new($ra_url);
-                               $ra_invalid = undef;
-                       }
+                       $reload_ra->() if $ra_invalid;

                }
                # pre-fill the .rev_db since it'll eventually get filled in
                # with '0' x40 if something new gets committed
                }
                # pre-fill the .rev_db since it'll eventually get filled in
                # with '0' x40 if something new gets committed


@@ -472,6 +475,8 @@ sub gs_fetch_loop_common {
                $min = $max + 1;
                $max += $inc;
                $max = $head if ($max > $head);
                $min = $max + 1;
                $max += $inc;
                $max = $head if ($max > $head);

+
+               $reload_ra->();

        }
        Git::SVN::gc();
 }
        }
        Git::SVN::gc();
 }