gitweb: speed up project listing on large work trees by limiting find depth

Signed-off-by: Luke Lu <git@vicaya.com>
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

diff --git a/Makefile b/Makefile
index d74ac93c116fe24d1a23df1c7c21d9ca0cc1c23f..9f81c73af7d58ec75c6ca448ff6f96a3ba97c48d 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -165,6 +165,7 @@ GITWEB_CONFIG = gitweb_config.perl
 GITWEB_HOME_LINK_STR = projects
 GITWEB_SITENAME =
 GITWEB_PROJECTROOT = /pub/git
 GITWEB_HOME_LINK_STR = projects
 GITWEB_SITENAME =
 GITWEB_PROJECTROOT = /pub/git

+GITWEB_PROJECT_MAXDEPTH = 2007

 GITWEB_EXPORT_OK =
 GITWEB_STRICT_EXPORT =
 GITWEB_BASE_URL =
 GITWEB_EXPORT_OK =
 GITWEB_STRICT_EXPORT =
 GITWEB_BASE_URL =


@@ -831,6 +832,7 @@ gitweb/gitweb.cgi: gitweb/gitweb.perl
            -e 's|++GITWEB_HOME_LINK_STR++|$(GITWEB_HOME_LINK_STR)|g' \
            -e 's|++GITWEB_SITENAME++|$(GITWEB_SITENAME)|g' \
            -e 's|++GITWEB_PROJECTROOT++|$(GITWEB_PROJECTROOT)|g' \
            -e 's|++GITWEB_HOME_LINK_STR++|$(GITWEB_HOME_LINK_STR)|g' \
            -e 's|++GITWEB_SITENAME++|$(GITWEB_SITENAME)|g' \
            -e 's|++GITWEB_PROJECTROOT++|$(GITWEB_PROJECTROOT)|g' \

+           -e 's|"++GITWEB_PROJECT_MAXDEPTH++"|$(GITWEB_PROJECT_MAXDEPTH)|g' \

            -e 's|++GITWEB_EXPORT_OK++|$(GITWEB_EXPORT_OK)|g' \
            -e 's|++GITWEB_STRICT_EXPORT++|$(GITWEB_STRICT_EXPORT)|g' \
            -e 's|++GITWEB_BASE_URL++|$(GITWEB_BASE_URL)|g' \
            -e 's|++GITWEB_EXPORT_OK++|$(GITWEB_EXPORT_OK)|g' \
            -e 's|++GITWEB_STRICT_EXPORT++|$(GITWEB_STRICT_EXPORT)|g' \
            -e 's|++GITWEB_BASE_URL++|$(GITWEB_BASE_URL)|g' \

diff --git a/gitweb/gitweb.perl b/gitweb/gitweb.perl
index 3064298f28837ced72897b6bc6401b3061882926..48e21dad6cec580cd874e32e981a86dc909893f7 100755 (executable)
--- a/gitweb/gitweb.perl
+++ b/gitweb/gitweb.perl
@@ -35,6 +35,10 @@ BEGIN
 #our $projectroot = "/pub/scm";
 our $projectroot = "++GITWEB_PROJECTROOT++";
 
 #our $projectroot = "/pub/scm";
 our $projectroot = "++GITWEB_PROJECTROOT++";
 

+# fs traversing limit for getting project list
+# the number is relative to the projectroot
+our $project_maxdepth = "++GITWEB_PROJECT_MAXDEPTH++";
+

 # target of the home link on top of all pages
 our $home_link = $my_uri || "/";
 
 # target of the home link on top of all pages
 our $home_link = $my_uri || "/";
 


@@ -1509,6 +1513,7 @@ sub git_get_projects_list {
                # remove the trailing "/"
                $dir =~ s!/+$!!;
                my $pfxlen = length("$dir");
                # remove the trailing "/"
                $dir =~ s!/+$!!;
                my $pfxlen = length("$dir");

+               my $pfxdepth = ($dir =~ tr!/!!);

 
                File::Find::find({
                        follow_fast => 1, # follow symbolic links
 
                File::Find::find({
                        follow_fast => 1, # follow symbolic links


@@ -1519,6 +1524,11 @@ sub git_get_projects_list {
                                return if (m!^[/.]$!);
                                # only directories can be git repositories
                                return unless (-d $_);
                                return if (m!^[/.]$!);
                                # only directories can be git repositories
                                return unless (-d $_);

+                               # don't traverse too deep (Find is super slow on os x)
+                               if (($File::Find::name =~ tr!/!!) - $pfxdepth > $project_maxdepth) {
+                                       $File::Find::prune = 1;
+                                       return;
+                               }

 
                                my $subdir = substr($File::Find::name, $pfxlen + 1);
                                # we check related file in $projectroot
 
                                my $subdir = substr($File::Find::name, $pfxlen + 1);
                                # we check related file in $projectroot

diff --git a/t/t9500-gitweb-standalone-no-errors.sh b/t/t9500-gitweb-standalone-no-errors.sh
index 642b836d64f2260aa0f21618e5af7f2a00cf97ec..f7bad5bb2f20cf274eb30f8ceed34a3f83000989 100755 (executable)
--- a/t/t9500-gitweb-standalone-no-errors.sh
+++ b/t/t9500-gitweb-standalone-no-errors.sh
@@ -18,6 +18,7 @@ gitweb_init () {
 our \$version = "current";
 our \$GIT = "git";
 our \$projectroot = "$(pwd)";
 our \$version = "current";
 our \$GIT = "git";
 our \$projectroot = "$(pwd)";

+our \$project_maxdepth = 8;

 our \$home_link_str = "projects";
 our \$site_name = "[localhost]";
 our \$site_header = "";
 our \$home_link_str = "projects";
 our \$site_name = "[localhost]";
 our \$site_header = "";