t / perf / aggregate.perlon commit Merge branch 'jk/trailers-use-config' (3a50607)
   1#!/usr/bin/perl
   2
   3use lib '../../perl/build/lib';
   4use strict;
   5use warnings;
   6use Getopt::Long;
   7use Git;
   8use Cwd qw(realpath);
   9
  10sub get_times {
  11        my $name = shift;
  12        open my $fh, "<", $name or return undef;
  13        my $line = <$fh>;
  14        return undef if not defined $line;
  15        close $fh or die "cannot close $name: $!";
  16        # times
  17        if ($line =~ /^(?:(\d+):)?(\d+):(\d+(?:\.\d+)?) (\d+(?:\.\d+)?) (\d+(?:\.\d+)?)$/) {
  18                my $rt = ((defined $1 ? $1 : 0.0)*60+$2)*60+$3;
  19                return ($rt, $4, $5);
  20        # size
  21        } elsif ($line =~ /^\d+$/) {
  22                return $&;
  23        } else {
  24                die "bad input line: $line";
  25        }
  26}
  27
  28sub relative_change {
  29        my ($r, $firstr) = @_;
  30        if ($firstr > 0) {
  31                return sprintf "%+.1f%%", 100.0*($r-$firstr)/$firstr;
  32        } elsif ($r == 0) {
  33                return "=";
  34        } else {
  35                return "+inf";
  36        }
  37}
  38
  39sub format_times {
  40        my ($r, $u, $s, $firstr) = @_;
  41        # no value means we did not finish the test
  42        if (!defined $r) {
  43                return "<missing>";
  44        }
  45        # a single value means we have a size, not times
  46        if (!defined $u) {
  47                return format_size($r, $firstr);
  48        }
  49        # otherwise, we have real/user/system times
  50        my $out = sprintf "%.2f(%.2f+%.2f)", $r, $u, $s;
  51        $out .= ' ' . relative_change($r, $firstr) if defined $firstr;
  52        return $out;
  53}
  54
  55sub usage {
  56        print <<EOT;
  57./aggregate.perl [options] [--] [<dir_or_rev>...] [--] [<test_script>...] >
  58
  59  Options:
  60    --codespeed          * Format output for Codespeed
  61    --reponame    <str>  * Send given reponame to codespeed
  62    --sort-by     <str>  * Sort output (only "regression" criteria is supported)
  63    --subsection  <str>  * Use results from given subsection
  64
  65EOT
  66        exit(1);
  67}
  68
  69sub human_size {
  70        my $n = shift;
  71        my @units = ('', qw(K M G));
  72        while ($n > 900 && @units > 1) {
  73                $n /= 1000;
  74                shift @units;
  75        }
  76        return $n unless length $units[0];
  77        return sprintf '%.1f%s', $n, $units[0];
  78}
  79
  80sub format_size {
  81        my ($size, $first) = @_;
  82        # match the width of a time: 0.00(0.00+0.00)
  83        my $out = sprintf '%15s', human_size($size);
  84        $out .= ' ' . relative_change($size, $first) if defined $first;
  85        return $out;
  86}
  87
  88my (@dirs, %dirnames, %dirabbrevs, %prefixes, @tests,
  89    $codespeed, $sortby, $subsection, $reponame);
  90
  91Getopt::Long::Configure qw/ require_order /;
  92
  93my $rc = GetOptions("codespeed"     => \$codespeed,
  94                    "reponame=s"    => \$reponame,
  95                    "sort-by=s"     => \$sortby,
  96                    "subsection=s"  => \$subsection);
  97usage() unless $rc;
  98
  99while (scalar @ARGV) {
 100        my $arg = $ARGV[0];
 101        my $dir;
 102        my $prefix = '';
 103        last if -f $arg or $arg eq "--";
 104        if (! -d $arg) {
 105                my $rev = Git::command_oneline(qw(rev-parse --verify), $arg);
 106                $dir = "build/".$rev;
 107        } elsif ($arg eq '.') {
 108                $dir = '.';
 109        } else {
 110                $dir = realpath($arg);
 111                $dirnames{$dir} = $dir;
 112                $prefix .= 'bindir';
 113        }
 114        push @dirs, $dir;
 115        $dirnames{$dir} ||= $arg;
 116        $prefix .= $dir;
 117        $prefix =~ tr/^a-zA-Z0-9/_/c;
 118        $prefixes{$dir} = $prefix . '.';
 119        shift @ARGV;
 120}
 121
 122if (not @dirs) {
 123        @dirs = ('.');
 124}
 125$dirnames{'.'} = $dirabbrevs{'.'} = "this tree";
 126$prefixes{'.'} = '';
 127
 128shift @ARGV if scalar @ARGV and $ARGV[0] eq "--";
 129
 130@tests = @ARGV;
 131if (not @tests) {
 132        @tests = glob "p????-*.sh";
 133}
 134
 135my $resultsdir = "test-results";
 136
 137if (! $subsection and
 138    exists $ENV{GIT_PERF_SUBSECTION} and
 139    $ENV{GIT_PERF_SUBSECTION} ne "") {
 140        $subsection = $ENV{GIT_PERF_SUBSECTION};
 141}
 142
 143if ($subsection) {
 144        $resultsdir .= "/" . $subsection;
 145}
 146
 147my @subtests;
 148my %shorttests;
 149for my $t (@tests) {
 150        $t =~ s{(?:.*/)?(p(\d+)-[^/]+)\.sh$}{$1} or die "bad test name: $t";
 151        my $n = $2;
 152        my $fname = "$resultsdir/$t.subtests";
 153        open my $fp, "<", $fname or die "cannot open $fname: $!";
 154        for (<$fp>) {
 155                chomp;
 156                /^(\d+)$/ or die "malformed subtest line: $_";
 157                push @subtests, "$t.$1";
 158                $shorttests{"$t.$1"} = "$n.$1";
 159        }
 160        close $fp or die "cannot close $fname: $!";
 161}
 162
 163sub read_descr {
 164        my $name = shift;
 165        open my $fh, "<", $name or return "<error reading description>";
 166        binmode $fh, ":utf8" or die "PANIC on binmode: $!";
 167        my $line = <$fh>;
 168        close $fh or die "cannot close $name";
 169        chomp $line;
 170        return $line;
 171}
 172
 173sub have_duplicate {
 174        my %seen;
 175        for (@_) {
 176                return 1 if exists $seen{$_};
 177                $seen{$_} = 1;
 178        }
 179        return 0;
 180}
 181sub have_slash {
 182        for (@_) {
 183                return 1 if m{/};
 184        }
 185        return 0;
 186}
 187
 188sub display_dir {
 189        my ($d) = @_;
 190        return exists $dirabbrevs{$d} ? $dirabbrevs{$d} : $dirnames{$d};
 191}
 192
 193sub print_default_results {
 194        my %descrs;
 195        my $descrlen = 4; # "Test"
 196        for my $t (@subtests) {
 197                $descrs{$t} = $shorttests{$t}.": ".read_descr("$resultsdir/$t.descr");
 198                $descrlen = length $descrs{$t} if length $descrs{$t}>$descrlen;
 199        }
 200
 201        my %newdirabbrevs = %dirabbrevs;
 202        while (!have_duplicate(values %newdirabbrevs)) {
 203                %dirabbrevs = %newdirabbrevs;
 204                last if !have_slash(values %dirabbrevs);
 205                %newdirabbrevs = %dirabbrevs;
 206                for (values %newdirabbrevs) {
 207                        s{^[^/]*/}{};
 208                }
 209        }
 210
 211        my %times;
 212        my @colwidth = ((0)x@dirs);
 213        for my $i (0..$#dirs) {
 214                my $w = length display_dir($dirs[$i]);
 215                $colwidth[$i] = $w if $w > $colwidth[$i];
 216        }
 217        for my $t (@subtests) {
 218                my $firstr;
 219                for my $i (0..$#dirs) {
 220                        my $d = $dirs[$i];
 221                        my $base = "$resultsdir/$prefixes{$d}$t";
 222                        $times{$prefixes{$d}.$t} = [];
 223                        foreach my $type (qw(times size)) {
 224                                if (-e "$base.$type") {
 225                                        $times{$prefixes{$d}.$t} = [get_times("$base.$type")];
 226                                        last;
 227                                }
 228                        }
 229                        my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
 230                        my $w = length format_times($r,$u,$s,$firstr);
 231                        $colwidth[$i] = $w if $w > $colwidth[$i];
 232                        $firstr = $r unless defined $firstr;
 233                }
 234        }
 235        my $totalwidth = 3*@dirs+$descrlen;
 236        $totalwidth += $_ for (@colwidth);
 237
 238        printf "%-${descrlen}s", "Test";
 239        for my $i (0..$#dirs) {
 240                printf "   %-$colwidth[$i]s", display_dir($dirs[$i]);
 241        }
 242        print "\n";
 243        print "-"x$totalwidth, "\n";
 244        for my $t (@subtests) {
 245                printf "%-${descrlen}s", $descrs{$t};
 246                my $firstr;
 247                for my $i (0..$#dirs) {
 248                        my $d = $dirs[$i];
 249                        my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
 250                        printf "   %-$colwidth[$i]s", format_times($r,$u,$s,$firstr);
 251                        $firstr = $r unless defined $firstr;
 252                }
 253                print "\n";
 254        }
 255}
 256
 257sub print_sorted_results {
 258        my ($sortby) = @_;
 259
 260        if ($sortby ne "regression") {
 261                print "Only 'regression' is supported as '--sort-by' argument\n";
 262                usage();
 263        }
 264
 265        my @evolutions;
 266        for my $t (@subtests) {
 267                my ($prevr, $prevu, $prevs, $prevrev);
 268                for my $i (0..$#dirs) {
 269                        my $d = $dirs[$i];
 270                        my ($r, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.times");
 271                        if ($i > 0 and defined $r and defined $prevr and $prevr > 0) {
 272                                my $percent = 100.0 * ($r - $prevr) / $prevr;
 273                                push @evolutions, { "percent"  => $percent,
 274                                                    "test"     => $t,
 275                                                    "prevrev"  => $prevrev,
 276                                                    "rev"      => $d,
 277                                                    "prevr"    => $prevr,
 278                                                    "r"        => $r,
 279                                                    "prevu"    => $prevu,
 280                                                    "u"        => $u,
 281                                                    "prevs"    => $prevs,
 282                                                    "s"        => $s};
 283                        }
 284                        ($prevr, $prevu, $prevs, $prevrev) = ($r, $u, $s, $d);
 285                }
 286        }
 287
 288        my @sorted_evolutions = sort { $b->{percent} <=> $a->{percent} } @evolutions;
 289
 290        for my $e (@sorted_evolutions) {
 291                printf "%+.1f%%", $e->{percent};
 292                print " " . $e->{test};
 293                print " " . format_times($e->{prevr}, $e->{prevu}, $e->{prevs});
 294                print " " . format_times($e->{r}, $e->{u}, $e->{s});
 295                print " " . display_dir($e->{prevrev});
 296                print " " . display_dir($e->{rev});
 297                print "\n";
 298        }
 299}
 300
 301sub print_codespeed_results {
 302        my ($subsection) = @_;
 303
 304        my $project = "Git";
 305
 306        my $executable = `uname -s -m`;
 307        chomp $executable;
 308
 309        if ($subsection) {
 310                $executable .= ", " . $subsection;
 311        }
 312
 313        my $environment;
 314        if ($reponame) {
 315                $environment = $reponame;
 316        } elsif (exists $ENV{GIT_PERF_REPO_NAME} and $ENV{GIT_PERF_REPO_NAME} ne "") {
 317                $environment = $ENV{GIT_PERF_REPO_NAME};
 318        } else {
 319                $environment = `uname -r`;
 320                chomp $environment;
 321        }
 322
 323        my @data;
 324
 325        for my $t (@subtests) {
 326                for my $d (@dirs) {
 327                        my $commitid = $prefixes{$d};
 328                        $commitid =~ s/^build_//;
 329                        $commitid =~ s/\.$//;
 330                        my ($result_value, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.times");
 331
 332                        my %vals = (
 333                                "commitid" => $commitid,
 334                                "project" => $project,
 335                                "branch" => $dirnames{$d},
 336                                "executable" => $executable,
 337                                "benchmark" => $shorttests{$t} . " " . read_descr("$resultsdir/$t.descr"),
 338                                "environment" => $environment,
 339                                "result_value" => $result_value,
 340                            );
 341                        push @data, \%vals;
 342                }
 343        }
 344
 345        require JSON;
 346        print JSON::to_json(\@data, {utf8 => 1, pretty => 1, canonical => 1}), "\n";
 347}
 348
 349binmode STDOUT, ":utf8" or die "PANIC on binmode: $!";
 350
 351if ($codespeed) {
 352        print_codespeed_results($subsection);
 353} elsif (defined $sortby) {
 354        print_sorted_results($sortby);
 355} else {
 356        print_default_results();
 357}