t / perf / aggregate.perlon commit Merge branch 'jh/trace2' (49bc8ce)
   1#!/usr/bin/perl
   2
   3use lib '../../perl/build/lib';
   4use strict;
   5use warnings;
   6use Getopt::Long;
   7use Git;
   8
   9sub get_times {
  10        my $name = shift;
  11        open my $fh, "<", $name or return undef;
  12        my $line = <$fh>;
  13        return undef if not defined $line;
  14        close $fh or die "cannot close $name: $!";
  15        # times
  16        if ($line =~ /^(?:(\d+):)?(\d+):(\d+(?:\.\d+)?) (\d+(?:\.\d+)?) (\d+(?:\.\d+)?)$/) {
  17                my $rt = ((defined $1 ? $1 : 0.0)*60+$2)*60+$3;
  18                return ($rt, $4, $5);
  19        # size
  20        } elsif ($line =~ /^\d+$/) {
  21                return $&;
  22        } else {
  23                die "bad input line: $line";
  24        }
  25}
  26
  27sub relative_change {
  28        my ($r, $firstr) = @_;
  29        if ($firstr > 0) {
  30                return sprintf "%+.1f%%", 100.0*($r-$firstr)/$firstr;
  31        } elsif ($r == 0) {
  32                return "=";
  33        } else {
  34                return "+inf";
  35        }
  36}
  37
  38sub format_times {
  39        my ($r, $u, $s, $firstr) = @_;
  40        # no value means we did not finish the test
  41        if (!defined $r) {
  42                return "<missing>";
  43        }
  44        # a single value means we have a size, not times
  45        if (!defined $u) {
  46                return format_size($r, $firstr);
  47        }
  48        # otherwise, we have real/user/system times
  49        my $out = sprintf "%.2f(%.2f+%.2f)", $r, $u, $s;
  50        $out .= ' ' . relative_change($r, $firstr) if defined $firstr;
  51        return $out;
  52}
  53
  54sub usage {
  55        print <<EOT;
  56./aggregate.perl [options] [--] [<dir_or_rev>...] [--] [<test_script>...] >
  57
  58  Options:
  59    --codespeed          * Format output for Codespeed
  60    --reponame    <str>  * Send given reponame to codespeed
  61    --sort-by     <str>  * Sort output (only "regression" criteria is supported)
  62    --subsection  <str>  * Use results from given subsection
  63
  64EOT
  65        exit(1);
  66}
  67
  68sub human_size {
  69        my $n = shift;
  70        my @units = ('', qw(K M G));
  71        while ($n > 900 && @units > 1) {
  72                $n /= 1000;
  73                shift @units;
  74        }
  75        return $n unless length $units[0];
  76        return sprintf '%.1f%s', $n, $units[0];
  77}
  78
  79sub format_size {
  80        my ($size, $first) = @_;
  81        # match the width of a time: 0.00(0.00+0.00)
  82        my $out = sprintf '%15s', human_size($size);
  83        $out .= ' ' . relative_change($size, $first) if defined $first;
  84        return $out;
  85}
  86
  87my (@dirs, %dirnames, %dirabbrevs, %prefixes, @tests,
  88    $codespeed, $sortby, $subsection, $reponame);
  89
  90Getopt::Long::Configure qw/ require_order /;
  91
  92my $rc = GetOptions("codespeed"     => \$codespeed,
  93                    "reponame=s"    => \$reponame,
  94                    "sort-by=s"     => \$sortby,
  95                    "subsection=s"  => \$subsection);
  96usage() unless $rc;
  97
  98while (scalar @ARGV) {
  99        my $arg = $ARGV[0];
 100        my $dir;
 101        last if -f $arg or $arg eq "--";
 102        if (! -d $arg) {
 103                my $rev = Git::command_oneline(qw(rev-parse --verify), $arg);
 104                $dir = "build/".$rev;
 105        } else {
 106                $arg =~ s{/*$}{};
 107                $dir = $arg;
 108                $dirabbrevs{$dir} = $dir;
 109        }
 110        push @dirs, $dir;
 111        $dirnames{$dir} = $arg;
 112        my $prefix = $dir;
 113        $prefix =~ tr/^a-zA-Z0-9/_/c;
 114        $prefixes{$dir} = $prefix . '.';
 115        shift @ARGV;
 116}
 117
 118if (not @dirs) {
 119        @dirs = ('.');
 120}
 121$dirnames{'.'} = $dirabbrevs{'.'} = "this tree";
 122$prefixes{'.'} = '';
 123
 124shift @ARGV if scalar @ARGV and $ARGV[0] eq "--";
 125
 126@tests = @ARGV;
 127if (not @tests) {
 128        @tests = glob "p????-*.sh";
 129}
 130
 131my $resultsdir = "test-results";
 132
 133if (! $subsection and
 134    exists $ENV{GIT_PERF_SUBSECTION} and
 135    $ENV{GIT_PERF_SUBSECTION} ne "") {
 136        $subsection = $ENV{GIT_PERF_SUBSECTION};
 137}
 138
 139if ($subsection) {
 140        $resultsdir .= "/" . $subsection;
 141}
 142
 143my @subtests;
 144my %shorttests;
 145for my $t (@tests) {
 146        $t =~ s{(?:.*/)?(p(\d+)-[^/]+)\.sh$}{$1} or die "bad test name: $t";
 147        my $n = $2;
 148        my $fname = "$resultsdir/$t.subtests";
 149        open my $fp, "<", $fname or die "cannot open $fname: $!";
 150        for (<$fp>) {
 151                chomp;
 152                /^(\d+)$/ or die "malformed subtest line: $_";
 153                push @subtests, "$t.$1";
 154                $shorttests{"$t.$1"} = "$n.$1";
 155        }
 156        close $fp or die "cannot close $fname: $!";
 157}
 158
 159sub read_descr {
 160        my $name = shift;
 161        open my $fh, "<", $name or return "<error reading description>";
 162        binmode $fh, ":utf8" or die "PANIC on binmode: $!";
 163        my $line = <$fh>;
 164        close $fh or die "cannot close $name";
 165        chomp $line;
 166        return $line;
 167}
 168
 169sub have_duplicate {
 170        my %seen;
 171        for (@_) {
 172                return 1 if exists $seen{$_};
 173                $seen{$_} = 1;
 174        }
 175        return 0;
 176}
 177sub have_slash {
 178        for (@_) {
 179                return 1 if m{/};
 180        }
 181        return 0;
 182}
 183
 184sub display_dir {
 185        my ($d) = @_;
 186        return exists $dirabbrevs{$d} ? $dirabbrevs{$d} : $dirnames{$d};
 187}
 188
 189sub print_default_results {
 190        my %descrs;
 191        my $descrlen = 4; # "Test"
 192        for my $t (@subtests) {
 193                $descrs{$t} = $shorttests{$t}.": ".read_descr("$resultsdir/$t.descr");
 194                $descrlen = length $descrs{$t} if length $descrs{$t}>$descrlen;
 195        }
 196
 197        my %newdirabbrevs = %dirabbrevs;
 198        while (!have_duplicate(values %newdirabbrevs)) {
 199                %dirabbrevs = %newdirabbrevs;
 200                last if !have_slash(values %dirabbrevs);
 201                %newdirabbrevs = %dirabbrevs;
 202                for (values %newdirabbrevs) {
 203                        s{^[^/]*/}{};
 204                }
 205        }
 206
 207        my %times;
 208        my @colwidth = ((0)x@dirs);
 209        for my $i (0..$#dirs) {
 210                my $w = length display_dir($dirs[$i]);
 211                $colwidth[$i] = $w if $w > $colwidth[$i];
 212        }
 213        for my $t (@subtests) {
 214                my $firstr;
 215                for my $i (0..$#dirs) {
 216                        my $d = $dirs[$i];
 217                        my $base = "$resultsdir/$prefixes{$d}$t";
 218                        $times{$prefixes{$d}.$t} = [];
 219                        foreach my $type (qw(times size)) {
 220                                if (-e "$base.$type") {
 221                                        $times{$prefixes{$d}.$t} = [get_times("$base.$type")];
 222                                        last;
 223                                }
 224                        }
 225                        my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
 226                        my $w = length format_times($r,$u,$s,$firstr);
 227                        $colwidth[$i] = $w if $w > $colwidth[$i];
 228                        $firstr = $r unless defined $firstr;
 229                }
 230        }
 231        my $totalwidth = 3*@dirs+$descrlen;
 232        $totalwidth += $_ for (@colwidth);
 233
 234        printf "%-${descrlen}s", "Test";
 235        for my $i (0..$#dirs) {
 236                printf "   %-$colwidth[$i]s", display_dir($dirs[$i]);
 237        }
 238        print "\n";
 239        print "-"x$totalwidth, "\n";
 240        for my $t (@subtests) {
 241                printf "%-${descrlen}s", $descrs{$t};
 242                my $firstr;
 243                for my $i (0..$#dirs) {
 244                        my $d = $dirs[$i];
 245                        my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
 246                        printf "   %-$colwidth[$i]s", format_times($r,$u,$s,$firstr);
 247                        $firstr = $r unless defined $firstr;
 248                }
 249                print "\n";
 250        }
 251}
 252
 253sub print_sorted_results {
 254        my ($sortby) = @_;
 255
 256        if ($sortby ne "regression") {
 257                print "Only 'regression' is supported as '--sort-by' argument\n";
 258                usage();
 259        }
 260
 261        my @evolutions;
 262        for my $t (@subtests) {
 263                my ($prevr, $prevu, $prevs, $prevrev);
 264                for my $i (0..$#dirs) {
 265                        my $d = $dirs[$i];
 266                        my ($r, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.times");
 267                        if ($i > 0 and defined $r and defined $prevr and $prevr > 0) {
 268                                my $percent = 100.0 * ($r - $prevr) / $prevr;
 269                                push @evolutions, { "percent"  => $percent,
 270                                                    "test"     => $t,
 271                                                    "prevrev"  => $prevrev,
 272                                                    "rev"      => $d,
 273                                                    "prevr"    => $prevr,
 274                                                    "r"        => $r,
 275                                                    "prevu"    => $prevu,
 276                                                    "u"        => $u,
 277                                                    "prevs"    => $prevs,
 278                                                    "s"        => $s};
 279                        }
 280                        ($prevr, $prevu, $prevs, $prevrev) = ($r, $u, $s, $d);
 281                }
 282        }
 283
 284        my @sorted_evolutions = sort { $b->{percent} <=> $a->{percent} } @evolutions;
 285
 286        for my $e (@sorted_evolutions) {
 287                printf "%+.1f%%", $e->{percent};
 288                print " " . $e->{test};
 289                print " " . format_times($e->{prevr}, $e->{prevu}, $e->{prevs});
 290                print " " . format_times($e->{r}, $e->{u}, $e->{s});
 291                print " " . display_dir($e->{prevrev});
 292                print " " . display_dir($e->{rev});
 293                print "\n";
 294        }
 295}
 296
 297sub print_codespeed_results {
 298        my ($subsection) = @_;
 299
 300        my $project = "Git";
 301
 302        my $executable = `uname -s -m`;
 303        chomp $executable;
 304
 305        if ($subsection) {
 306                $executable .= ", " . $subsection;
 307        }
 308
 309        my $environment;
 310        if ($reponame) {
 311                $environment = $reponame;
 312        } elsif (exists $ENV{GIT_PERF_REPO_NAME} and $ENV{GIT_PERF_REPO_NAME} ne "") {
 313                $environment = $ENV{GIT_PERF_REPO_NAME};
 314        } elsif (exists $ENV{GIT_TEST_INSTALLED} and $ENV{GIT_TEST_INSTALLED} ne "") {
 315                $environment = $ENV{GIT_TEST_INSTALLED};
 316                $environment =~ s|/bin-wrappers$||;
 317        } else {
 318                $environment = `uname -r`;
 319                chomp $environment;
 320        }
 321
 322        my @data;
 323
 324        for my $t (@subtests) {
 325                for my $d (@dirs) {
 326                        my $commitid = $prefixes{$d};
 327                        $commitid =~ s/^build_//;
 328                        $commitid =~ s/\.$//;
 329                        my ($result_value, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.times");
 330
 331                        my %vals = (
 332                                "commitid" => $commitid,
 333                                "project" => $project,
 334                                "branch" => $dirnames{$d},
 335                                "executable" => $executable,
 336                                "benchmark" => $shorttests{$t} . " " . read_descr("$resultsdir/$t.descr"),
 337                                "environment" => $environment,
 338                                "result_value" => $result_value,
 339                            );
 340                        push @data, \%vals;
 341                }
 342        }
 343
 344        require JSON;
 345        print JSON::to_json(\@data, {utf8 => 1, pretty => 1, canonical => 1}), "\n";
 346}
 347
 348binmode STDOUT, ":utf8" or die "PANIC on binmode: $!";
 349
 350if ($codespeed) {
 351        print_codespeed_results($subsection);
 352} elsif (defined $sortby) {
 353        print_sorted_results($sortby);
 354} else {
 355        print_default_results();
 356}