c8f4a78903086f0eb343ec82ef584b6e5ac04ccd
1#!/usr/bin/perl
2
3use lib '../../perl/build/lib';
4use strict;
5use warnings;
6use JSON;
7use Getopt::Long;
8use Git;
9use Cwd qw(realpath);
10
11sub get_times {
12 my $name = shift;
13 open my $fh, "<", $name or return undef;
14 my $line = <$fh>;
15 return undef if not defined $line;
16 close $fh or die "cannot close $name: $!";
17 # times
18 if ($line =~ /^(?:(\d+):)?(\d+):(\d+(?:\.\d+)?) (\d+(?:\.\d+)?) (\d+(?:\.\d+)?)$/) {
19 my $rt = ((defined $1 ? $1 : 0.0)*60+$2)*60+$3;
20 return ($rt, $4, $5);
21 # size
22 } elsif ($line =~ /^\d+$/) {
23 return $&;
24 } else {
25 die "bad input line: $line";
26 }
27}
28
29sub relative_change {
30 my ($r, $firstr) = @_;
31 if ($firstr > 0) {
32 return sprintf "%+.1f%%", 100.0*($r-$firstr)/$firstr;
33 } elsif ($r == 0) {
34 return "=";
35 } else {
36 return "+inf";
37 }
38}
39
40sub format_times {
41 my ($r, $u, $s, $firstr) = @_;
42 # no value means we did not finish the test
43 if (!defined $r) {
44 return "<missing>";
45 }
46 # a single value means we have a size, not times
47 if (!defined $u) {
48 return format_size($r, $firstr);
49 }
50 # otherwise, we have real/user/system times
51 my $out = sprintf "%.2f(%.2f+%.2f)", $r, $u, $s;
52 $out .= ' ' . relative_change($r, $firstr) if defined $firstr;
53 return $out;
54}
55
56sub usage {
57 print <<EOT;
58./aggregate.perl [options] [--] [<dir_or_rev>...] [--] [<test_script>...] >
59
60 Options:
61 --codespeed * Format output for Codespeed
62 --reponame <str> * Send given reponame to codespeed
63 --sort-by <str> * Sort output (only "regression" criteria is supported)
64 --subsection <str> * Use results from given subsection
65
66EOT
67 exit(1);
68}
69
70sub human_size {
71 my $n = shift;
72 my @units = ('', qw(K M G));
73 while ($n > 900 && @units > 1) {
74 $n /= 1000;
75 shift @units;
76 }
77 return $n unless length $units[0];
78 return sprintf '%.1f%s', $n, $units[0];
79}
80
81sub format_size {
82 my ($size, $first) = @_;
83 # match the width of a time: 0.00(0.00+0.00)
84 my $out = sprintf '%15s', human_size($size);
85 $out .= ' ' . relative_change($size, $first) if defined $first;
86 return $out;
87}
88
89my (@dirs, %dirnames, %dirabbrevs, %prefixes, @tests,
90 $codespeed, $sortby, $subsection, $reponame);
91
92Getopt::Long::Configure qw/ require_order /;
93
94my $rc = GetOptions("codespeed" => \$codespeed,
95 "reponame=s" => \$reponame,
96 "sort-by=s" => \$sortby,
97 "subsection=s" => \$subsection);
98usage() unless $rc;
99
100while (scalar @ARGV) {
101 my $arg = $ARGV[0];
102 my $dir;
103 last if -f $arg or $arg eq "--";
104 if (! -d $arg) {
105 my $rev = Git::command_oneline(qw(rev-parse --verify), $arg);
106 $dir = "build/".$rev;
107 } elsif ($arg eq '.') {
108 $dir = '.';
109 } else {
110 $dir = realpath($arg);
111 $dirnames{$dir} = $dir;
112 }
113 push @dirs, $dir;
114 $dirnames{$dir} ||= $arg;
115 my $prefix = $dir;
116 $prefix =~ tr/^a-zA-Z0-9/_/c;
117 $prefixes{$dir} = $prefix . '.';
118 shift @ARGV;
119}
120
121if (not @dirs) {
122 @dirs = ('.');
123}
124$dirnames{'.'} = $dirabbrevs{'.'} = "this tree";
125$prefixes{'.'} = '';
126
127shift @ARGV if scalar @ARGV and $ARGV[0] eq "--";
128
129@tests = @ARGV;
130if (not @tests) {
131 @tests = glob "p????-*.sh";
132}
133
134my $resultsdir = "test-results";
135
136if (! $subsection and
137 exists $ENV{GIT_PERF_SUBSECTION} and
138 $ENV{GIT_PERF_SUBSECTION} ne "") {
139 $subsection = $ENV{GIT_PERF_SUBSECTION};
140}
141
142if ($subsection) {
143 $resultsdir .= "/" . $subsection;
144}
145
146my @subtests;
147my %shorttests;
148for my $t (@tests) {
149 $t =~ s{(?:.*/)?(p(\d+)-[^/]+)\.sh$}{$1} or die "bad test name: $t";
150 my $n = $2;
151 my $fname = "$resultsdir/$t.subtests";
152 open my $fp, "<", $fname or die "cannot open $fname: $!";
153 for (<$fp>) {
154 chomp;
155 /^(\d+)$/ or die "malformed subtest line: $_";
156 push @subtests, "$t.$1";
157 $shorttests{"$t.$1"} = "$n.$1";
158 }
159 close $fp or die "cannot close $fname: $!";
160}
161
162sub read_descr {
163 my $name = shift;
164 open my $fh, "<", $name or return "<error reading description>";
165 binmode $fh, ":utf8" or die "PANIC on binmode: $!";
166 my $line = <$fh>;
167 close $fh or die "cannot close $name";
168 chomp $line;
169 return $line;
170}
171
172sub have_duplicate {
173 my %seen;
174 for (@_) {
175 return 1 if exists $seen{$_};
176 $seen{$_} = 1;
177 }
178 return 0;
179}
180sub have_slash {
181 for (@_) {
182 return 1 if m{/};
183 }
184 return 0;
185}
186
187sub display_dir {
188 my ($d) = @_;
189 return exists $dirabbrevs{$d} ? $dirabbrevs{$d} : $dirnames{$d};
190}
191
192sub print_default_results {
193 my %descrs;
194 my $descrlen = 4; # "Test"
195 for my $t (@subtests) {
196 $descrs{$t} = $shorttests{$t}.": ".read_descr("$resultsdir/$t.descr");
197 $descrlen = length $descrs{$t} if length $descrs{$t}>$descrlen;
198 }
199
200 my %newdirabbrevs = %dirabbrevs;
201 while (!have_duplicate(values %newdirabbrevs)) {
202 %dirabbrevs = %newdirabbrevs;
203 last if !have_slash(values %dirabbrevs);
204 %newdirabbrevs = %dirabbrevs;
205 for (values %newdirabbrevs) {
206 s{^[^/]*/}{};
207 }
208 }
209
210 my %times;
211 my @colwidth = ((0)x@dirs);
212 for my $i (0..$#dirs) {
213 my $w = length display_dir($dirs[$i]);
214 $colwidth[$i] = $w if $w > $colwidth[$i];
215 }
216 for my $t (@subtests) {
217 my $firstr;
218 for my $i (0..$#dirs) {
219 my $d = $dirs[$i];
220 my $base = "$resultsdir/$prefixes{$d}$t";
221 $times{$prefixes{$d}.$t} = [];
222 foreach my $type (qw(times size)) {
223 if (-e "$base.$type") {
224 $times{$prefixes{$d}.$t} = [get_times("$base.$type")];
225 last;
226 }
227 }
228 my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
229 my $w = length format_times($r,$u,$s,$firstr);
230 $colwidth[$i] = $w if $w > $colwidth[$i];
231 $firstr = $r unless defined $firstr;
232 }
233 }
234 my $totalwidth = 3*@dirs+$descrlen;
235 $totalwidth += $_ for (@colwidth);
236
237 printf "%-${descrlen}s", "Test";
238 for my $i (0..$#dirs) {
239 printf " %-$colwidth[$i]s", display_dir($dirs[$i]);
240 }
241 print "\n";
242 print "-"x$totalwidth, "\n";
243 for my $t (@subtests) {
244 printf "%-${descrlen}s", $descrs{$t};
245 my $firstr;
246 for my $i (0..$#dirs) {
247 my $d = $dirs[$i];
248 my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
249 printf " %-$colwidth[$i]s", format_times($r,$u,$s,$firstr);
250 $firstr = $r unless defined $firstr;
251 }
252 print "\n";
253 }
254}
255
256sub print_sorted_results {
257 my ($sortby) = @_;
258
259 if ($sortby ne "regression") {
260 print "Only 'regression' is supported as '--sort-by' argument\n";
261 usage();
262 }
263
264 my @evolutions;
265 for my $t (@subtests) {
266 my ($prevr, $prevu, $prevs, $prevrev);
267 for my $i (0..$#dirs) {
268 my $d = $dirs[$i];
269 my ($r, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.times");
270 if ($i > 0 and defined $r and defined $prevr and $prevr > 0) {
271 my $percent = 100.0 * ($r - $prevr) / $prevr;
272 push @evolutions, { "percent" => $percent,
273 "test" => $t,
274 "prevrev" => $prevrev,
275 "rev" => $d,
276 "prevr" => $prevr,
277 "r" => $r,
278 "prevu" => $prevu,
279 "u" => $u,
280 "prevs" => $prevs,
281 "s" => $s};
282 }
283 ($prevr, $prevu, $prevs, $prevrev) = ($r, $u, $s, $d);
284 }
285 }
286
287 my @sorted_evolutions = sort { $b->{percent} <=> $a->{percent} } @evolutions;
288
289 for my $e (@sorted_evolutions) {
290 printf "%+.1f%%", $e->{percent};
291 print " " . $e->{test};
292 print " " . format_times($e->{prevr}, $e->{prevu}, $e->{prevs});
293 print " " . format_times($e->{r}, $e->{u}, $e->{s});
294 print " " . display_dir($e->{prevrev});
295 print " " . display_dir($e->{rev});
296 print "\n";
297 }
298}
299
300sub print_codespeed_results {
301 my ($subsection) = @_;
302
303 my $project = "Git";
304
305 my $executable = `uname -s -m`;
306 chomp $executable;
307
308 if ($subsection) {
309 $executable .= ", " . $subsection;
310 }
311
312 my $environment;
313 if ($reponame) {
314 $environment = $reponame;
315 } elsif (exists $ENV{GIT_PERF_REPO_NAME} and $ENV{GIT_PERF_REPO_NAME} ne "") {
316 $environment = $ENV{GIT_PERF_REPO_NAME};
317 } else {
318 $environment = `uname -r`;
319 chomp $environment;
320 }
321
322 my @data;
323
324 for my $t (@subtests) {
325 for my $d (@dirs) {
326 my $commitid = $prefixes{$d};
327 $commitid =~ s/^build_//;
328 $commitid =~ s/\.$//;
329 my ($result_value, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.times");
330
331 my %vals = (
332 "commitid" => $commitid,
333 "project" => $project,
334 "branch" => $dirnames{$d},
335 "executable" => $executable,
336 "benchmark" => $shorttests{$t} . " " . read_descr("$resultsdir/$t.descr"),
337 "environment" => $environment,
338 "result_value" => $result_value,
339 );
340 push @data, \%vals;
341 }
342 }
343
344 print to_json(\@data, {utf8 => 1, pretty => 1, canonical => 1}), "\n";
345}
346
347binmode STDOUT, ":utf8" or die "PANIC on binmode: $!";
348
349if ($codespeed) {
350 print_codespeed_results($subsection);
351} elsif (defined $sortby) {
352 print_sorted_results($sortby);
353} else {
354 print_default_results();
355}