Performance data of sark vs. v2.0.8 with a chart generator

doc/threads/.gitignore

@@ -0,0 +1,2 @@
+*.html
+*.pdf

doc/threads/Makefile

@@ -13,3 +13,17 @@ all: $(foreach ch,$(CHAPTERS),$(addprefix $(ch),$(SUFFICES)))
 
 %-wordpress.html: %.html Makefile
 	sed -r 's#</p>#\n#g; s#<p>##g; s#<(/?)code>#<\1tt>#g; s#<pre><tt>#<code>#g; s#</tt></pre>#</code>#g; s#</?figure>##g; s#<figcaption>#<p style="text-align: center">#; s#</figcaption>#</p>#; ' $< > $@
+
+stats-%.csv: stats.csv stats-filter.pl
+	perl stats-filter.pl $< $* > $@
+
+STATS_VARIANTS := multi imex mulimex single
+stats-all: $(patsubst %,stats-%.csv,$(STATS_VARIANTS))
+
+stats-2d-%.pdf: stats.csv stats-filter-2d.pl
+	perl stats-filter-2d.pl $< $* $@
+
+stats-2d-%.png: stats-2d-%.pdf
+	gs -dBATCH -dNOPAUSE -sDEVICE=pngalpha -sOutputFile=$@ -r300 $<
+
+stats-all-2d: $(foreach suf,pdf png,$(patsubst %,stats-2d-%.$(suf),$(STATS_VARIANTS)))

doc/threads/stats-draw.gnuplot

@@ -0,0 +1,41 @@
+set datafile columnheaders
+set datafile separator ";"
+#set isosample 15
+set dgrid3d 8,8
+set logscale
+set view 80,15,1,1
+set autoscale xy
+#set pm3d
+
+set term pdfcairo size 20cm,15cm
+
+set xlabel "TOTAL ROUTES" offset 0,-1.5
+set xrange [10000:320000]
+set xtics offset 0,-0.5
+set xtics (10000,15000,30000,50000,100000,150000,300000)
+
+set ylabel "PEERS"
+#set yrange [10:320]
+#set ytics (10,15,30,50,100,150,300)
+set yrange [10:320]
+set ytics (10,15,30,50,100,150,300)
+
+set zrange [1:2000]
+set xyplane at 1
+
+set border 895
+
+#set grid ztics lt 20
+
+set output ARG1 . "-" . ARG4 . ".pdf"
+
+splot \
+  ARG1 . ".csv" \
+    using "TOTAL_ROUTES":"PEERS":ARG2."/".ARG4 \
+    with lines \
+    title ARG2."/".ARG4, \
+  "" \
+    using "TOTAL_ROUTES":"PEERS":ARG3."/".ARG4 \
+    with lines \
+    title ARG3."/".ARG4
+

doc/threads/stats-filter-2d.pl

@@ -0,0 +1,137 @@
+#!/usr/bin/perl
+
+use common::sense;
+use Data::Dump;
+use List::Util;
+
+my @GROUP_BY = qw/VERSION PEERS TOTAL_ROUTES/;
+my @VALUES = qw/TIMEDIF/;
+
+my ($FILE, $TYPE, $OUTPUT) = @ARGV;
+
+### Load data ###
+my %data;
+open F, "<", $FILE or die $!;
+my @header = split /;/, <F>;
+chomp @header;
+
+my $line = undef;
+while ($line = <F>)
+{
+  chomp $line;
+  $line =~ s/;;(.*);;/;;\1;/;
+  $line =~ s/v2\.0\.8-1[89][^;]+/bgp/;
+  $line =~ s/v2\.0\.8-[^;]+/sark/;
+  $line =~ s/master;/v2.0.8;/;
+  my %row;
+  @row{@header} = split /;/, $line;
+  push @{$data{join ";", @row{@GROUP_BY}}}, { %row } if $row{TYPE} eq $TYPE;
+}
+
+### Do statistics ###
+sub avg {
+  return List::Util::sum(@_) / @_;
+}
+
+sub stdev {
+  my $avg = shift;
+  return 0 if @_ <= 1;
+  return sqrt(List::Util::sum(map { ($avg - $_)**2 } @_) / (@_-1));
+}
+
+my %output;
+my %vers;
+my %peers;
+
+STATS:
+foreach my $k (keys %data)
+{
+  my %cols = map { my $vk = $_; $vk => [ map { $_->{$vk} } @{$data{$k}} ]; } @VALUES;
+
+  my %avg = map { $_ => avg(@{$cols{$_}})} @VALUES;
+  my %stdev = map { $_ => stdev($avg{$_}, @{$cols{$_}})} @VALUES;
+
+  foreach my $v (@VALUES) {
+    next if $stdev{$v} / $avg{$v} < 0.035;
+
+    for (my $i=0; $i<@{$cols{$v}}; $i++)
+    {
+      my $dif = $cols{$v}[$i] - $avg{$v};
+      next if $dif < $stdev{$v} * 2 and $dif > $stdev{$v} * (-2);
+=cut
+      printf "Removing an outlier for %s/%s: avg=%f, stdev=%f, variance=%.1f%%, val=%f, valratio=%.1f%%\n",
+	$k, $v, $avg{$v}, $stdev{$v}, (100 * $stdev{$v} / $avg{$v}), $cols{$v}[$i], (100 * $dif / $stdev{$v});
+=cut
+      splice @{$data{$k}}, $i, 1, ();
+      redo STATS;
+    }
+  }
+
+  $vers{$data{$k}[0]{VERSION}}++;
+  $peers{$data{$k}[0]{PEERS}}++;
+  $output{$data{$k}[0]{VERSION}}{$data{$k}[0]{PEERS}}{$data{$k}[0]{TOTAL_ROUTES}} = { %avg };
+}
+
+(3 == scalar %vers) and $vers{sark} and $vers{bgp} and $vers{"v2.0.8"} or die "vers size is " . (scalar %vers) . ", items ", join ", ", keys %vers;
+
+### Export the data ###
+
+open PLOT, "|-", "gnuplot" or die $!;
+
+say PLOT <<EOF;
+set logscale
+
+set term pdfcairo size 20cm,15cm
+
+set xlabel "Total number of routes" offset 0,-1.5
+set xrange [10000:1500000]
+set xtics offset 0,-0.5
+set xtics (10000,15000,30000,50000,100000,150000,300000,500000,1000000)
+
+set ylabel "Time to converge (s)"
+set yrange [0.5:10800]
+
+set grid
+
+set key right bottom
+
+set output "$OUTPUT"
+EOF
+
+my @colors = (
+  [ 1, 0.3, 0.3 ],
+  [ 1, 0.7, 0 ],
+  [ 0.3, 1, 0 ],
+  [ 0, 1, 0.3 ],
+  [ 0, 0.7, 1 ],
+  [ 0.3, 0.3, 1 ],
+);
+
+my $steps = (scalar %peers) - 1;
+
+my @plot_data;
+foreach my $v (sort keys %vers) {
+  my $color = shift @colors;
+  my $endcolor = shift @colors;
+  my $stepcolor = [ map +( ($endcolor->[$_] - $color->[$_]) / $steps ), (0, 1, 2) ];
+
+  foreach my $p (sort { int $a <=> int $b } keys %peers) {
+    my $vnodot = $v; $vnodot =~ s/\.//g;
+    say PLOT "\$data_${vnodot}_${p} << EOD";
+    foreach my $tr (sort { int $a <=> int $b } keys %{$output{$v}{$p}}) {
+      say PLOT "$tr $output{$v}{$p}{$tr}{TIMEDIF}";
+    }
+    say PLOT "EOD";
+
+    my $colorstr = sprintf "linecolor rgbcolor \"#%02x%02x%02x\"", map +( int($color->[$_] * 255 + 0.5)), (0, 1, 2);
+    push @plot_data, "\$data_${vnodot}_${p} using 1:2 with lines $colorstr linewidth 2 title \"$v, $p peers\"";
+    $color = [ map +( $color->[$_] + $stepcolor->[$_] ), (0, 1, 2) ];
+  }
+}
+
+push @plot_data, "2 with lines lt 1 dashtype 2 title \"Measurement instability\"";
+
+say PLOT "plot ", join ", ", @plot_data;
+close PLOT;
+
+

doc/threads/stats-filter.pl

@@ -0,0 +1,84 @@
+#!/usr/bin/perl
+
+use common::sense;
+use Data::Dump;
+use List::Util;
+
+my @GROUP_BY = qw/VERSION PEERS TOTAL_ROUTES/;
+my @VALUES = qw/RSS SZ VSZ TIMEDIF/;
+
+my ($FILE, $TYPE) = @ARGV;
+
+### Load data ###
+my %data;
+open F, "<", $FILE or die $!;
+my @header = split /;/, <F>;
+chomp @header;
+
+my $line = undef;
+while ($line = <F>)
+{
+  chomp $line;
+  my %row;
+  @row{@header} = split /;/, $line;
+  push @{$data{join ";", @row{@GROUP_BY}}}, { %row } if $row{TYPE} eq $TYPE;
+}
+
+### Do statistics ###
+sub avg {
+  return List::Util::sum(@_) / @_;
+}
+
+sub stdev {
+  my $avg = shift;
+  return 0 if @_ <= 1;
+  return sqrt(List::Util::sum(map { ($avg - $_)**2 } @_) / (@_-1));
+}
+
+my %output;
+my %vers;
+
+STATS:
+foreach my $k (keys %data)
+{
+  my %cols = map { my $vk = $_; $vk => [ map { $_->{$vk} } @{$data{$k}} ]; } @VALUES;
+
+  my %avg = map { $_ => avg(@{$cols{$_}})} @VALUES;
+  my %stdev = map { $_ => stdev($avg{$_}, @{$cols{$_}})} @VALUES;
+
+  foreach my $v (@VALUES) {
+    next if $stdev{$v} / $avg{$v} < 0.035;
+
+    for (my $i=0; $i<@{$cols{$v}}; $i++)
+    {
+      my $dif = $cols{$v}[$i] - $avg{$v};
+      next if $dif < $stdev{$v} * 2 and $dif > $stdev{$v} * (-2);
+=cut
+      printf "Removing an outlier for %s/%s: avg=%f, stdev=%f, variance=%.1f%%, val=%f, valratio=%.1f%%\n",
+	$k, $v, $avg{$v}, $stdev{$v}, (100 * $stdev{$v} / $avg{$v}), $cols{$v}[$i], (100 * $dif / $stdev{$v});
+=cut
+      splice @{$data{$k}}, $i, 1, ();
+      redo STATS;
+    }
+  }
+
+  $vers{$data{$k}[0]{VERSION}}++;
+  $output{"$data{$k}[0]{PEERS};$data{$k}[0]{TOTAL_ROUTES}"}{$data{$k}[0]{VERSION}} = { %avg };
+}
+
+### Export the data ###
+
+say "PEERS;TOTAL_ROUTES;" . join ";", ( map { my $vk = $_; map { "$_/$vk" } keys %vers; } @VALUES );
+
+sub keysort {
+  my ($pa, $ta) = split /;/, $_[0];
+  my ($pb, $tb) = split /;/, $_[1];
+
+  return (int $ta) <=> (int $tb) if $pa eq $pb;
+  return (int $pa) <=> (int $pb);
+}
+  
+foreach my $k (sort { keysort($a, $b); } keys %output)
+{
+  say "$k;" . join ";", ( map { my $vk = $_; map { $output{$k}{$_}{$vk}; } keys %vers; } @VALUES );
+}