Added various graphing and filtering utilities.

diff --git a/post/graph-hist-flowcount b/post/graph-hist-flowcount
new file mode 100755 (executable)
index 0000000..b6544e3
--- /dev/null
+++ b/post/graph-hist-flowcount
@@ -0,0 +1,59 @@
+#!/usr/bin/perl
+# Draw graphs of TCP flow statistics from netgrind histograms
+# (c) 2003 Martin Mares <mj@ucw.cz>, GPL'ed
+
+use strict;
+use warnings;
+use POSIX;
+
+my $datafile = "gp.tmp";
+open D, ">$datafile" or die;
+
+my @colnames = ();
+my $total = 0;
+my @ewma = ( 0 );
+my $ew = 0.9;
+my $eww = 1-$ew;
+my $w = 60;
+while (<>) {
+       chomp;
+       if (/^#/) {
+               s/^#\s+//;
+               @colnames = split /\t/;
+       } else {
+               my $i = 0;
+               s/\([^)]*\)//g;
+               my %row = map { $colnames[$i++] => $_ } split /\t/;
+               my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'});
+               $total += $row{'FlowsTotal'};
+               $total -= $row{'FlowsClosed'} + $row{'FlowsReset'} + $row{'FlowsTimeout'} + $row{'FlowsBad'} + $row{'FlowsDoomsday'};
+               my @r = (
+                       $total
+               );
+               print D "$time";
+               for (my $i=0; $i<=$#r; $i++) {
+                       $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i];
+                       print D "\t", $ewma[$i];
+               }
+               print D "\n";
+       }
+}
+
+close D;
+open GP, "|gnuplot" or die;
+print GP <<EOF
+set terminal png
+set grid
+set style data lines
+set title "TCP flows: Total number"
+set xlabel "Time"
+set ylabel "Flows [EWMA $ew]"
+set xdata time
+set timefmt "%d-%m-%Y %H:%M:%S"
+set format x "%d/%m\\n%H:%M"
+#set xtics 86400
+set mxtics 3600
+plot   "$datafile" using 1:3 title "Total flows"
+EOF
+;
+close GP;

diff --git a/post/graph-hist-flows b/post/graph-hist-flows
new file mode 100755 (executable)
index 0000000..19c160f
--- /dev/null
+++ b/post/graph-hist-flows
@@ -0,0 +1,65 @@
+#!/usr/bin/perl
+# Draw graphs of TCP flow statistics from netgrind histograms
+# (c) 2003 Martin Mares <mj@ucw.cz>, GPL'ed
+
+use strict;
+use warnings;
+use POSIX;
+
+my $datafile = "gp.tmp";
+open D, ">$datafile" or die;
+
+my @colnames = ();
+my @ewma = ( 0, 0, 0, 0, 0 );
+my $ew = 0.9;
+my $eww = 1-$ew;
+my $w = 60;
+while (<>) {
+       chomp;
+       if (/^#/) {
+               s/^#\s+//;
+               @colnames = split /\t/;
+       } else {
+               my $i = 0;
+               s/\([^)]*\)//g;
+               my %row = map { $colnames[$i++] => $_ } split /\t/;
+               my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'});
+               my @r = (
+                       $row{'FlowsTotal'} / $w,
+                       $row{'FlowsClosed'} / $w,
+                       $row{'FlowsReset'} / $w,
+                       $row{'FlowsTimeout'} / $w,
+                       $row{'FlowsBad'} / $w
+               );
+               print D "$time";
+               for (my $i=0; $i<=$#r; $i++) {
+                       $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i];
+                       print D "\t", $ewma[$i];
+               }
+               print D "\n";
+       }
+}
+
+close D;
+open GP, "|gnuplot" or die;
+print GP <<EOF
+set terminal png
+set grid
+set style data lines
+set title "TCP flows: opens and closes"
+set xlabel "Time"
+set ylabel "Flows/s [EWMA $ew]"
+set xdata time
+set timefmt "%d-%m-%Y %H:%M:%S"
+set format x "%d/%m\\n%H:%M"
+#set logscale y 2
+#set xtics 86400
+set mxtics 3600
+plot   "$datafile" using 1:3 title "Newly created", \\
+       "$datafile" using 1:4 title "Closed", \\
+       "$datafile" using 1:5 title "Reset", \\
+       "$datafile" using 1:6 title "Timed out", \\
+       "$datafile" using 1:7 title "Corrupted"
+EOF
+;
+close GP;

diff --git a/post/graph-hist-ip b/post/graph-hist-ip
new file mode 100755 (executable)
index 0000000..2c8ca40
--- /dev/null
+++ b/post/graph-hist-ip
@@ -0,0 +1,64 @@
+#!/usr/bin/perl
+# Draw graphs of IP traffic from netgrind histograms
+# (c) 2003 Martin Mares <mj@ucw.cz>, GPL'ed
+
+use strict;
+use warnings;
+use POSIX;
+
+my $datafile = "gp.tmp";
+open D, ">$datafile" or die;
+
+my @colnames = ();
+my @ewma = ( 0, 0, 0, 0, 0 );
+my $ew = 0.9;
+my $eww = 1-$ew;
+my $w = 60;
+while (<>) {
+       chomp;
+       if (/^#/) {
+               s/^#\s+//;
+               @colnames = split /\t/;
+       } else {
+               my $i = 0;
+               s/\([^)]*\)//g;
+               my %row = map { $colnames[$i++] => $_ } split /\t/;
+               my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'});
+               my @r = (
+                       $row{'IPIn'} / $w,
+                       ($row{'IPBad'} + $row{'IPBadSum'}) / $w,
+                       $row{'IPUnint'} / $w,
+                       $row{'TCPIn'} / $w,
+                       $row{'IPFrag'} / $w
+               );
+               print D "$time";
+               for (my $i=0; $i<=$#r; $i++) {
+                       $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i];
+                       print D "\t", $ewma[$i];
+               }
+               print D "\n";
+       }
+}
+
+close D;
+open GP, "|gnuplot" or die;
+print GP <<EOF
+set terminal png
+set grid
+set style data lines
+set title "IP packet spectrum"
+set xlabel "Time"
+set ylabel "Packets / s [EWMA $ew]"
+set xdata time
+set timefmt "%d-%m-%Y %H:%M:%S"
+set format x "%d/%m\\n%H:%M"
+#set xtics 86400
+set mxtics 3600
+plot   "$datafile" using 1:3 title "All IP packets", \\
+       "$datafile" using 1:4 title "Bad IP header", \\
+       "$datafile" using 1:5 title "Non-TCP", \\
+       "$datafile" using 1:6 title "Fragmented TCP", \\
+       "$datafile" using 1:7 title "Processed TCP"
+EOF
+;
+close GP;

diff --git a/post/graph-hist-link b/post/graph-hist-link
new file mode 100755 (executable)
index 0000000..6388a1f
--- /dev/null
+++ b/post/graph-hist-link
@@ -0,0 +1,62 @@
+#!/usr/bin/perl
+# Draw graphs of link traffic from netgrind histograms
+# (c) 2003 Martin Mares <mj@ucw.cz>, GPL'ed
+
+use strict;
+use warnings;
+use POSIX;
+
+my $datafile = "gp.tmp";
+open D, ">$datafile" or die;
+
+my @colnames = ();
+my @ewma = ( 0, 0, 0 );
+my $ew = 0.9;
+my $eww = 1-$ew;
+my $w = 60;
+while (<>) {
+       chomp;
+       if (/^#/) {
+               s/^#\s+//;
+               @colnames = split /\t/;
+       } else {
+               my $i = 0;
+               my %row = map { $colnames[$i++] => $_ } split /\t/;
+               my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'});
+               my $linkin = $row{'LinkIn'};
+               $linkin =~ s/.*\((.*)\)/$1/;
+               my $ipin = $row{'IPIn'};
+               $ipin =~ s/.*\((.*)\)/$1/;
+               my @r = (
+                        $linkin * 8 / $w / 1024,
+                        $ipin * 8 / $w / 1024
+               );
+               $r[1] = $r[0] - $r[1];
+               print D "$time";
+               for (my $i=0; $i<=$#r; $i++) {
+                       $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i];
+                       print D "\t", $ewma[$i];
+               }
+               print D "\n";
+       }
+}
+
+close D;
+open GP, "|gnuplot" or die;
+print GP <<EOF
+set terminal png
+set grid
+set style data lines
+set title "Captured traffic"
+set xlabel "Time"
+set ylabel "Kbit/s [EWMA $ew]"
+set xdata time
+set timefmt "%d-%m-%Y %H:%M:%S"
+set format x "%d/%m\\n%H:%M"
+#set xtics 86400
+set mxtics 3600
+plot   "$datafile" using 1:3 title "Total traffic", \\
+       "$datafile" using 1:4 title "Non-IP traffic"
+EOF
+;
+close GP;

diff --git a/post/graph-hist-tcp b/post/graph-hist-tcp
new file mode 100755 (executable)
index 0000000..0576278
--- /dev/null
+++ b/post/graph-hist-tcp
@@ -0,0 +1,64 @@
+#!/usr/bin/perl
+# Draw graphs of TCP traffic from netgrind histograms
+# (c) 2003 Martin Mares <mj@ucw.cz>, GPL'ed
+
+use strict;
+use warnings;
+use POSIX;
+
+my $datafile = "gp.tmp";
+open D, ">$datafile" or die;
+
+my @colnames = ();
+my @ewma = ( 0, 0, 0, 0, 0 );
+my $ew = 0.9;
+my $eww = 1-$ew;
+my $w = 60;
+while (<>) {
+       chomp;
+       if (/^#/) {
+               s/^#\s+//;
+               @colnames = split /\t/;
+       } else {
+               my $i = 0;
+               s/\([^)]*\)//g;
+               my %row = map { $colnames[$i++] => $_ } split /\t/;
+               my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'});
+               my @r = (
+                       $row{'TCPIn'} / $w,
+                       $row{'TCPBad'} / $w,
+                       $row{'TCPBadSum'} / $w,
+                       $row{'TCPUnmatch'} / $w,
+                       $row{'TCPBadState'} / $w
+               );
+               print D "$time";
+               for (my $i=0; $i<=$#r; $i++) {
+                       $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i];
+                       print D "\t", $ewma[$i];
+               }
+               print D "\n";
+       }
+}
+
+close D;
+open GP, "|gnuplot" or die;
+print GP <<EOF
+set terminal png
+set grid
+set style data lines
+set title "TCP packet spectrum"
+set xlabel "Time"
+set ylabel "Packets/s [EWMA $ew]"
+set xdata time
+set timefmt "%d-%m-%Y %H:%M:%S"
+set format x "%d/%m\\n%H:%M"
+#set xtics 86400
+set mxtics 3600
+plot   "$datafile" using 1:3 title "All TCP packets", \\
+       "$datafile" using 1:4 title "Bad TCP header", \\
+       "$datafile" using 1:5 title "Bad checksum", \\
+       "$datafile" using 1:6 title "For unknown connection", \\
+       "$datafile" using 1:7 title "In bad state"
+EOF
+;
+close GP;

diff --git a/post/graph-http-cache b/post/graph-http-cache
new file mode 100755 (executable)
index 0000000..39bc104
--- /dev/null
+++ b/post/graph-http-cache
@@ -0,0 +1,62 @@
+#!/usr/bin/perl
+# Draw graphs of HTTP transaction caching from http-stats output
+# (c) 2003 Martin Mares <mj@ucw.cz>, GPL'ed
+
+use strict;
+use warnings;
+use POSIX;
+
+my $datafile = "gp.tmp";
+open D, ">$datafile" or die;
+
+my @colnames = ();
+my @ewma = ( 0, 0, 0, 0 );
+my $ew = 0.9;
+my $eww = 1-$ew;
+while (<>) {
+       chomp;
+       if (/^#/) {
+               s/^#\s+//;
+               @colnames = split /\t/;
+       } else {
+               my $i = 0;
+               s/\([^)]*\)//g;
+               my %row = map { $colnames[$i++] => $_ } split /\t/;
+               my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'});
+               my $w = $row{'width'};
+               my @r = (
+                       $row{'Total'} / $w,
+                       $row{'Nocache'} / $w,
+                       $row{'CacheHit'} / $w,
+                       $row{'CacheMiss'} / $w
+               );
+               print D "$time";
+               for (my $i=0; $i<=$#r; $i++) {
+                       $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i];
+                       print D "\t", $ewma[$i];
+               }
+               print D "\n";
+       }
+}
+
+close D;
+open GP, "|gnuplot" or die;
+print GP <<EOF
+set terminal png
+set grid
+set style data lines
+set title "HTTP Transactions: Cacheability"
+set xlabel "Time"
+set ylabel "Transactions/s [EWMA $ew]"
+set xdata time
+set timefmt "%d-%m-%Y %H:%M:%S"
+set format x "%d/%m\\n%H:%M"
+#set xtics 86400
+set mxtics 3600
+plot   "$datafile" using 1:3 title "Total", \\
+       "$datafile" using 1:4 title "Not cacheable", \\
+       "$datafile" using 1:5 title "Cache hits", \\
+       "$datafile" using 1:6 title "Cache misses"
+EOF
+;
+close GP;

diff --git a/post/graph-http-len b/post/graph-http-len
new file mode 100755 (executable)
index 0000000..2549144
--- /dev/null
+++ b/post/graph-http-len
@@ -0,0 +1,56 @@
+#!/usr/bin/perl
+# Draw graphs of HTTP transaction lengths from http-stats output
+# (c) 2003 Martin Mares <mj@ucw.cz>, GPL'ed
+
+use strict;
+use warnings;
+use POSIX;
+
+my $datafile = "gp.tmp";
+open D, ">$datafile" or die;
+
+my @colnames = ();
+my @ewma = ( 0 );
+my $ew = 0;
+my $eww = 1-$ew;
+while (<>) {
+       chomp;
+       if (/^#/) {
+               s/^#\s+//;
+               @colnames = split /\t/;
+       } else {
+               my $i = 0;
+               s/\([^)]*\)//g;
+               my %row = map { $colnames[$i++] => $_ } split /\t/;
+               my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'});
+               my @r = (
+                       ($row{'Total'} ? $row{'LenTotal'}/$row{'Total'} : 0) / 1024
+               );
+               print D "$time";
+               for (my $i=0; $i<=$#r; $i++) {
+                       $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i];
+                       print D "\t", $ewma[$i];
+               }
+               print D "\n";
+       }
+}
+
+close D;
+open GP, "|gnuplot" or die;
+print GP <<EOF
+set terminal png
+set grid
+set style data lines
+set title "HTTP Transactions: Average size"
+set xlabel "Time"
+set ylabel "KBytes [EWMA $ew]"
+set xdata time
+set timefmt "%d-%m-%Y %H:%M:%S"
+set format x "%d/%m\\n%H:%M"
+set logscale y 2
+#set xtics 86400
+set mxtics 3600
+plot   "$datafile" using 1:3 title "Size"
+EOF
+;
+close GP;

diff --git a/post/graph-http-summary b/post/graph-http-summary
new file mode 100755 (executable)
index 0000000..69a521d
--- /dev/null
+++ b/post/graph-http-summary
@@ -0,0 +1,60 @@
+#!/usr/bin/perl
+# Draw graphs of HTTP transactions from http-stats output
+# (c) 2003 Martin Mares <mj@ucw.cz>, GPL'ed
+
+use strict;
+use warnings;
+use POSIX;
+
+my $datafile = "gp.tmp";
+open D, ">$datafile" or die;
+
+my @colnames = ();
+my @ewma = ( 0, 0, 0 );
+my $ew = 0.9;
+my $eww = 1-$ew;
+while (<>) {
+       chomp;
+       if (/^#/) {
+               s/^#\s+//;
+               @colnames = split /\t/;
+       } else {
+               my $i = 0;
+               s/\([^)]*\)//g;
+               my %row = map { $colnames[$i++] => $_ } split /\t/;
+               my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'});
+               my $w = $row{'width'};
+               my @r = (
+                       $row{'Total'} / $w,
+                       $row{'TCPErr'} / $w,
+                       ($row{'Total'} - $row{'NTimings'}) / $w
+               );
+               print D "$time";
+               for (my $i=0; $i<=$#r; $i++) {
+                       $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i];
+                       print D "\t", $ewma[$i];
+               }
+               print D "\n";
+       }
+}
+
+close D;
+open GP, "|gnuplot" or die;
+print GP <<EOF
+set terminal png
+set grid
+set style data lines
+set title "HTTP Transactions: Summary"
+set xlabel "Time"
+set ylabel "Transactions/s [EWMA $ew]"
+set xdata time
+set timefmt "%d-%m-%Y %H:%M:%S"
+set format x "%d/%m\\n%H:%M"
+#set xtics 86400
+set mxtics 3600
+plot   "$datafile" using 1:3 title "Total", \\
+       "$datafile" using 1:4 title "Terminated by TCP error", \\
+       "$datafile" using 1:5 title "Not considered for timings"
+EOF
+;
+close GP;

diff --git a/post/graph-http-wait b/post/graph-http-wait
new file mode 100755 (executable)
index 0000000..b8577d9
--- /dev/null
+++ b/post/graph-http-wait
@@ -0,0 +1,73 @@
+#!/usr/bin/perl
+# Draw graphs of HTTP transaction delays from http-stats output
+# (c) 2003 Martin Mares <mj@ucw.cz>, GPL'ed
+
+use strict;
+use warnings;
+use POSIX;
+
+my $maxs = (@ARGV >= 1 && $ARGV[0] eq "--max");
+
+my $datafile = "gp.tmp";
+open D, ">$datafile" or die;
+
+my @colnames = ();
+my @ewma = ( 0, 0, 0, 0, 0, 0 );
+my $ew = 0.9;
+my $eww = 1-$ew;
+while (<STDIN>) {
+       chomp;
+       if (/^#/) {
+               s/^#\s+//;
+               @colnames = split /\t/;
+       } else {
+               my $i = 0;
+               s/\([^)]*\)//g;
+               my %row = map { $colnames[$i++] => $_ } split /\t/;
+               my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'});
+               my $n = $row{'NTimings'};
+               my @r;
+               if ($n) {
+                       @r = (
+                               $row{'TimeTotal'} / $n,
+                               $row{'TimeTMin'},
+                               $row{'TimeTMax'},
+                               $row{'TimeWait'} / $n,
+                               $row{'TimeWMin'},
+                               $row{'TimeWMax'}
+                       );
+               } else {
+                       @r = ( 0, 0, 0, 0, 0, 0 );
+               }
+               print D "$time";
+               for (my $i=0; $i<=$#r; $i++) {
+                       $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i];
+                       print D "\t", $ewma[$i];
+               }
+               print D "\n";
+       }
+}
+close D;
+
+my $ct = $maxs ? "Maximum" : "Average";
+my $c1 = $maxs ? 5 : 3;
+my $c2 = $maxs ? 8 : 6;
+
+open GP, "|gnuplot" or die;
+print GP <<EOF
+set terminal png
+set grid
+set style data lines
+set title "HTTP Transactions: $ct Timings"
+set xlabel "Time"
+set ylabel "seconds [EWMA $ew]"
+set xdata time
+set timefmt "%d-%m-%Y %H:%M:%S"
+set format x "%d/%m\\n%H:%M"
+#set xtics 86400
+set mxtics 3600
+plot   "$datafile" using 1:$c1 title "Total time", \\
+       "$datafile" using 1:$c2 title "Reply delay"
+EOF
+;
+close GP;

diff --git a/post/http-filter b/post/http-filter
new file mode 100755 (executable)
index 0000000..e8137bb
--- /dev/null
+++ b/post/http-filter
@@ -0,0 +1,19 @@
+#!/usr/bin/perl
+# Filter HTTP transactions
+# (c) 2003 Martin Mares <mj@ucw.cz>, GPL'ed
+
+use strict;
+use warnings;
+use POSIX;
+
+1 == @ARGV or die "Usage: http-filter <perl-expression>";
+eval '
+while (<STDIN>) {
+       chomp;
+       /^#/ && next;
+       my ($t1, $t2, $src, $dst, $ffor, $result, $cache, $queue, $length, $totaltime, $waittime, $ctype, $method, $url) = split /\s+/;
+       if (' . $ARGV[0] . ') {
+               print $_, "\n";
+       }
+}
+';

diff --git a/post/http-stats b/post/http-stats
new file mode 100755 (executable)
index 0000000..076a908
--- /dev/null
+++ b/post/http-stats
@@ -0,0 +1,45 @@
+#!/usr/bin/perl
+# Calculate statistics of HTTP transactions from netgrind output
+# (c) 2003 Martin Mares <mj@ucw.cz>, GPL'ed
+
+use strict;
+use warnings;
+use POSIX;
+
+print "# time  width   Total   TCPErr  Nocache CacheHit        CacheMiss       LenTotal        NTimings        TimeTotal       TimeTMin        TimeTMax        TimeWait        TimeWMin        TimeWMax        NGETs   NPOSTs\n";
+my $histogran = 60;
+my %hist = ();
+while (<>) {
+       chomp;
+       /^#/ && next;
+       my ($t1, $t2, $src, $dst, $ffor, $result, $cache, $queue, $length, $totaltime, $waittime, $ctype, $method, $url) = split /\s+/;
+       my ($tY, $tm, $td) = split (/-/, $t1);
+       my ($tH, $tM, $tS) = split (/:/, $t2);
+       my $tt = POSIX::mktime($tS, $tM, $tH, $td, $tm-1, $tY-1900);
+       $tt -= $tt % $histogran;
+       my $h = $hist{$tt};
+       if (!defined $h) {
+               $h = [ $histogran, 0, 0, 0, 0, 0, 0, 0, 0, 1000000000, 0, 0, 1000000000, 0, 0, 0 ];
+               $hist{$tt} = $h;
+       }
+       $h->[1]++;
+       $h->[2]++ if $result =~ /^[TC]/;
+       $h->[3]++ if $cache =~ /^((N..)|(.[NP].))$/;
+       $h->[4]++ if $cache =~ /\+$/;
+       $h->[5]++ if $cache =~ /-$/;
+       $h->[6]+=$length;
+       if ($result =~ /^\d+$/) {
+               $h->[7]++;
+               $h->[8]+=$totaltime;
+               $h->[9]=$totaltime if $h->[9] > $totaltime;
+               $h->[10]=$totaltime if $h->[9] < $totaltime;
+               $h->[11]+=$waittime;
+               $h->[12]=$waittime if $h->[12] > $waittime;
+               $h->[13]=$waittime if $h->[12] < $waittime;
+       }
+       $h->[14]++ if $method eq "GET";
+       $h->[15]++ if $method eq "POST";
+}
+foreach my $x (sort keys %hist) {
+       print "$x\t", join("\t", @{$hist{$x}}), "\n";
+}