From 51c4d8c10be0e0a0cfe3450aecca65a759721ce3 Mon Sep 17 00:00:00 2001 From: Martin Mares Date: Fri, 20 Jun 2003 14:13:00 +0000 Subject: [PATCH] Added various graphing and filtering utilities. --- post/graph-hist-flowcount | 59 +++++++++++++++++++++++++++++++ post/graph-hist-flows | 65 ++++++++++++++++++++++++++++++++++ post/graph-hist-ip | 64 ++++++++++++++++++++++++++++++++++ post/graph-hist-link | 62 +++++++++++++++++++++++++++++++++ post/graph-hist-tcp | 64 ++++++++++++++++++++++++++++++++++ post/graph-http-cache | 62 +++++++++++++++++++++++++++++++++ post/graph-http-len | 56 ++++++++++++++++++++++++++++++ post/graph-http-summary | 60 ++++++++++++++++++++++++++++++++ post/graph-http-wait | 73 +++++++++++++++++++++++++++++++++++++++ post/http-filter | 19 ++++++++++ post/http-stats | 45 ++++++++++++++++++++++++ 11 files changed, 629 insertions(+) create mode 100755 post/graph-hist-flowcount create mode 100755 post/graph-hist-flows create mode 100755 post/graph-hist-ip create mode 100755 post/graph-hist-link create mode 100755 post/graph-hist-tcp create mode 100755 post/graph-http-cache create mode 100755 post/graph-http-len create mode 100755 post/graph-http-summary create mode 100755 post/graph-http-wait create mode 100755 post/http-filter create mode 100755 post/http-stats diff --git a/post/graph-hist-flowcount b/post/graph-hist-flowcount new file mode 100755 index 0000000..b6544e3 --- /dev/null +++ b/post/graph-hist-flowcount @@ -0,0 +1,59 @@ +#!/usr/bin/perl +# Draw graphs of TCP flow statistics from netgrind histograms +# (c) 2003 Martin Mares , GPL'ed + +use strict; +use warnings; +use POSIX; + +my $datafile = "gp.tmp"; +open D, ">$datafile" or die; + +my @colnames = (); +my $total = 0; +my @ewma = ( 0 ); +my $ew = 0.9; +my $eww = 1-$ew; +my $w = 60; +while (<>) { + chomp; + if (/^#/) { + s/^#\s+//; + @colnames = split /\t/; + } else { + my $i = 0; + s/\([^)]*\)//g; + my %row = map { $colnames[$i++] => $_ } split /\t/; + my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'}); + $total += $row{'FlowsTotal'}; + $total -= $row{'FlowsClosed'} + $row{'FlowsReset'} + $row{'FlowsTimeout'} + $row{'FlowsBad'} + $row{'FlowsDoomsday'}; + my @r = ( + $total + ); + print D "$time"; + for (my $i=0; $i<=$#r; $i++) { + $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i]; + print D "\t", $ewma[$i]; + } + print D "\n"; + } +} + +close D; +open GP, "|gnuplot" or die; +print GP <, GPL'ed + +use strict; +use warnings; +use POSIX; + +my $datafile = "gp.tmp"; +open D, ">$datafile" or die; + +my @colnames = (); +my @ewma = ( 0, 0, 0, 0, 0 ); +my $ew = 0.9; +my $eww = 1-$ew; +my $w = 60; +while (<>) { + chomp; + if (/^#/) { + s/^#\s+//; + @colnames = split /\t/; + } else { + my $i = 0; + s/\([^)]*\)//g; + my %row = map { $colnames[$i++] => $_ } split /\t/; + my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'}); + my @r = ( + $row{'FlowsTotal'} / $w, + $row{'FlowsClosed'} / $w, + $row{'FlowsReset'} / $w, + $row{'FlowsTimeout'} / $w, + $row{'FlowsBad'} / $w + ); + print D "$time"; + for (my $i=0; $i<=$#r; $i++) { + $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i]; + print D "\t", $ewma[$i]; + } + print D "\n"; + } +} + +close D; +open GP, "|gnuplot" or die; +print GP <, GPL'ed + +use strict; +use warnings; +use POSIX; + +my $datafile = "gp.tmp"; +open D, ">$datafile" or die; + +my @colnames = (); +my @ewma = ( 0, 0, 0, 0, 0 ); +my $ew = 0.9; +my $eww = 1-$ew; +my $w = 60; +while (<>) { + chomp; + if (/^#/) { + s/^#\s+//; + @colnames = split /\t/; + } else { + my $i = 0; + s/\([^)]*\)//g; + my %row = map { $colnames[$i++] => $_ } split /\t/; + my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'}); + my @r = ( + $row{'IPIn'} / $w, + ($row{'IPBad'} + $row{'IPBadSum'}) / $w, + $row{'IPUnint'} / $w, + $row{'TCPIn'} / $w, + $row{'IPFrag'} / $w + ); + print D "$time"; + for (my $i=0; $i<=$#r; $i++) { + $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i]; + print D "\t", $ewma[$i]; + } + print D "\n"; + } +} + +close D; +open GP, "|gnuplot" or die; +print GP <, GPL'ed + +use strict; +use warnings; +use POSIX; + +my $datafile = "gp.tmp"; +open D, ">$datafile" or die; + +my @colnames = (); +my @ewma = ( 0, 0, 0 ); +my $ew = 0.9; +my $eww = 1-$ew; +my $w = 60; +while (<>) { + chomp; + if (/^#/) { + s/^#\s+//; + @colnames = split /\t/; + } else { + my $i = 0; + my %row = map { $colnames[$i++] => $_ } split /\t/; + my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'}); + my $linkin = $row{'LinkIn'}; + $linkin =~ s/.*\((.*)\)/$1/; + my $ipin = $row{'IPIn'}; + $ipin =~ s/.*\((.*)\)/$1/; + my @r = ( + $linkin * 8 / $w / 1024, + $ipin * 8 / $w / 1024 + ); + $r[1] = $r[0] - $r[1]; + print D "$time"; + for (my $i=0; $i<=$#r; $i++) { + $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i]; + print D "\t", $ewma[$i]; + } + print D "\n"; + } +} + +close D; +open GP, "|gnuplot" or die; +print GP <, GPL'ed + +use strict; +use warnings; +use POSIX; + +my $datafile = "gp.tmp"; +open D, ">$datafile" or die; + +my @colnames = (); +my @ewma = ( 0, 0, 0, 0, 0 ); +my $ew = 0.9; +my $eww = 1-$ew; +my $w = 60; +while (<>) { + chomp; + if (/^#/) { + s/^#\s+//; + @colnames = split /\t/; + } else { + my $i = 0; + s/\([^)]*\)//g; + my %row = map { $colnames[$i++] => $_ } split /\t/; + my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'}); + my @r = ( + $row{'TCPIn'} / $w, + $row{'TCPBad'} / $w, + $row{'TCPBadSum'} / $w, + $row{'TCPUnmatch'} / $w, + $row{'TCPBadState'} / $w + ); + print D "$time"; + for (my $i=0; $i<=$#r; $i++) { + $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i]; + print D "\t", $ewma[$i]; + } + print D "\n"; + } +} + +close D; +open GP, "|gnuplot" or die; +print GP <, GPL'ed + +use strict; +use warnings; +use POSIX; + +my $datafile = "gp.tmp"; +open D, ">$datafile" or die; + +my @colnames = (); +my @ewma = ( 0, 0, 0, 0 ); +my $ew = 0.9; +my $eww = 1-$ew; +while (<>) { + chomp; + if (/^#/) { + s/^#\s+//; + @colnames = split /\t/; + } else { + my $i = 0; + s/\([^)]*\)//g; + my %row = map { $colnames[$i++] => $_ } split /\t/; + my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'}); + my $w = $row{'width'}; + my @r = ( + $row{'Total'} / $w, + $row{'Nocache'} / $w, + $row{'CacheHit'} / $w, + $row{'CacheMiss'} / $w + ); + print D "$time"; + for (my $i=0; $i<=$#r; $i++) { + $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i]; + print D "\t", $ewma[$i]; + } + print D "\n"; + } +} + +close D; +open GP, "|gnuplot" or die; +print GP <, GPL'ed + +use strict; +use warnings; +use POSIX; + +my $datafile = "gp.tmp"; +open D, ">$datafile" or die; + +my @colnames = (); +my @ewma = ( 0 ); +my $ew = 0; +my $eww = 1-$ew; +while (<>) { + chomp; + if (/^#/) { + s/^#\s+//; + @colnames = split /\t/; + } else { + my $i = 0; + s/\([^)]*\)//g; + my %row = map { $colnames[$i++] => $_ } split /\t/; + my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'}); + my @r = ( + ($row{'Total'} ? $row{'LenTotal'}/$row{'Total'} : 0) / 1024 + ); + print D "$time"; + for (my $i=0; $i<=$#r; $i++) { + $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i]; + print D "\t", $ewma[$i]; + } + print D "\n"; + } +} + +close D; +open GP, "|gnuplot" or die; +print GP <, GPL'ed + +use strict; +use warnings; +use POSIX; + +my $datafile = "gp.tmp"; +open D, ">$datafile" or die; + +my @colnames = (); +my @ewma = ( 0, 0, 0 ); +my $ew = 0.9; +my $eww = 1-$ew; +while (<>) { + chomp; + if (/^#/) { + s/^#\s+//; + @colnames = split /\t/; + } else { + my $i = 0; + s/\([^)]*\)//g; + my %row = map { $colnames[$i++] => $_ } split /\t/; + my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'}); + my $w = $row{'width'}; + my @r = ( + $row{'Total'} / $w, + $row{'TCPErr'} / $w, + ($row{'Total'} - $row{'NTimings'}) / $w + ); + print D "$time"; + for (my $i=0; $i<=$#r; $i++) { + $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i]; + print D "\t", $ewma[$i]; + } + print D "\n"; + } +} + +close D; +open GP, "|gnuplot" or die; +print GP <, GPL'ed + +use strict; +use warnings; +use POSIX; + +my $maxs = (@ARGV >= 1 && $ARGV[0] eq "--max"); + +my $datafile = "gp.tmp"; +open D, ">$datafile" or die; + +my @colnames = (); +my @ewma = ( 0, 0, 0, 0, 0, 0 ); +my $ew = 0.9; +my $eww = 1-$ew; +while () { + chomp; + if (/^#/) { + s/^#\s+//; + @colnames = split /\t/; + } else { + my $i = 0; + s/\([^)]*\)//g; + my %row = map { $colnames[$i++] => $_ } split /\t/; + my $time = POSIX::strftime("%d-%m-%Y %H:%M:%S", localtime $row{'time'}); + my $n = $row{'NTimings'}; + my @r; + if ($n) { + @r = ( + $row{'TimeTotal'} / $n, + $row{'TimeTMin'}, + $row{'TimeTMax'}, + $row{'TimeWait'} / $n, + $row{'TimeWMin'}, + $row{'TimeWMax'} + ); + } else { + @r = ( 0, 0, 0, 0, 0, 0 ); + } + print D "$time"; + for (my $i=0; $i<=$#r; $i++) { + $ewma[$i] = $ew*$ewma[$i] + $eww*$r[$i]; + print D "\t", $ewma[$i]; + } + print D "\n"; + } +} +close D; + +my $ct = $maxs ? "Maximum" : "Average"; +my $c1 = $maxs ? 5 : 3; +my $c2 = $maxs ? 8 : 6; + +open GP, "|gnuplot" or die; +print GP <, GPL'ed + +use strict; +use warnings; +use POSIX; + +1 == @ARGV or die "Usage: http-filter "; +eval ' +while () { + chomp; + /^#/ && next; + my ($t1, $t2, $src, $dst, $ffor, $result, $cache, $queue, $length, $totaltime, $waittime, $ctype, $method, $url) = split /\s+/; + if (' . $ARGV[0] . ') { + print $_, "\n"; + } +} +'; diff --git a/post/http-stats b/post/http-stats new file mode 100755 index 0000000..076a908 --- /dev/null +++ b/post/http-stats @@ -0,0 +1,45 @@ +#!/usr/bin/perl +# Calculate statistics of HTTP transactions from netgrind output +# (c) 2003 Martin Mares , GPL'ed + +use strict; +use warnings; +use POSIX; + +print "# time width Total TCPErr Nocache CacheHit CacheMiss LenTotal NTimings TimeTotal TimeTMin TimeTMax TimeWait TimeWMin TimeWMax NGETs NPOSTs\n"; +my $histogran = 60; +my %hist = (); +while (<>) { + chomp; + /^#/ && next; + my ($t1, $t2, $src, $dst, $ffor, $result, $cache, $queue, $length, $totaltime, $waittime, $ctype, $method, $url) = split /\s+/; + my ($tY, $tm, $td) = split (/-/, $t1); + my ($tH, $tM, $tS) = split (/:/, $t2); + my $tt = POSIX::mktime($tS, $tM, $tH, $td, $tm-1, $tY-1900); + $tt -= $tt % $histogran; + my $h = $hist{$tt}; + if (!defined $h) { + $h = [ $histogran, 0, 0, 0, 0, 0, 0, 0, 0, 1000000000, 0, 0, 1000000000, 0, 0, 0 ]; + $hist{$tt} = $h; + } + $h->[1]++; + $h->[2]++ if $result =~ /^[TC]/; + $h->[3]++ if $cache =~ /^((N..)|(.[NP].))$/; + $h->[4]++ if $cache =~ /\+$/; + $h->[5]++ if $cache =~ /-$/; + $h->[6]+=$length; + if ($result =~ /^\d+$/) { + $h->[7]++; + $h->[8]+=$totaltime; + $h->[9]=$totaltime if $h->[9] > $totaltime; + $h->[10]=$totaltime if $h->[9] < $totaltime; + $h->[11]+=$waittime; + $h->[12]=$waittime if $h->[12] > $waittime; + $h->[13]=$waittime if $h->[12] < $waittime; + } + $h->[14]++ if $method eq "GET"; + $h->[15]++ if $method eq "POST"; +} +foreach my $x (sort keys %hist) { + print "$x\t", join("\t", @{$hist{$x}}), "\n"; +} -- 2.39.5