Implemented dumping of HTTP transactions

Also changed format of HTTP transaction log, so that it includes
transaction IDs.

diff --git a/TODO b/TODO
index 5b53e7314aa2e38ed0f1b294967649c6186702c7..bb690c61ddb0dd18e6fd6903dec905daac2941d9 100644 (file)
--- a/TODO
+++ b/TODO
@@ -4,4 +4,3 @@ Various ideas:
 - decoding SOCKS
 - data type recognition ?
 - count extra data between IP end and eth end
-- dump HTTP transaction to files

diff --git a/netgrind/http.c b/netgrind/http.c
index ee7b4fc7af1ab8e0656a66b1726ceafe16b9d33a..e27f57dc67740bd284343de6e96a5cf858e706c4 100644 (file)
--- a/netgrind/http.c
+++ b/netgrind/http.c
@@ -1,7 +1,7 @@
 /*
  *     Netgrind -- HTTP Analyser
  *
- *     (c) 2003 Martin Mares <mj@ucw.cz>
+ *     (c) 2003--2013 Martin Mares <mj@ucw.cz>
  *
  *     This software may be freely distributed and used according to the terms
  *     of the GNU General Public License.
@@ -29,6 +29,7 @@ struct http_header {
 };
 
 struct http_state {
+  struct flow *flow;
   enum {
     HTTP_IDLE,                 /* initialized, waiting for request */
     HTTP_ERROR,                        /* protocol error, ignoring everything else */
@@ -56,12 +57,16 @@ struct http_state {
   uns body_end_state;
   uns body_total_size;
   uns req_counter;
+  FILE *log_file;
 };
 
+char *http_log_dir;
+
 static void http_open(struct flow *f, u64 when)
 {
   static int http_counter;
   struct http_state *s = xmalloc_zero(sizeof(*s));
+  s->flow = f;
   f->appl_data = s;
   s->id = http_counter++;
   DBG("HTTP: %d NEW %d.%d.%d.%d:%d -> %d.%d.%d.%d:%d\n", s->id,
@@ -104,13 +109,60 @@ static uns find_token(byte *hay, byte *needle)
   return 0;
 }
 
+static void http_log_start(struct http_state *s)
+{
+  if (!http_log_dir)
+    return;
+
+  char name[256], stamp[TIMESTAMP_LEN];
+  struct flow *f = s->flow;
+
+  sprintf(name, "%s/%06u-%d.%d.%d.%d:%d-%d.%d.%d.%d:%d", http_log_dir, s->id,
+    IPQUAD(f->saddr), ntohs(f->sport), IPQUAD(f->daddr), ntohs(f->dport));
+  if (!(s->log_file = fopen(name, "w")))
+    die("Unable to create %s: %m", name);
+
+  format_timestamp(stamp, s->req_start_time);
+  fprintf(s->log_file, "; [%s] From %d.%d.%d.%d:%d to %d.%d.%d.%d:%d (req %u)\n",
+    stamp, IPQUAD(f->saddr), ntohs(f->sport), IPQUAD(f->daddr), ntohs(f->dport),
+    s->req_counter + 1);
+}
+
+static void http_log_end(struct http_state *s)
+{
+  if (!s->log_file)
+    return;
+  fclose(s->log_file);
+  s->log_file = NULL;
+}
+
+static void http_log_req_line(struct http_state *s, byte *line)
+{
+  if (s->log_file)
+    fprintf(s->log_file, "> %s\n", line);
+}
+
+static void http_log_resp_line(struct http_state *s, byte *line)
+{
+  if (s->log_file)
+    fprintf(s->log_file, "< %s\n", line);
+}
+
+static void http_log_body(struct http_state *s, byte *data, uns len)
+{
+  if (s->log_file)
+    fwrite(data, len, 1, s->log_file);
+}
+
 static void http_report(struct flow *f, struct http_state *s, u64 when, byte *reason)
 {
   byte *method, *url, *x, *y, *stat;
-  static uns http_counter;
 
   if (!(method = s->req_line))
-    return;
+    {
+      http_log_end(s);
+      return;
+    }
 
   /* Analyse request line */
   url = method;
@@ -193,11 +245,11 @@ static void http_report(struct flow *f, struct http_state *s, u64 when, byte *re
   byte *sep;
   if (sep = strchr(ctype, ';'))
     *sep = 0;
-  if (!http_counter++)
-    printf("# timestamp             source                destination           forwarded-for   res cac que   length total time  wait time ctype      method URL\n");
-         /* 2003-06-06 22:53:38.642 81.27.194.19:1175     205.217.153.53:80     123.123.123.123 200 ...   0    14030      0.957      0.444 text/plain GET http://... */
-  printf("%s %-21s %-21s %-15s %-3s %c%c%c %3d %8d %6d.%03d %6d.%03d %-12s %s %s\n",
-        stamp, src, dst, (ffor ? : "-"), reason,
+  if (!s->id)
+    printf("# id   timestamp               source                destination           forwarded-for   res cac que   length total time  wait time ctype      method URL\n");
+         /* 000000 2003-06-06 22:53:38.642 81.27.194.19:1175     205.217.153.53:80     123.123.123.123 200 ...   0    14030      0.957      0.444 text/plain GET http://... */
+  printf("%06u %s %-21s %-21s %-15s %-3s %c%c%c %3d %8d %6d.%03d %6d.%03d %-12s %s %s\n",
+        s->id, stamp, src, dst, (ffor ? : "-"), reason,
         rq_cflag, rp_cflag, rp_hit,
         s->req_counter,
         s->body_total_size,
@@ -205,6 +257,7 @@ static void http_report(struct flow *f, struct http_state *s, u64 when, byte *re
         (uns)(tresp/1000000), (uns)(tresp%1000000)/1000,
         ctype, method, url);
 
+  http_log_end(s);
   s->req_counter++;
 }
 
@@ -235,6 +288,7 @@ static void http_close(struct flow *f, int cause, u64 when)
        break;
       default: ;
       }
+  http_log_end(s);
   pkt_flush_queue(&s->rx_queue);
   pkt_flush_queue(&s->tx_queue);
   if (s->pool)
@@ -287,6 +341,7 @@ static int http_skip_body_bytes(struct http_state *s)
       uns avail = pkt_len(p);
       uns want = s->body_len;
       uns go = MIN(avail, want);
+      http_log_body(s, p->data, go);
       p->data += go;
       s->body_len -= go;
       s->body_total_size += go;
@@ -325,6 +380,8 @@ static void http_init_xact(struct http_state *s)
   s->req_line = s->resp_line = NULL;
   s->line_len = 0;
   s->body_total_size = 0;
+
+  http_log_start(s);
 }
 
 static void http_parse_hdr(list *l, struct http_header *h)
@@ -436,9 +493,9 @@ static void http_input(struct flow *f, int dir, struct pkt *p)
          if (fin_tx || !http_have_input(&s->tx_queue))
            return;
          s->state = HTTP_REQUEST;
-         http_init_xact(s);
          if (!s->req_start_time)
            s->req_start_time = p->timestamp;
+         http_init_xact(s);
          break;
        case HTTP_REQUEST:
          if (fin_tx || fin_rx)
@@ -446,6 +503,7 @@ static void http_input(struct flow *f, int dir, struct pkt *p)
          if (!(h = http_get_line(s, &s->tx_queue)))
            return;
          DBG("\t>> %s\n", h->buf);
+         http_log_req_line(s, h->buf);
          if (!s->req_line)
            {
              if (!h->buf[0])
@@ -512,6 +570,7 @@ static void http_input(struct flow *f, int dir, struct pkt *p)
          if (!(h = http_get_line(s, &s->rx_queue)))
            return;
          DBG("\t<< %s\n", h->buf);
+         http_log_resp_line(s, h->buf);
          if (!s->resp_line)
            {
              if (!h->buf[0])

diff --git a/netgrind/netgrind.c b/netgrind/netgrind.c
index da020748a0340ca46fb3aef1cd01fab29db68a3d..b3d1a3c621c516c2c38e17a6990d4590e23ccf5c 100644 (file)
--- a/netgrind/netgrind.c
+++ b/netgrind/netgrind.c
@@ -1,7 +1,7 @@
 /*
  *     Netgrind -- The Network Traffic Analyser
  *
- *     (c) 2003 Martin Mares <mj@ucw.cz>
+ *     (c) 2003--2013 Martin Mares <mj@ucw.cz>
  *
  *     This software may be freely distributed and used according to the terms
  *     of the GNU General Public License.
@@ -125,6 +125,7 @@ static void usage(void)
 -s             Dump connection summary\n\
 -t             Calculate statistics only\n\
 -w             TCP: Wait for ACK before processing packets\n\
+-x <dir>       Dump HTTP transactions to a given directory\n\
 ");
   exit(1);
 }
@@ -141,7 +142,7 @@ int main(int argc, char **argv)
   byte *histogram = NULL;
 
   tcp_default_appl = &appl_mux;
-  while ((c = getopt(argc, argv, "ac:d:D:f:h:stw")) >= 0)
+  while ((c = getopt(argc, argv, "ac:d:D:f:h:stwx:")) >= 0)
     switch (c)
       {
       case 'a':
@@ -173,6 +174,9 @@ int main(int argc, char **argv)
       case 't':
        tcp_default_appl = &appl_sink;
        break;
+      case 'x':
+       http_log_dir = optarg;
+       break;
       default:
        usage();
       }
@@ -242,6 +246,7 @@ int main(int argc, char **argv)
     }
   tcp_cleanup(last_timestamp);
   histogram_cleanup();
+  printf("\n");
   printf("# Netgrind statistics:\n");
   printf("# Pcap: %Ld(%Ld) in, %Ld(%Ld) incomplete\n",
         stat_pcap_in.packets, stat_pcap_in.bytes,

diff --git a/netgrind/netgrind.h b/netgrind/netgrind.h
index fae60718b22b5349e3a4606762d0088e2e7be6a1..a72fc3d64ae84c3d03bc68ba9bcffde9c17c7936 100644 (file)
--- a/netgrind/netgrind.h
+++ b/netgrind/netgrind.h
@@ -1,7 +1,7 @@
 /*
  *     Netgrind -- The Network Traffic Analyser
  *
- *     (c) 2003 Martin Mares <mj@ucw.cz>
+ *     (c) 2003--2013 Martin Mares <mj@ucw.cz>
  *
  *     This software may be freely distributed and used according to the terms
  *     of the GNU General Public License.
@@ -103,6 +103,7 @@ void format_timestamp(byte *buf, u64 time);
 /* http.c */
 
 extern struct appl_hooks appl_http;
+extern char *http_log_dir;
 
 /* histogram.c */