From bd645a07df17ba2d812f19231af43061d8dfaa61 Mon Sep 17 00:00:00 2001 From: Martin Mares Date: Mon, 17 Jun 2013 18:19:05 +0200 Subject: [PATCH] Implemented dumping of HTTP transactions Also changed format of HTTP transaction log, so that it includes transaction IDs. --- TODO | 1 - netgrind/http.c | 77 +++++++++++++++++++++++++++++++++++++++------ netgrind/netgrind.c | 9 ++++-- netgrind/netgrind.h | 3 +- 4 files changed, 77 insertions(+), 13 deletions(-) diff --git a/TODO b/TODO index 5b53e73..bb690c6 100644 --- a/TODO +++ b/TODO @@ -4,4 +4,3 @@ Various ideas: - decoding SOCKS - data type recognition ? - count extra data between IP end and eth end -- dump HTTP transaction to files diff --git a/netgrind/http.c b/netgrind/http.c index ee7b4fc..e27f57d 100644 --- a/netgrind/http.c +++ b/netgrind/http.c @@ -1,7 +1,7 @@ /* * Netgrind -- HTTP Analyser * - * (c) 2003 Martin Mares + * (c) 2003--2013 Martin Mares * * This software may be freely distributed and used according to the terms * of the GNU General Public License. @@ -29,6 +29,7 @@ struct http_header { }; struct http_state { + struct flow *flow; enum { HTTP_IDLE, /* initialized, waiting for request */ HTTP_ERROR, /* protocol error, ignoring everything else */ @@ -56,12 +57,16 @@ struct http_state { uns body_end_state; uns body_total_size; uns req_counter; + FILE *log_file; }; +char *http_log_dir; + static void http_open(struct flow *f, u64 when) { static int http_counter; struct http_state *s = xmalloc_zero(sizeof(*s)); + s->flow = f; f->appl_data = s; s->id = http_counter++; DBG("HTTP: %d NEW %d.%d.%d.%d:%d -> %d.%d.%d.%d:%d\n", s->id, @@ -104,13 +109,60 @@ static uns find_token(byte *hay, byte *needle) return 0; } +static void http_log_start(struct http_state *s) +{ + if (!http_log_dir) + return; + + char name[256], stamp[TIMESTAMP_LEN]; + struct flow *f = s->flow; + + sprintf(name, "%s/%06u-%d.%d.%d.%d:%d-%d.%d.%d.%d:%d", http_log_dir, s->id, + IPQUAD(f->saddr), ntohs(f->sport), IPQUAD(f->daddr), ntohs(f->dport)); + if (!(s->log_file = fopen(name, "w"))) + die("Unable to create %s: %m", name); + + format_timestamp(stamp, s->req_start_time); + fprintf(s->log_file, "; [%s] From %d.%d.%d.%d:%d to %d.%d.%d.%d:%d (req %u)\n", + stamp, IPQUAD(f->saddr), ntohs(f->sport), IPQUAD(f->daddr), ntohs(f->dport), + s->req_counter + 1); +} + +static void http_log_end(struct http_state *s) +{ + if (!s->log_file) + return; + fclose(s->log_file); + s->log_file = NULL; +} + +static void http_log_req_line(struct http_state *s, byte *line) +{ + if (s->log_file) + fprintf(s->log_file, "> %s\n", line); +} + +static void http_log_resp_line(struct http_state *s, byte *line) +{ + if (s->log_file) + fprintf(s->log_file, "< %s\n", line); +} + +static void http_log_body(struct http_state *s, byte *data, uns len) +{ + if (s->log_file) + fwrite(data, len, 1, s->log_file); +} + static void http_report(struct flow *f, struct http_state *s, u64 when, byte *reason) { byte *method, *url, *x, *y, *stat; - static uns http_counter; if (!(method = s->req_line)) - return; + { + http_log_end(s); + return; + } /* Analyse request line */ url = method; @@ -193,11 +245,11 @@ static void http_report(struct flow *f, struct http_state *s, u64 when, byte *re byte *sep; if (sep = strchr(ctype, ';')) *sep = 0; - if (!http_counter++) - printf("# timestamp source destination forwarded-for res cac que length total time wait time ctype method URL\n"); - /* 2003-06-06 22:53:38.642 81.27.194.19:1175 205.217.153.53:80 123.123.123.123 200 ... 0 14030 0.957 0.444 text/plain GET http://... */ - printf("%s %-21s %-21s %-15s %-3s %c%c%c %3d %8d %6d.%03d %6d.%03d %-12s %s %s\n", - stamp, src, dst, (ffor ? : "-"), reason, + if (!s->id) + printf("# id timestamp source destination forwarded-for res cac que length total time wait time ctype method URL\n"); + /* 000000 2003-06-06 22:53:38.642 81.27.194.19:1175 205.217.153.53:80 123.123.123.123 200 ... 0 14030 0.957 0.444 text/plain GET http://... */ + printf("%06u %s %-21s %-21s %-15s %-3s %c%c%c %3d %8d %6d.%03d %6d.%03d %-12s %s %s\n", + s->id, stamp, src, dst, (ffor ? : "-"), reason, rq_cflag, rp_cflag, rp_hit, s->req_counter, s->body_total_size, @@ -205,6 +257,7 @@ static void http_report(struct flow *f, struct http_state *s, u64 when, byte *re (uns)(tresp/1000000), (uns)(tresp%1000000)/1000, ctype, method, url); + http_log_end(s); s->req_counter++; } @@ -235,6 +288,7 @@ static void http_close(struct flow *f, int cause, u64 when) break; default: ; } + http_log_end(s); pkt_flush_queue(&s->rx_queue); pkt_flush_queue(&s->tx_queue); if (s->pool) @@ -287,6 +341,7 @@ static int http_skip_body_bytes(struct http_state *s) uns avail = pkt_len(p); uns want = s->body_len; uns go = MIN(avail, want); + http_log_body(s, p->data, go); p->data += go; s->body_len -= go; s->body_total_size += go; @@ -325,6 +380,8 @@ static void http_init_xact(struct http_state *s) s->req_line = s->resp_line = NULL; s->line_len = 0; s->body_total_size = 0; + + http_log_start(s); } static void http_parse_hdr(list *l, struct http_header *h) @@ -436,9 +493,9 @@ static void http_input(struct flow *f, int dir, struct pkt *p) if (fin_tx || !http_have_input(&s->tx_queue)) return; s->state = HTTP_REQUEST; - http_init_xact(s); if (!s->req_start_time) s->req_start_time = p->timestamp; + http_init_xact(s); break; case HTTP_REQUEST: if (fin_tx || fin_rx) @@ -446,6 +503,7 @@ static void http_input(struct flow *f, int dir, struct pkt *p) if (!(h = http_get_line(s, &s->tx_queue))) return; DBG("\t>> %s\n", h->buf); + http_log_req_line(s, h->buf); if (!s->req_line) { if (!h->buf[0]) @@ -512,6 +570,7 @@ static void http_input(struct flow *f, int dir, struct pkt *p) if (!(h = http_get_line(s, &s->rx_queue))) return; DBG("\t<< %s\n", h->buf); + http_log_resp_line(s, h->buf); if (!s->resp_line) { if (!h->buf[0]) diff --git a/netgrind/netgrind.c b/netgrind/netgrind.c index da02074..b3d1a3c 100644 --- a/netgrind/netgrind.c +++ b/netgrind/netgrind.c @@ -1,7 +1,7 @@ /* * Netgrind -- The Network Traffic Analyser * - * (c) 2003 Martin Mares + * (c) 2003--2013 Martin Mares * * This software may be freely distributed and used according to the terms * of the GNU General Public License. @@ -125,6 +125,7 @@ static void usage(void) -s Dump connection summary\n\ -t Calculate statistics only\n\ -w TCP: Wait for ACK before processing packets\n\ +-x Dump HTTP transactions to a given directory\n\ "); exit(1); } @@ -141,7 +142,7 @@ int main(int argc, char **argv) byte *histogram = NULL; tcp_default_appl = &appl_mux; - while ((c = getopt(argc, argv, "ac:d:D:f:h:stw")) >= 0) + while ((c = getopt(argc, argv, "ac:d:D:f:h:stwx:")) >= 0) switch (c) { case 'a': @@ -173,6 +174,9 @@ int main(int argc, char **argv) case 't': tcp_default_appl = &appl_sink; break; + case 'x': + http_log_dir = optarg; + break; default: usage(); } @@ -242,6 +246,7 @@ int main(int argc, char **argv) } tcp_cleanup(last_timestamp); histogram_cleanup(); + printf("\n"); printf("# Netgrind statistics:\n"); printf("# Pcap: %Ld(%Ld) in, %Ld(%Ld) incomplete\n", stat_pcap_in.packets, stat_pcap_in.bytes, diff --git a/netgrind/netgrind.h b/netgrind/netgrind.h index fae6071..a72fc3d 100644 --- a/netgrind/netgrind.h +++ b/netgrind/netgrind.h @@ -1,7 +1,7 @@ /* * Netgrind -- The Network Traffic Analyser * - * (c) 2003 Martin Mares + * (c) 2003--2013 Martin Mares * * This software may be freely distributed and used according to the terms * of the GNU General Public License. @@ -103,6 +103,7 @@ void format_timestamp(byte *buf, u64 time); /* http.c */ extern struct appl_hooks appl_http; +extern char *http_log_dir; /* histogram.c */ -- 2.39.5