From: Martin Mares <mj@ucw.cz>
Date: Mon, 10 Sep 2007 11:38:43 +0000 (+0200)
Subject: Slightly improved user interface of {asio,file}-test and renamed them
X-Git-Tag: holmes-import~506^2~13^2~34
X-Git-Url: http://mj.ucw.cz/gitweb/?a=commitdiff_plain;h=7ebe76cbc0bf64a033e9d978dd22cea3257e128b;p=libucw.git

Slightly improved user interface of {asio,file}-test and renamed them
to radix-*.
---

diff --git a/debug/sorter/Makefile b/debug/sorter/Makefile
index e5800213..93fa99cc 100644
--- a/debug/sorter/Makefile
+++ b/debug/sorter/Makefile
@@ -1,8 +1,8 @@
-# Testing the new sorter
+# Tests related to the new sorter
 
 DIRS+=debug/sorter
-PROGS+=$(addprefix $(o)/debug/sorter/,retros file-test asio-test)
+PROGS+=$(addprefix $(o)/debug/sorter/,retros radix-file-test radix-asio-test)
 
 $(o)/debug/sorter/retros: $(o)/debug/sorter/retros.o $(LIBSH)
-$(o)/debug/sorter/file-test: $(o)/debug/sorter/file-test.o $(LIBSH)
-$(o)/debug/sorter/asio-test: $(o)/debug/sorter/asio-test.o $(LIBSH)
+$(o)/debug/sorter/radix-file-test: $(o)/debug/sorter/radix-file-test.o $(LIBSH)
+$(o)/debug/sorter/radix-asio-test: $(o)/debug/sorter/radix-asio-test.o $(LIBSH)
diff --git a/debug/sorter/asio-test.c b/debug/sorter/asio-test.c
deleted file mode 100644
index 40df77b3..00000000
--- a/debug/sorter/asio-test.c
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- *  An experiment with parallel reading and writing of files using ASIO.
- */
-
-#include "lib/lib.h"
-#include "lib/lfs.h"
-#include "lib/asio.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <fcntl.h>
-#include <unistd.h>
-
-#define COPY
-#define DIRECT O_DIRECT
-
-static timestamp_t timer;
-
-#define P_INIT do { cnt = 0; cnt_rep = 0; cnt_ms = 1; } while(0)
-#define P_UPDATE(cc) do { \
-  cnt += cc; \
-  if (cnt >= cnt_rep) { cnt_ms += get_timer(&timer); \
-    printf("%d of %d MB (%.2f MB/sec)\r", (int)(cnt >> 20), (int)(total_size >> 20), (double)cnt / 1048576 * 1000 / cnt_ms); \
-    fflush(stdout); cnt_rep += 1<<26; } } while(0)
-#define P_FINAL do { \
-  cnt_ms += get_timer(&timer); \
-  msg(L_INFO, "Spent %.3f sec (%.2f MB/sec)", (double)cnt_ms/1000, (double)cnt / 1048576 * 1000 / cnt_ms); \
-} while(0)
-
-static struct asio_queue io_queue;
-
-int main(int argc, char **argv)
-{
-  ASSERT(argc == 4);
-  uns files = atol(argv[1]);
-  uns bufsize = atol(argv[2]) * 1024;				// Kbytes
-  u64 total_size = (u64)atol(argv[3]) * 1024*1024*1024;		// Gbytes
-  u64 cnt, cnt_rep;
-  uns cnt_ms;
-  int fd[files];
-  byte name[files][16];
-  struct asio_request *req[files];
-
-  init_timer(&timer);
-
-  io_queue.buffer_size = bufsize;
-  io_queue.max_writebacks = 2;
-  asio_init_queue(&io_queue);
-
-#ifdef COPY
-  msg(L_INFO, "Creating input file");
-  int in_fd = sh_open("tmp/ft-in", O_RDWR | O_CREAT | O_TRUNC | DIRECT, 0666);
-  ASSERT(in_fd >= 0);
-  ASSERT(!(total_size % bufsize));
-  P_INIT;
-  for (uns i=0; i<total_size/bufsize; i++)
-    {
-      struct asio_request *r = asio_get(&io_queue);
-      r->op = ASIO_WRITE_BACK;
-      r->fd = in_fd;
-      r->len = bufsize;
-      byte *xbuf = r->buffer;
-      for (uns j=0; j<bufsize; j++)
-	xbuf[j] = i+j;
-      asio_submit(r);
-      P_UPDATE(bufsize);
-    }
-  asio_sync(&io_queue);
-  lseek(in_fd, 0, SEEK_SET);
-  sync();
-  P_FINAL;
-#endif
-
-  msg(L_INFO, "Initializing output files");
-  for (uns i=0; i<files; i++)
-    {
-      sprintf(name[i], "tmp/ft-%d", i);
-      fd[i] = sh_open(name[i], O_RDWR | O_CREAT | O_TRUNC | DIRECT, 0666);
-      if (fd[i] < 0)
-	die("Cannot create %s: %m", name[i]);
-    }
-  sync();
-  get_timer(&timer);
-
-  msg(L_INFO, "Writing %d MB to %d files in parallel with %d byte buffers", (int)(total_size >> 20), files, bufsize);
-  P_INIT;
-  for (uns i=0; i<files; i++)
-    req[i] = asio_get(&io_queue);
-  for (uns round=0; round<total_size/bufsize/files; round++)
-    {
-      for (uns i=0; i<files; i++)
-	{
-	  struct asio_request *r = req[i];
-#ifdef COPY
-	  struct asio_request *rr, *rd = asio_get(&io_queue);
-	  rd->op = ASIO_READ;
-	  rd->fd = in_fd;
-	  rd->len = bufsize;
-	  asio_submit(rd);
-	  rr = asio_wait(&io_queue);
-	  ASSERT(rr == rd && rd->status == (int)rd->len);
-	  memcpy(r->buffer, rd->buffer, bufsize);
-	  asio_put(rr);
-#else
-	  for (uns j=0; j<bufsize; j++)
-	    r->buffer[j] = round+i+j;
-#endif
-	  r->op = ASIO_WRITE_BACK;
-	  r->fd = fd[i];
-	  r->len = bufsize;
-	  asio_submit(r);
-	  P_UPDATE(bufsize);
-	  req[i] = asio_get(&io_queue);
-	}
-    }
-  for (uns i=0; i<files; i++)
-    asio_put(req[i]);
-  asio_sync(&io_queue);
-#ifdef COPY
-  close(in_fd);
-#endif
-  msg(L_INFO, "Syncing");
-  sync();
-  P_FINAL;
-
-  msg(L_INFO, "Reading the files sequentially");
-  P_INIT;
-  for (uns i=0; i<files; i++)
-    {
-      lseek(fd[i], 0, SEEK_SET);
-      for (uns round=0; round<total_size/bufsize/files; round++)
-	{
-	  struct asio_request *rr, *r = asio_get(&io_queue);
-	  r->op = ASIO_READ;
-	  r->fd = fd[i];
-	  r->len = bufsize;
-	  asio_submit(r);
-	  rr = asio_wait(&io_queue);
-	  ASSERT(rr == r && r->status == (int)bufsize);
-	  asio_put(r);
-	  P_UPDATE(bufsize);
-	}
-      close(fd[i]);
-    }
-  P_FINAL;
-
-  for (uns i=0; i<files; i++)
-    unlink(name[i]);
-#ifdef COPY
-  unlink("tmp/ft-in");
-#endif
-
-  asio_cleanup_queue(&io_queue);
-  msg(L_INFO, "Done");
-  return 0;
-}
diff --git a/debug/sorter/file-test.c b/debug/sorter/file-test.c
deleted file mode 100644
index 430cb30c..00000000
--- a/debug/sorter/file-test.c
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- *  An experiment with parallel reading and writing of files.
- */
-
-#include "lib/lib.h"
-#include "lib/lfs.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <fcntl.h>
-#include <unistd.h>
-
-#define COPY
-#define DIRECT O_DIRECT
-
-static timestamp_t timer;
-
-#define P_INIT do { cnt = 0; cnt_rep = 0; cnt_ms = 1; } while(0)
-#define P_UPDATE(cc) do { \
-  cnt += cc; \
-  if (cnt >= cnt_rep) { cnt_ms += get_timer(&timer); \
-    printf("%d of %d MB (%.2f MB/sec)\r", (int)(cnt >> 20), (int)(total_size >> 20), (double)cnt / 1048576 * 1000 / cnt_ms); \
-    fflush(stdout); cnt_rep += 1<<26; } } while(0)
-#define P_FINAL do { \
-  cnt_ms += get_timer(&timer); \
-  msg(L_INFO, "Spent %.3f sec (%.2f MB/sec)", (double)cnt_ms/1000, (double)cnt / 1048576 * 1000 / cnt_ms); \
-} while(0)
-
-int main(int argc, char **argv)
-{
-  ASSERT(argc == 4);
-  uns files = atol(argv[1]);
-  uns bufsize = atol(argv[2]) * 1024;				// Kbytes
-  u64 total_size = (u64)atol(argv[3]) * 1024*1024*1024;		// Gbytes
-  u64 cnt, cnt_rep;
-  uns cnt_ms;
-  int fd[files];
-  byte *buf[files], name[files][16];
-  uns xbufsize = bufsize;					// Used for single-file I/O
-  byte *xbuf = big_alloc(xbufsize);
-
-  init_timer(&timer);
-
-#ifdef COPY
-  msg(L_INFO, "Creating input file");
-  int in_fd = sh_open("tmp/ft-in", O_RDWR | O_CREAT | O_TRUNC | DIRECT, 0666);
-  ASSERT(in_fd >= 0);
-  ASSERT(!(total_size % xbufsize));
-  P_INIT;
-  for (uns i=0; i<total_size/xbufsize; i++)
-    {
-      for (uns j=0; j<xbufsize; j++)
-	xbuf[j] = i+j;
-      uns c = write(in_fd, xbuf, xbufsize);
-      ASSERT(c == xbufsize);
-      P_UPDATE(c);
-    }
-  lseek(in_fd, 0, SEEK_SET);
-  sync();
-  P_FINAL;
-#endif
-
-  msg(L_INFO, "Initializing output files");
-  for (uns i=0; i<files; i++)
-    {
-      sprintf(name[i], "tmp/ft-%d", i);
-      fd[i] = sh_open(name[i], O_RDWR | O_CREAT | O_TRUNC | DIRECT, 0666);
-      if (fd[i] < 0)
-	die("Cannot create %s: %m", name[i]);
-      buf[i] = big_alloc(bufsize);
-    }
-  sync();
-  get_timer(&timer);
-
-  msg(L_INFO, "Writing %d MB to %d files in parallel with %d byte buffers", (int)(total_size >> 20), files, bufsize);
-  P_INIT;
-  for (uns r=0; r<total_size/bufsize/files; r++)
-    {
-      for (uns i=0; i<files; i++)
-	{
-#ifdef COPY
-	  uns ci = read(in_fd, buf[i], bufsize);
-	  ASSERT(ci == bufsize);
-#else
-	  for (uns j=0; j<bufsize; j++)
-	    buf[i][j] = r+i+j;
-#endif
-	  uns c = write(fd[i], buf[i], bufsize);
-	  ASSERT(c == bufsize);
-	  P_UPDATE(c);
-	}
-    }
-#ifdef COPY
-  close(in_fd);
-#endif
-  msg(L_INFO, "Syncing");
-  sync();
-  P_FINAL;
-
-  msg(L_INFO, "Reading the files sequentially");
-  P_INIT;
-  for (uns i=0; i<files; i++)
-    {
-      lseek(fd[i], 0, SEEK_SET);
-      for (uns r=0; r<total_size/xbufsize/files; r++)
-	{
-	  uns c = read(fd[i], xbuf, xbufsize);
-	  ASSERT(c == xbufsize);
-	  P_UPDATE(c);
-	}
-      close(fd[i]);
-    }
-  P_FINAL;
-
-  for (uns i=0; i<files; i++)
-    unlink(name[i]);
-#ifdef COPY
-  unlink("tmp/ft-in");
-#endif
-  msg(L_INFO, "Done");
-  return 0;
-}
diff --git a/debug/sorter/radix-asio-test.c b/debug/sorter/radix-asio-test.c
new file mode 100644
index 00000000..f2800f87
--- /dev/null
+++ b/debug/sorter/radix-asio-test.c
@@ -0,0 +1,164 @@
+/*
+ *  An experiment with parallel reading and writing of files using ASIO.
+ */
+
+#include "lib/lib.h"
+#include "lib/conf.h"
+#include "lib/lfs.h"
+#include "lib/asio.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#define COPY
+#define DIRECT O_DIRECT
+
+static timestamp_t timer;
+
+#define P_INIT do { cnt = 0; cnt_rep = 0; cnt_ms = 1; } while(0)
+#define P_UPDATE(cc) do { \
+  cnt += cc; \
+  if (cnt >= cnt_rep) { cnt_ms += get_timer(&timer); \
+    printf("%d of %d MB (%.2f MB/sec)\r", (int)(cnt >> 20), (int)(total_size >> 20), (double)cnt / 1048576 * 1000 / cnt_ms); \
+    fflush(stdout); cnt_rep += 1<<26; } } while(0)
+#define P_FINAL do { \
+  cnt_ms += get_timer(&timer); \
+  msg(L_INFO, "Spent %.3f sec (%.2f MB/sec)", (double)cnt_ms/1000, (double)cnt / 1048576 * 1000 / cnt_ms); \
+} while(0)
+
+static struct asio_queue io_queue;
+
+int main(int argc, char **argv)
+{
+  uns files, bufsize;
+  u64 total_size;
+  if (argc != 4 ||
+      cf_parse_int(argv[1], (int*) &files) ||
+      cf_parse_int(argv[2], (int*) &bufsize) ||
+      cf_parse_u64(argv[3], &total_size))
+    {
+      fprintf(stderr, "Usage: asio-test <nr-files> <bufsize> <totalsize>\n");
+      return 1;
+    }
+  u64 cnt, cnt_rep;
+  uns cnt_ms;
+  int fd[files];
+  byte name[files][16];
+  struct asio_request *req[files];
+
+  init_timer(&timer);
+
+  io_queue.buffer_size = bufsize;
+  io_queue.max_writebacks = 2;
+  asio_init_queue(&io_queue);
+
+#ifdef COPY
+  msg(L_INFO, "Creating input file");
+  int in_fd = sh_open("tmp/ft-in", O_RDWR | O_CREAT | O_TRUNC | DIRECT, 0666);
+  ASSERT(in_fd >= 0);
+  ASSERT(!(total_size % bufsize));
+  P_INIT;
+  for (uns i=0; i<total_size/bufsize; i++)
+    {
+      struct asio_request *r = asio_get(&io_queue);
+      r->op = ASIO_WRITE_BACK;
+      r->fd = in_fd;
+      r->len = bufsize;
+      byte *xbuf = r->buffer;
+      for (uns j=0; j<bufsize; j++)
+	xbuf[j] = i+j;
+      asio_submit(r);
+      P_UPDATE(bufsize);
+    }
+  asio_sync(&io_queue);
+  lseek(in_fd, 0, SEEK_SET);
+  sync();
+  P_FINAL;
+#endif
+
+  msg(L_INFO, "Initializing output files");
+  for (uns i=0; i<files; i++)
+    {
+      sprintf(name[i], "tmp/ft-%d", i);
+      fd[i] = sh_open(name[i], O_RDWR | O_CREAT | O_TRUNC | DIRECT, 0666);
+      if (fd[i] < 0)
+	die("Cannot create %s: %m", name[i]);
+    }
+  sync();
+  get_timer(&timer);
+
+  msg(L_INFO, "Writing %d MB to %d files in parallel with %d byte buffers", (int)(total_size >> 20), files, bufsize);
+  P_INIT;
+  for (uns i=0; i<files; i++)
+    req[i] = asio_get(&io_queue);
+  for (uns round=0; round<total_size/bufsize/files; round++)
+    {
+      for (uns i=0; i<files; i++)
+	{
+	  struct asio_request *r = req[i];
+#ifdef COPY
+	  struct asio_request *rr, *rd = asio_get(&io_queue);
+	  rd->op = ASIO_READ;
+	  rd->fd = in_fd;
+	  rd->len = bufsize;
+	  asio_submit(rd);
+	  rr = asio_wait(&io_queue);
+	  ASSERT(rr == rd && rd->status == (int)rd->len);
+	  memcpy(r->buffer, rd->buffer, bufsize);
+	  asio_put(rr);
+#else
+	  for (uns j=0; j<bufsize; j++)
+	    r->buffer[j] = round+i+j;
+#endif
+	  r->op = ASIO_WRITE_BACK;
+	  r->fd = fd[i];
+	  r->len = bufsize;
+	  asio_submit(r);
+	  P_UPDATE(bufsize);
+	  req[i] = asio_get(&io_queue);
+	}
+    }
+  for (uns i=0; i<files; i++)
+    asio_put(req[i]);
+  asio_sync(&io_queue);
+#ifdef COPY
+  close(in_fd);
+#endif
+  msg(L_INFO, "Syncing");
+  sync();
+  P_FINAL;
+
+  msg(L_INFO, "Reading the files sequentially");
+  P_INIT;
+  for (uns i=0; i<files; i++)
+    {
+      lseek(fd[i], 0, SEEK_SET);
+      for (uns round=0; round<total_size/bufsize/files; round++)
+	{
+	  struct asio_request *rr, *r = asio_get(&io_queue);
+	  r->op = ASIO_READ;
+	  r->fd = fd[i];
+	  r->len = bufsize;
+	  asio_submit(r);
+	  rr = asio_wait(&io_queue);
+	  ASSERT(rr == r && r->status == (int)bufsize);
+	  asio_put(r);
+	  P_UPDATE(bufsize);
+	}
+      close(fd[i]);
+    }
+  P_FINAL;
+
+  for (uns i=0; i<files; i++)
+    unlink(name[i]);
+#ifdef COPY
+  unlink("tmp/ft-in");
+#endif
+
+  asio_cleanup_queue(&io_queue);
+  msg(L_INFO, "Done");
+  return 0;
+}
diff --git a/debug/sorter/radix-file-test.c b/debug/sorter/radix-file-test.c
new file mode 100644
index 00000000..b776d1ea
--- /dev/null
+++ b/debug/sorter/radix-file-test.c
@@ -0,0 +1,129 @@
+/*
+ *  An experiment with parallel reading and writing of files.
+ */
+
+#include "lib/lib.h"
+#include "lib/conf.h"
+#include "lib/lfs.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#define COPY
+#define DIRECT 0		// or O_DIRECT
+
+static timestamp_t timer;
+
+#define P_INIT do { cnt = 0; cnt_rep = 0; cnt_ms = 1; } while(0)
+#define P_UPDATE(cc) do { \
+  cnt += cc; \
+  if (cnt >= cnt_rep) { cnt_ms += get_timer(&timer); \
+    printf("%d of %d MB (%.2f MB/sec)\r", (int)(cnt >> 20), (int)(total_size >> 20), (double)cnt / 1048576 * 1000 / cnt_ms); \
+    fflush(stdout); cnt_rep += 1<<26; } } while(0)
+#define P_FINAL do { \
+  cnt_ms += get_timer(&timer); \
+  msg(L_INFO, "Spent %.3f sec (%.2f MB/sec)", (double)cnt_ms/1000, (double)cnt / 1048576 * 1000 / cnt_ms); \
+} while(0)
+
+int main(int argc, char **argv)
+{
+  uns files, bufsize;
+  u64 total_size;
+  if (argc != 4 ||
+      cf_parse_int(argv[1], (int*) &files) ||
+      cf_parse_int(argv[2], (int*) &bufsize) ||
+      cf_parse_u64(argv[3], &total_size))
+    {
+      fprintf(stderr, "Usage: file-test <nr-files> <bufsize> <totalsize>\n");
+      return 1;
+    }
+  u64 cnt, cnt_rep;
+  uns cnt_ms;
+  int fd[files];
+  byte *buf[files], name[files][16];
+  uns xbufsize = bufsize;					// Used for single-file I/O
+  byte *xbuf = big_alloc(xbufsize);
+
+  init_timer(&timer);
+
+#ifdef COPY
+  msg(L_INFO, "Creating input file");
+  int in_fd = sh_open("tmp/ft-in", O_RDWR | O_CREAT | O_TRUNC | DIRECT, 0666);
+  ASSERT(in_fd >= 0);
+  ASSERT(!(total_size % xbufsize));
+  P_INIT;
+  for (uns i=0; i<total_size/xbufsize; i++)
+    {
+      for (uns j=0; j<xbufsize; j++)
+	xbuf[j] = i+j;
+      uns c = write(in_fd, xbuf, xbufsize);
+      ASSERT(c == xbufsize);
+      P_UPDATE(c);
+    }
+  lseek(in_fd, 0, SEEK_SET);
+  sync();
+  P_FINAL;
+#endif
+
+  msg(L_INFO, "Initializing output files");
+  for (uns i=0; i<files; i++)
+    {
+      sprintf(name[i], "tmp/ft-%d", i);
+      fd[i] = sh_open(name[i], O_RDWR | O_CREAT | O_TRUNC | DIRECT, 0666);
+      if (fd[i] < 0)
+	die("Cannot create %s: %m", name[i]);
+      buf[i] = big_alloc(bufsize);
+    }
+  sync();
+  get_timer(&timer);
+
+  msg(L_INFO, "Writing %d MB to %d files in parallel with %d byte buffers", (int)(total_size >> 20), files, bufsize);
+  P_INIT;
+  for (uns r=0; r<total_size/bufsize/files; r++)
+    {
+      for (uns i=0; i<files; i++)
+	{
+#ifdef COPY
+	  uns ci = read(in_fd, buf[i], bufsize);
+	  ASSERT(ci == bufsize);
+#else
+	  for (uns j=0; j<bufsize; j++)
+	    buf[i][j] = r+i+j;
+#endif
+	  uns c = write(fd[i], buf[i], bufsize);
+	  ASSERT(c == bufsize);
+	  P_UPDATE(c);
+	}
+    }
+#ifdef COPY
+  close(in_fd);
+#endif
+  msg(L_INFO, "Syncing");
+  sync();
+  P_FINAL;
+
+  msg(L_INFO, "Reading the files sequentially");
+  P_INIT;
+  for (uns i=0; i<files; i++)
+    {
+      lseek(fd[i], 0, SEEK_SET);
+      for (uns r=0; r<total_size/xbufsize/files; r++)
+	{
+	  uns c = read(fd[i], xbuf, xbufsize);
+	  ASSERT(c == xbufsize);
+	  P_UPDATE(c);
+	}
+      close(fd[i]);
+    }
+  P_FINAL;
+
+  for (uns i=0; i<files; i++)
+    unlink(name[i]);
+#ifdef COPY
+  unlink("tmp/ft-in");
+#endif
+  msg(L_INFO, "Done");
+  return 0;
+}