Use a different file access method for small inputs.

diff --git a/lib/sorter/TODO b/lib/sorter/TODO
index 250aa9fb98503fbeb08751f61588b96c26d2c816..79a60204b09ea716c0f99192322586bc62d10d03 100644 (file)
--- a/lib/sorter/TODO
+++ b/lib/sorter/TODO
@@ -11,10 +11,6 @@ o  Log messages should show both original and new size of the data. The speed
    should be probably calculated from the former.
 o  Buffer sizing in shep-export.
 o  Problems with thread stack limit in radix-sorting of arrays.
-o  Prediction of destination buffer in array radix-sorts.
-
-Improvements:
-o  Switching between direct and normal I/O. Should use normal I/O if the input is small enough.
 
 Users of lib/sorter/array.h which might use radix-sorting:
 indexer/chewer.c

diff --git a/lib/sorter/common.h b/lib/sorter/common.h
index a97b4dd2feab25a992ea147a5d806b1067a81958..92423046d4f62bf7dbfc0ef61c17b1e6220ec97d 100644 (file)
--- a/lib/sorter/common.h
+++ b/lib/sorter/common.h
@@ -18,8 +18,8 @@ extern uns sorter_debug, sorter_min_radix_bits, sorter_max_radix_bits, sorter_ad
 extern uns sorter_min_multiway_bits, sorter_max_multiway_bits;
 extern uns sorter_threads, sorter_thread_threshold, sorter_thread_chunk;
 extern uns sorter_radix_threshold;
-extern u64 sorter_bufsize;
-extern struct fb_params sorter_fb_params;
+extern u64 sorter_bufsize, sorter_small_input;
+extern struct fb_params sorter_fb_params, sorter_small_fb_params;
 
 #define SORT_TRACE(x...) do { if (sorter_trace) msg(L_DEBUG, x); } while(0)
 #define SORT_XTRACE(level, x...) do { if (sorter_trace >= level) msg(L_DEBUG, x); } while(0)
@@ -41,6 +41,7 @@ struct sort_context {
   struct fastbuf *out_fb;
   uns hash_bits;
   u64 in_size;
+  struct fb_params *fb_params;
 
   struct mempool *pool;
   clist bucket_list;

diff --git a/lib/sorter/config.c b/lib/sorter/config.c
index d12a65bd8c81cd56ee6aeedac75a35f16c68d14c..e0927ba876edcb70462fe7a028a2cc1557d9af29 100644 (file)
--- a/lib/sorter/config.c
+++ b/lib/sorter/config.c
@@ -13,7 +13,6 @@
 #include "lib/sorter/common.h"
 
 uns sorter_trace;
-uns sorter_stream_bufsize = 65536;
 u64 sorter_bufsize = 65536;
 uns sorter_debug;
 uns sorter_min_radix_bits;
@@ -26,12 +25,15 @@ uns sorter_thread_threshold = 1048576;
 uns sorter_thread_chunk = 4096;
 uns sorter_radix_threshold = 4096;
 struct fb_params sorter_fb_params;
+struct fb_params sorter_small_fb_params;
+u64 sorter_small_input;
 
 static struct cf_section sorter_config = {
   CF_ITEMS {
     CF_UNS("Trace", &sorter_trace),
-    CF_UNS("StreamBuffer", &sorter_stream_bufsize),
     CF_SECTION("FileAccess", &sorter_fb_params, &fbpar_cf),
+    CF_SECTION("SmallFileAccess", &sorter_fb_params, &fbpar_cf),
+    CF_U64("SmallInput", &sorter_small_input),
     CF_U64("SortBuffer", &sorter_bufsize),
     CF_UNS("Debug", &sorter_debug),
     CF_UNS("MinRadixBits", &sorter_min_radix_bits),

diff --git a/lib/sorter/govern.c b/lib/sorter/govern.c
index 619b351d4690db9d4d4ad124d1c1afc7c8f81943..5ce884732a4209a44c962890c0df3de90d594804 100644 (file)
--- a/lib/sorter/govern.c
+++ b/lib/sorter/govern.c
@@ -413,6 +413,7 @@ sorter_run(struct sort_context *ctx)
   bin->hash_bits = ctx->hash_bits;
   clist_add_tail(&ctx->bucket_list, &bin->n);
   SORT_XTRACE(2, "Input size: %s, %d hash bits", F_BSIZE(bin), bin->hash_bits);
+  ctx->fb_params = (bin->size < sorter_small_input) ? &sorter_small_fb_params : &sorter_fb_params;
 
   // Create bucket for the output
   struct sort_bucket *bout = sbuck_new(ctx);

diff --git a/lib/sorter/sbuck.c b/lib/sorter/sbuck.c
index bff52d6616ca9b0bfec6cd1908da4374984b4f71..1150c9d6733a663df18104b39991625d16b52aad 100644 (file)
--- a/lib/sorter/sbuck.c
+++ b/lib/sorter/sbuck.c
@@ -69,7 +69,7 @@ sbuck_swap_in(struct sort_bucket *b)
 {
   if (b->flags & SBF_SWAPPED_OUT)
     {
-      b->fb = bopen_file(b->filename, O_RDWR, &sorter_fb_params);      /* FIXME: Something different for small buckets? */
+      b->fb = bopen_file(b->filename, O_RDWR, b->ctx->fb_params);
       if (b->flags & SBF_OPEN_WRITE)
        bseek(b->fb, 0, SEEK_END);
       bconfig(b->fb, BCONFIG_IS_TEMP_FILE, 1);
@@ -104,7 +104,7 @@ sbuck_write(struct sort_bucket *b)
   else
     {
       ASSERT(!(b->flags & (SBF_OPEN_READ | SBF_DESTROYED)));
-      b->fb = bopen_tmp_file(&sorter_fb_params);
+      b->fb = bopen_tmp_file(b->ctx->fb_params);
       if (sorter_debug & SORT_DEBUG_KEEP_BUCKETS)
        bconfig(b->fb, BCONFIG_IS_TEMP_FILE, 0);
       b->flags |= SBF_OPEN_WRITE;