Added a trivial implementation of radix sorting.

diff --git a/lib/sorter/common.h b/lib/sorter/common.h
index bd5ed9653f26b4d9f73e8bdfbf872cad831dc049..52949657ad411c2face099d6d3b4fd0b4ea85d36 100644 (file)
--- a/lib/sorter/common.h
+++ b/lib/sorter/common.h
@@ -24,6 +24,7 @@ enum sort_debug {
   SORT_DEBUG_NO_PRESORT = 1,
   SORT_DEBUG_NO_JOIN = 2,
   SORT_DEBUG_KEEP_BUCKETS = 4,
+  SORT_DEBUG_NO_RADIX = 8,
 };
 
 struct sort_bucket;
@@ -46,6 +47,8 @@ struct sort_context {
   // Two-way split/merge: merge up to 2 source buckets to up to 2 destination buckets.
   // Bucket arrays are NULL-terminated.
   void (*twoway_merge)(struct sort_context *ctx, struct sort_bucket **ins, struct sort_bucket **outs);
+  // Radix split according to hash function
+  void (*radix_split)(struct sort_context *ctx, struct sort_bucket *in, struct sort_bucket **outs, uns bitpos, uns numbits);
 
   // State variables of internal_sort
   void *key_buf;

diff --git a/lib/sorter/govern.c b/lib/sorter/govern.c
index 98b9e9d45da0486c0745cabf9d2074f8833114a3..9b8ee774ef6f345531aa21197cf2bd31d6307117 100644 (file)
--- a/lib/sorter/govern.c
+++ b/lib/sorter/govern.c
@@ -103,7 +103,7 @@ sorter_twoway(struct sort_context *ctx, struct sort_bucket *b)
       ins[0] = sbuck_new(ctx);
       if (!sorter_presort(ctx, b, ins[0], join ? : ins[0]))
        {
-         SORT_TRACE("Sorted in memory");
+         SORT_XTRACE(((b->flags & SBF_SOURCE) ? 1 : 2), "Sorted in memory");
          if (join)
            sbuck_drop(ins[0]);
          else
@@ -163,6 +163,46 @@ sorter_twoway(struct sort_context *ctx, struct sort_bucket *b)
   clist_insert_after(&ins[0]->n, list_pos);
 }
 
+static int
+sorter_radix_p(struct sort_context *ctx, struct sort_bucket *b)
+{
+  return b->hash_bits && ctx->radix_split &&
+    !(sorter_debug & SORT_DEBUG_NO_RADIX) &&
+    sbuck_size(b) > (sh_off_t)sorter_bufsize;
+}
+
+static void
+sorter_radix(struct sort_context *ctx, struct sort_bucket *b)
+{
+  uns bits = MIN(b->hash_bits, 4);     /* FIXME */
+  uns nbuck = 1 << bits;
+  SORT_XTRACE(2, "Running radix sort on %s with %d bits of %d", F_BSIZE(b), bits, b->hash_bits);
+  sorter_start_timer(ctx);
+
+  struct sort_bucket *outs[nbuck];
+  for (uns i=nbuck; i--; )
+    {
+      outs[i] = sbuck_new(ctx);
+      outs[i]->hash_bits = b->hash_bits - bits;
+      clist_insert_after(&outs[i]->n, &b->n);
+    }
+
+  ctx->radix_split(ctx, b, outs, b->hash_bits - bits, bits);
+
+  u64 min = ~0U, max = 0, sum = 0;
+  for (uns i=0; i<nbuck; i++)
+    {
+      u64 s = sbuck_size(outs[i]);
+      min = MIN(min, s);
+      max = MAX(max, s);
+      sum += s;
+    }
+
+  SORT_TRACE("Radix split (%d buckets, %s min, %s max, %s avg, %dMB/s)", nbuck,
+            F_SIZE(min), F_SIZE(max), F_SIZE(sum / nbuck), sorter_speed(ctx, sum));
+  sbuck_drop(b);
+}
+
 void
 sorter_run(struct sort_context *ctx)
 {
@@ -179,7 +219,7 @@ sorter_run(struct sort_context *ctx)
   else
     bin->fb = ctx->in_fb;
   bin->ident = "in";
-  bin->size = ctx->in_size;
+  bin->size = ctx->in_size;            /* Sizes should be either sh_off_t or u64, not both; beware of ~0U */
   bin->hash_bits = ctx->hash_bits;
   clist_add_tail(&ctx->bucket_list, &bin->n);
   SORT_XTRACE(2, "Input size: %s", (ctx->in_size == ~(u64)0 ? (byte*)"unknown" : F_BSIZE(bin)));
@@ -200,6 +240,8 @@ sorter_run(struct sort_context *ctx)
        sbuck_drop(b);
       else if (b->runs == 1)
        sorter_join(b);
+      else if (sorter_radix_p(ctx, b))
+       sorter_radix(ctx, b);
       else
        sorter_twoway(ctx, b);
     }

diff --git a/lib/sorter/s-radix.h b/lib/sorter/s-radix.h
new file mode 100644 (file)
index 0000000..c83d604
--- /dev/null
+++ b/lib/sorter/s-radix.h
@@ -0,0 +1,32 @@
+/*
+ *     UCW Library -- Universal Sorter: Radix-Split Module
+ *
+ *     (c) 2007 Martin Mares <mj@ucw.cz>
+ *
+ *     This software may be freely distributed and used according to the terms
+ *     of the GNU Lesser General Public License.
+ */
+
+/* FIXME: This is a very trivial implementation so far. Use fbdirect and such things to speed up. */
+
+#include <string.h>
+
+static void P(radix_split)(struct sort_context *ctx UNUSED, struct sort_bucket *bin, struct sort_bucket **bouts, uns bitpos, uns numbits)
+{
+  uns nbucks = 1 << numbits;
+  uns mask = nbucks - 1;
+  struct fastbuf *in = sbuck_read(bin);
+  P(key) k;
+
+  struct fastbuf *outs[nbucks];
+  bzero(outs, sizeof(outs));
+
+  while (P(read_key)(in, &k))
+    {
+      uns h = P(hash)(&k);
+      uns i = (h >> bitpos) & mask;
+      if (unlikely(!outs[i]))
+       outs[i] = sbuck_write(bouts[i]);
+      P(copy_data)(&k, in, outs[i]);
+    }
+}

diff --git a/lib/sorter/sort-test.c b/lib/sorter/sort-test.c
index 2909599f838d64e245c4126283e7aba70a7331c9..5464f744fd1eb226d2af7df4a82d8b43805021fb 100644 (file)
--- a/lib/sorter/sort-test.c
+++ b/lib/sorter/sort-test.c
@@ -18,18 +18,21 @@
 #include <stdio.h>
 #include <string.h>
 #include <fcntl.h>
+#include <unistd.h>
 
 /*** Time measurement ***/
 
 static void
 start(void)
 {
+  sync();
   init_timer();
 }
 
 static void
 stop(void)
 {
+  sync();
   log(L_INFO, "Test took %.3fs", get_timer() / 1000.);
 }
 

diff --git a/lib/sorter/sorter.h b/lib/sorter/sorter.h
index d9ae7a00db4572bbd09d92810e53d7c6458b1856..98dd8f776e3aa0bacc0e4423217c26a440ec328a 100644 (file)
--- a/lib/sorter/sorter.h
+++ b/lib/sorter/sorter.h
@@ -52,7 +52,7 @@
  *  SORT_HASH_BITS     signals that a monotone hashing function returning a given number of
  *                     bits is available. Monotone hash is a function f such that f(x) < f(y)
  *                     implies x < y and which is approximately uniformly distributed.
- *  uns PREFIX_hash(SORT_KEY *a, SORT_KEY *b)
+ *  uns PREFIX_hash(SORT_KEY *a)
  *
  *  Unification:
  *
@@ -175,6 +175,10 @@ static inline void P(copy_data)(P(key) *key, struct fastbuf *in, struct fastbuf
 #include "lib/sorter/s-internal.h"
 #include "lib/sorter/s-twoway.h"
 
+#if defined(SORT_HASH_BITS) || defined(SORT_INT)
+#include "lib/sorter/s-radix.h"
+#endif
+
 static struct fastbuf *P(sort)(
 #ifdef SORT_INPUT_FILE
                               byte *in,
@@ -223,10 +227,12 @@ static struct fastbuf *P(sort)(
 
 #ifdef SORT_HASH_BITS
   ctx.hash_bits = SORT_HASH_BITS;
+  ctx.radix_split = P(radix_split);
 #elif defined(SORT_INT)
   ctx.hash_bits = 0;
   while (ctx.hash_bits < 32 && (int_range >> ctx.hash_bits))
     ctx.hash_bits++;
+  ctx.radix_split = P(radix_split);
 #endif
 
   ctx.internal_sort = P(internal);