format_size() no longer exists.

[libucw.git] / lib / sorter / sorter.h

diff --git a/lib/sorter/sorter.h b/lib/sorter/sorter.h
index cd5e843f84bd0930d11e18ba7879452ca1fd83e6..a0b6b39b76a1407fa7425f300bd0551b75cbac76 100644 (file)
--- a/lib/sorter/sorter.h
+++ b/lib/sorter/sorter.h
@@ -46,6 +46,7 @@
  *                     is supplied automatically and the sorting function gets an extra
  *                     parameter specifying a range of the integers. The better the range
  *                     fits, the faster we sort. Sets up SORT_HASH_xxx automatically.
  *                     is supplied automatically and the sorting function gets an extra
  *                     parameter specifying a range of the integers. The better the range
  *                     fits, the faster we sort. Sets up SORT_HASH_xxx automatically.

+ *  SORT_INT64(key)    the same for 64-bit integers.

  *
  *  Hashing (optional, but it can speed sorting up):
  *
  *
  *  Hashing (optional, but it can speed sorting up):
  *


@@ -61,11 +62,12 @@
  *                     takes n records in memory with keys which compare equal and writes
  *                     a single record to the given fastbuf. `buf' points to a buffer which
  *                     is guaranteed to hold the sum of workspace requirements (see below)
  *                     takes n records in memory with keys which compare equal and writes
  *                     a single record to the given fastbuf. `buf' points to a buffer which
  *                     is guaranteed to hold the sum of workspace requirements (see below)

- *                     over all given records.
+ *                     over all given records. The function is allowed to modify all its inputs.

  *  void PREFIX_copy_merged(SORT_KEY **keys, struct fastbuf **data, uns n, struct fastbuf *dest)
  *                     takes n records with keys in memory and data in fastbufs and writes
  *                     a single record. Used only if SORT_DATA_SIZE or SORT_UNIFY_WORKSPACE is defined.
  *  void PREFIX_copy_merged(SORT_KEY **keys, struct fastbuf **data, uns n, struct fastbuf *dest)
  *                     takes n records with keys in memory and data in fastbufs and writes
  *                     a single record. Used only if SORT_DATA_SIZE or SORT_UNIFY_WORKSPACE is defined.

- *  SORT_UNIFY_WORKSPACE(key)  gets a key and returns the amount of workspace required when merging
+ *  SORT_UNIFY_WORKSPACE(key)
+ *                     gets a key and returns the amount of workspace required when merging

  *                     the given record. Defaults to 0.
  *
  *  Input (choose one of these):
  *                     the given record. Defaults to 0.
  *
  *  Input (choose one of these):


@@ -74,7 +76,7 @@
  *  SORT_INPUT_FB      seekable fastbuf stream
  *  SORT_INPUT_PIPE    non-seekable fastbuf stream
  *  SORT_INPUT_PRESORT custom presorter. Calls function
  *  SORT_INPUT_FB      seekable fastbuf stream
  *  SORT_INPUT_PIPE    non-seekable fastbuf stream
  *  SORT_INPUT_PRESORT custom presorter. Calls function

- *  int PREFIX_presort(struct fastbuf *dest, void *buf, size_t bufsize);
+ *  int PREFIX_presort(struct fastbuf *dest, void *buf, size_t bufsize)

  *                     to get successive batches of pre-sorted data.
  *                     The function is passed a page-aligned presorting buffer.
  *                     It returns 1 on success or 0 on EOF.
  *                     to get successive batches of pre-sorted data.
  *                     The function is passed a page-aligned presorting buffer.
  *                     It returns 1 on success or 0 on EOF.


@@ -126,6 +128,13 @@ typedef SORT_KEY P(key);
 #error Missing definition of sorting key.
 #endif
 
 #error Missing definition of sorting key.
 #endif
 

+#ifdef SORT_INT64
+typedef u64 P(hash_t);
+#define SORT_INT SORT_INT64
+#else
+typedef uns P(hash_t);
+#endif
+

 #ifdef SORT_INT
 static inline int P(compare) (P(key) *x, P(key) *y)
 {
 #ifdef SORT_INT
 static inline int P(compare) (P(key) *x, P(key) *y)
 {


@@ -137,7 +146,7 @@ static inline int P(compare) (P(key) *x, P(key) *y)
 }
 
 #ifndef SORT_HASH_BITS
 }
 
 #ifndef SORT_HASH_BITS

-static inline int P(hash) (P(key) *x)
+static inline P(hash_t) P(hash) (P(key) *x)

 {
   return SORT_INT((*x));
 }
 {
   return SORT_INT((*x));
 }


@@ -192,6 +201,10 @@ static inline void P(copy_merged)(P(key) **keys, struct fastbuf **data UNUSED, u
 
 #include "lib/sorter/s-twoway.h"
 
 
 #include "lib/sorter/s-twoway.h"
 

+#ifndef SORT_UNIFY
+#include "lib/sorter/s-multiway.h"
+#endif
+

 #if defined(SORT_HASH_BITS) || defined(SORT_INT)
 #include "lib/sorter/s-radix.h"
 #endif
 #if defined(SORT_HASH_BITS) || defined(SORT_INT)
 #include "lib/sorter/s-radix.h"
 #endif


@@ -208,7 +221,7 @@ static struct fastbuf *P(sort)(
                               struct fastbuf *out
 #endif
 #ifdef SORT_INT
                               struct fastbuf *out
 #endif
 #ifdef SORT_INT

-                              , uns int_range
+                              , u64 int_range

 #endif
                               )
 {
 #endif
                               )
 {


@@ -216,7 +229,7 @@ static struct fastbuf *P(sort)(
   bzero(&ctx, sizeof(ctx));
 
 #ifdef SORT_INPUT_FILE
   bzero(&ctx, sizeof(ctx));
 
 #ifdef SORT_INPUT_FILE

-  ctx.in_fb = bopen(in, O_RDONLY, sorter_stream_bufsize);
+  ctx.in_fb = bopen_file(in, O_RDONLY, &sorter_fb_params);

   ctx.in_size = bfilesize(ctx.in_fb);
 #elif defined(SORT_INPUT_FB)
   ctx.in_fb = in;
   ctx.in_size = bfilesize(ctx.in_fb);
 #elif defined(SORT_INPUT_FB)
   ctx.in_fb = in;


@@ -251,7 +264,7 @@ static struct fastbuf *P(sort)(
   ctx.radix_split = P(radix_split);
 #elif defined(SORT_INT)
   ctx.hash_bits = 0;
   ctx.radix_split = P(radix_split);
 #elif defined(SORT_INT)
   ctx.hash_bits = 0;

-  while (ctx.hash_bits < 32 && (int_range >> ctx.hash_bits))
+  while (ctx.hash_bits < 64 && (int_range >> ctx.hash_bits))

     ctx.hash_bits++;
   ctx.radix_split = P(radix_split);
 #endif
     ctx.hash_bits++;
   ctx.radix_split = P(radix_split);
 #endif


@@ -260,13 +273,14 @@ static struct fastbuf *P(sort)(
   ctx.internal_estimate = P(internal_estimate);
   ctx.twoway_merge = P(twoway_merge);
 
   ctx.internal_estimate = P(internal_estimate);
   ctx.twoway_merge = P(twoway_merge);
 

+#ifndef SORT_UNIFY
+  ctx.multiway_merge = P(multiway_merge);
+#endif
+

   sorter_run(&ctx);
 
 #ifdef SORT_OUTPUT_FILE
   sorter_run(&ctx);
 
 #ifdef SORT_OUTPUT_FILE

-  if (rename(ctx.out_fb->name, out) < 0)
-    die("Cannot rename %s to %s: %m", ctx.out_fb->name, out);
-  bconfig(ctx.out_fb, BCONFIG_IS_TEMP_FILE, 0);
-  bclose(ctx.out_fb);
+  bfix_tmp_file(ctx.out_fb, out);

   ctx.out_fb = NULL;
 #endif
   return ctx.out_fb;
   ctx.out_fb = NULL;
 #endif
   return ctx.out_fb;


@@ -280,6 +294,7 @@ static struct fastbuf *P(sort)(
 #undef SORT_VAR_KEY
 #undef SORT_VAR_DATA
 #undef SORT_INT
 #undef SORT_VAR_KEY
 #undef SORT_VAR_DATA
 #undef SORT_INT

+#undef SORT_INT64

 #undef SORT_HASH_BITS
 #undef SORT_UNIFY
 #undef SORT_UNIFY_WORKSPACE
 #undef SORT_HASH_BITS
 #undef SORT_UNIFY
 #undef SORT_UNIFY_WORKSPACE