Empty final bucket should be turned into a file as well.

[libucw.git] / lib / sorter / govern.c

diff --git a/lib/sorter/govern.c b/lib/sorter/govern.c
index f962810b3ba0ce72a47d8d63c4f3f8b4b54329eb..3606163c557cb827542a9c378a11d0e680498d45 100644 (file)
--- a/lib/sorter/govern.c
+++ b/lib/sorter/govern.c
@@ -12,6 +12,8 @@
 #include "lib/mempool.h"
 #include "lib/sorter/common.h"
 
+#include <fcntl.h>
+
 void *
 sorter_alloc(struct sort_context *ctx, uns size)
 {
@@ -21,7 +23,9 @@ sorter_alloc(struct sort_context *ctx, uns size)
 struct sort_bucket *
 sbuck_new(struct sort_context *ctx)
 {
-  return sorter_alloc(ctx, sizeof(struct sort_bucket));
+  struct sort_bucket *b = sorter_alloc(ctx, sizeof(struct sort_bucket));
+  b->ctx = ctx;
+  return b;
 }
 
 void
@@ -41,7 +45,7 @@ sbuck_drop(struct sort_bucket *b)
 sh_off_t
 sbuck_size(struct sort_bucket *b)
 {
-  if (b->flags & SBF_OPEN_WRITE)
+  if ((b->flags & SBF_OPEN_WRITE) && !(b->flags & SBF_SWAPPED_OUT))
     return btell(b->fb);
   else
     return b->size;
@@ -53,11 +57,30 @@ sbuck_have(struct sort_bucket *b)
   return b && sbuck_size(b);
 }
 
+static int
+sbuck_has_file(struct sort_bucket *b)
+{
+  return (b->fb || (b->flags & SBF_SWAPPED_OUT));
+}
+
+static void
+sbuck_swap_in(struct sort_bucket *b)
+{
+  if (b->flags & SBF_SWAPPED_OUT)
+    {
+      b->fb = bopen(b->filename, O_RDWR, sorter_stream_bufsize);
+      if (b->flags & SBF_OPEN_WRITE)
+       bseek(b->fb, 0, SEEK_END);
+      bconfig(b->fb, BCONFIG_IS_TEMP_FILE, 1);
+      b->flags &= ~SBF_SWAPPED_OUT;
+      SORT_XTRACE("Swapped in %s", b->filename);
+    }
+}
+
 struct fastbuf *
 sbuck_read(struct sort_bucket *b)
 {
-  /* FIXME: These functions should handle buckets with no fb and only name. */
-  ASSERT(b->fb);
+  sbuck_swap_in(b);
   if (b->flags & SBF_OPEN_READ)
     return b->fb;
   else if (b->flags & SBF_OPEN_WRITE)
@@ -74,6 +97,7 @@ sbuck_read(struct sort_bucket *b)
 struct fastbuf *
 sbuck_write(struct sort_bucket *b)
 {
+  sbuck_swap_in(b);
   if (b->flags & SBF_OPEN_WRITE)
     ASSERT(b->fb);
   else
@@ -81,10 +105,26 @@ sbuck_write(struct sort_bucket *b)
       ASSERT(!(b->flags & (SBF_OPEN_READ | SBF_DESTROYED)));
       b->fb = bopen_tmp(sorter_stream_bufsize);
       b->flags |= SBF_OPEN_WRITE;
+      b->filename = mp_strdup(b->ctx->pool, b->fb->name);
     }
   return b->fb;
 }
 
+void
+sbuck_swap_out(struct sort_bucket *b)
+{
+  if ((b->flags & (SBF_OPEN_READ | SBF_OPEN_WRITE)) && b->fb)
+    {
+      if (b->flags & SBF_OPEN_WRITE)
+       b->size = btell(b->fb);
+      bconfig(b->fb, BCONFIG_IS_TEMP_FILE, 0);
+      bclose(b->fb);
+      b->fb = NULL;
+      b->flags |= SBF_SWAPPED_OUT;
+      SORT_XTRACE("Swapped out %s", b->filename);
+    }
+}
+
 void
 sorter_alloc_buf(struct sort_context *ctx)
 {
@@ -124,6 +164,9 @@ static int sorter_presort(struct sort_context *ctx, struct sort_bucket *in, stru
 static inline struct sort_bucket *
 sbuck_join_to(struct sort_bucket *b)
 {
+  if (sorter_debug & SORT_DEBUG_NO_JOIN)
+    return NULL;
+
   struct sort_bucket *out = (struct sort_bucket *) b->n.prev;  // Such bucket is guaranteed to exist
   return (out->flags & SBF_FINAL) ? out : NULL;
 }
@@ -131,16 +174,26 @@ sbuck_join_to(struct sort_bucket *b)
 static void
 sorter_join(struct sort_bucket *b)
 {
-  struct sort_bucket *join = sbuck_join_to(b);
-  ASSERT(join);
-
-  // FIXME: What if the final bucket doesn't contain any file yet?
+  struct sort_bucket *join = (struct sort_bucket *) b->n.prev;
+  ASSERT(join->flags & SBF_FINAL);
+  ASSERT(b->runs == 1);
 
-  SORT_TRACE("Copying %jd bytes to output file", (uintmax_t) sbuck_size(b));
-  struct fastbuf *src = sbuck_read(b);
-  struct fastbuf *dest = sbuck_write(join);
-  bbcopy(src, dest, ~0U);
-  sbuck_drop(b);
+  if (!sbuck_has_file(join))
+    {
+      // The final bucket doesn't have any file associated yet, so replace
+      // it with the new bucket.
+      SORT_XTRACE("Replaced final bucket");
+      b->flags |= SBF_FINAL;
+      sbuck_drop(join);
+    }
+  else
+    {
+      SORT_TRACE("Copying %jd bytes to output file", (uintmax_t) sbuck_size(b));
+      struct fastbuf *src = sbuck_read(b);
+      struct fastbuf *dest = sbuck_write(join);
+      bbcopy(src, dest, ~0U);
+      sbuck_drop(b);
+    }
 }
 
 static void
@@ -149,16 +202,14 @@ sorter_twoway(struct sort_context *ctx, struct sort_bucket *b)
   struct sort_bucket *ins[3] = { NULL }, *outs[3] = { NULL };
   cnode *list_pos = b->n.prev;
   struct sort_bucket *join = sbuck_join_to(b);
-  if (sorter_debug & SORT_DEBUG_NO_JOIN)
-    join = NULL;
 
   if (!(sorter_debug & SORT_DEBUG_NO_PRESORT) || (b->flags & SBF_CUSTOM_PRESORT))
     {
       SORT_TRACE("Presorting");
       ins[0] = sbuck_new(ctx);
-      sbuck_read(b);
       if (!sorter_presort(ctx, b, ins[0], join ? : ins[0]))
        {
+         SORT_XTRACE("Sorted in memory");
          if (join)
            sbuck_drop(ins[0]);
          else
@@ -238,7 +289,7 @@ sorter_run(struct sort_context *ctx)
   clist_add_head(&ctx->bucket_list, &bout->n);
 
   struct sort_bucket *b;
-  while (b = clist_next(&ctx->bucket_list, &bout->n))
+  while (bout = clist_head(&ctx->bucket_list), b = clist_next(&ctx->bucket_list, &bout->n))
     {
       if (!sbuck_have(b))
        sbuck_drop(b);
@@ -249,6 +300,7 @@ sorter_run(struct sort_context *ctx)
     }
 
   sorter_free_buf(ctx);
+  sbuck_write(bout);           // Force empty bucket to a file
   SORT_XTRACE("Final size: %jd", (uintmax_t) sbuck_size(bout));
   ctx->out_fb = sbuck_read(bout);
 }