Added very simple functions for emulating a fastbuf stream over a static

[libucw.git] / lib / bucket.c

diff --git a/lib/bucket.c b/lib/bucket.c
index db00678a3a8f58ca39616cb6e8d89984817ce4da..af6b46bf1ffcd6a40ca9624ea481bfa97d548943 100644 (file)
--- a/lib/bucket.c
+++ b/lib/bucket.c
@@ -1,7 +1,7 @@
 /*
  *     Sherlock Library -- Object Buckets
  *
- *     (c) 2001--2002 Martin Mares <mj@ucw.cz>
+ *     (c) 2001--2003 Martin Mares <mj@ucw.cz>
  *
  *     This software may be freely distributed and used according to the terms
  *     of the GNU Lesser General Public License.
@@ -30,12 +30,14 @@ static sh_off_t bucket_start, bucket_current;
 byte *obuck_name = "not/configured";
 static uns obuck_io_buflen = 65536;
 static int obuck_shake_buflen = 1048576;
+static uns obuck_slurp_buflen = 65536;
 
 static struct cfitem obuck_config[] = {
   { "Buckets",         CT_SECTION,     NULL },
   { "BucketFile",      CT_STRING,      &obuck_name },
   { "BufSize",         CT_INT,         &obuck_io_buflen },
   { "ShakeBufSize",    CT_INT,         &obuck_shake_buflen },
+  { "SlurpBufSize",    CT_INT,         &obuck_slurp_buflen },
   { NULL,              CT_STOP,        NULL }
 };
 
@@ -270,8 +272,15 @@ obuck_fetch_end(struct fastbuf *b UNUSED)
 {
 }
 
+oid_t
+obuck_predict_last_oid(void)
+{
+  sh_off_t size = sh_seek(obuck_fd, 0, SEEK_END);
+  return size >> OBUCK_SHIFT;
+}
+
 struct fastbuf *
-obuck_create(void)
+obuck_create(u32 type)
 {
   obuck_lock_write();
   bflush(obuck_fb);
@@ -280,7 +289,8 @@ obuck_create(void)
     obuck_broken("Misaligned file");
   obuck_hdr.magic = OBUCK_INCOMPLETE_MAGIC;
   obuck_hdr.oid = bucket_start >> OBUCK_SHIFT;
-  obuck_hdr.length = obuck_hdr.orig_length = 0;
+  obuck_hdr.length = 0;
+  obuck_hdr.type = type;
   bucket_current = bucket_start;
   bwrite(obuck_fb, &obuck_hdr, sizeof(obuck_hdr));
   obuck_fb->pos = -sizeof(obuck_hdr);
@@ -292,7 +302,7 @@ obuck_create_end(struct fastbuf *b UNUSED, struct obuck_header *hdrp)
 {
   int pad;
   obuck_hdr.magic = OBUCK_MAGIC;
-  obuck_hdr.length = obuck_hdr.orig_length = btell(obuck_fb);
+  obuck_hdr.length = btell(obuck_fb);
   pad = (OBUCK_ALIGN - sizeof(obuck_hdr) - obuck_hdr.length - 4) & (OBUCK_ALIGN - 1);
   while (pad--)
     bputc(obuck_fb, 0);
@@ -314,6 +324,74 @@ obuck_delete(oid_t oid)
   obuck_unlock();
 }
 
+/*** Fast reading of the whole pool ***/
+
+static struct fastbuf *obuck_rpf;
+
+static int
+obuck_slurp_refill(struct fastbuf *f)
+{
+  uns l;
+
+  if (!obuck_remains)
+    return 0;
+  l = bdirect_read_prepare(obuck_rpf, &f->buffer);
+  if (!l)
+    obuck_broken("Incomplete object");
+  l = MIN(l, obuck_remains);
+  bdirect_read_commit(obuck_rpf, f->buffer + l);
+  obuck_remains -= l;
+  f->bptr = f->buffer;
+  f->bufend = f->bstop = f->buffer + l;
+  return 1;
+}
+
+struct fastbuf *
+obuck_slurp_pool(struct obuck_header *hdrp)
+{
+  static struct fastbuf limiter;
+  uns l;
+
+  do
+    {
+      if (!obuck_rpf)
+       {
+         obuck_lock_read();
+         obuck_rpf = bopen(obuck_name, O_RDONLY, obuck_slurp_buflen);
+       }
+      else
+       {
+         bsetpos(obuck_rpf, bucket_current - 4);
+         if (bgetl(obuck_rpf) != OBUCK_TRAILER)
+           obuck_broken("Missing trailer");
+       }
+      bucket_start = btell(obuck_rpf);
+      l = bread(obuck_rpf, hdrp, sizeof(struct obuck_header));
+      if (!l)
+       {
+         bclose(obuck_rpf);
+         obuck_rpf = NULL;
+         obuck_unlock();
+         return NULL;
+       }
+      if (l != sizeof(struct obuck_header))
+       obuck_broken("Short header read");
+      if (hdrp->magic != OBUCK_MAGIC)
+       obuck_broken("Missing magic number");
+      bucket_current = (bucket_start + sizeof(obuck_hdr) + hdrp->length +
+                       4 + OBUCK_ALIGN - 1) & ~((sh_off_t)(OBUCK_ALIGN - 1));
+    }
+  while (hdrp->oid == OBUCK_OID_DELETED);
+  if (obuck_get_pos(hdrp->oid) != bucket_start)
+    obuck_broken("Invalid backlink");
+  obuck_remains = hdrp->length;
+  limiter.bptr = limiter.bstop = limiter.buffer = limiter.bufend = NULL;
+  limiter.name = "Bucket";
+  limiter.pos = 0;
+  limiter.refill = obuck_slurp_refill;
+  return &limiter;
+}
+
 /*** Shakedown ***/
 
 void
@@ -346,6 +424,18 @@ obuck_shakedown(int (*kibitz)(struct obuck_header *old, oid_t new, byte *buck))
          goto broken;
        }
       l = (sizeof(struct obuck_header) + rhdr->length + 4 + OBUCK_ALIGN - 1) & ~(OBUCK_ALIGN-1);
+      if (l > obuck_shake_buflen)
+       {
+         if (rhdr->oid != OBUCK_OID_DELETED)
+           {
+             msg = "bucket longer than ShakeBufSize";
+             goto broken;
+           }
+         rstart = bucket_start + l;
+         roff = 0;
+         rsize = 0;
+         goto reread;
+       }
       if (rsize - roff < l)
        goto reread;
       if (GET_U32(rbuf + roff + l - 4) != OBUCK_TRAILER)
@@ -417,7 +507,7 @@ obuck_shakedown(int (*kibitz)(struct obuck_header *old, oid_t new, byte *buck))
   return;
 
  broken:
-  log(L_ERROR, "Error during object pool shakedown: %s (pos=%Ld), gathering debris", msg, (long long) bucket_start);
+  log(L_ERROR, "Error during object pool shakedown: %s (pos=%Ld, id=%x), gathering debris", msg, (long long) bucket_start, (uns)(bucket_start >> OBUCK_SHIFT));
   if (woff)
     {
       sh_pwrite(obuck_fd, wbuf, woff, wstart);
@@ -432,7 +522,6 @@ obuck_shakedown(int (*kibitz)(struct obuck_header *old, oid_t new, byte *buck))
        obuck_hdr.length = bucket_start - wstart - sizeof(obuck_hdr) - 4;
       else
        obuck_hdr.length = 0x40000000 - sizeof(obuck_hdr) - 4;
-      obuck_hdr.orig_length = obuck_hdr.length;
       sh_pwrite(obuck_fd, &obuck_hdr, sizeof(obuck_hdr), wstart);
       wstart += sizeof(obuck_hdr) + obuck_hdr.length + 4;
       sh_pwrite(obuck_fd, &check, 4, wstart-4);
@@ -472,7 +561,7 @@ int main(int argc, char **argv)
       for(i=0; i<LEN(j); i++)
         bputc(b, (i+j) % 256);
       obuck_create_end(b, &h);
-      printf("Writing %08x %d -> %d\n", h.oid, h.orig_length, h.length);
+      printf("Writing %08x %d\n", h.oid, h.length);
       ids[j] = h.oid;
     }
   for(j=0; j<COUNT; j++)
@@ -489,10 +578,10 @@ int main(int argc, char **argv)
        h.oid = ids[j];
        obuck_find_by_oid(&h);
        b = obuck_fetch();
-       printf("Reading %08x %d -> %d\n", h.oid, h.orig_length, h.length);
-       if (h.orig_length != LEN(j))
+       printf("Reading %08x %d\n", h.oid, h.length);
+       if (h.length != LEN(j))
          die("Invalid length");
-       for(i=0; i<h.orig_length; i++)
+       for(i=0; i<h.length; i++)
          if ((unsigned) bgetc(b) != (i+j) % 256)
            die("Contents mismatch");
        if (bgetc(b) != EOF)
@@ -502,7 +591,7 @@ int main(int argc, char **argv)
   if (obuck_find_first(&h, 0))
     do
       {
-       printf("<<< %08x\t%d\n", h.oid, h.orig_length);
+       printf("<<< %08x\t%d\n", h.oid, h.length);
        cnt--;
       }
     while (obuck_find_next(&h, 0));