Preliminary version of object bucket code.

diff --git a/lib/Makefile b/lib/Makefile
index af8fb34818af3366613c87f1b89c98748e8d2b03..27c05a53c5d4f79b643693f2677276affcd48872 100644 (file)
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -6,7 +6,7 @@ PROGS+=obj/lib/db-test obj/lib/db-rebuild
 SHLIB_OBJS=alloc.o alloc_str.o ctmatch.o db.o fastbuf.o fb-file.o fb-mem.o lists.o \
        log.o log2.o md5.o md5hex.o mmap.o pagecache.o patimatch.o patmatch.o pool.o \
        prime.o random.o realloc.o regex.o temp.o timer.o url.o wildmatch.o \
-       wordsplit.o str_ctype.o str_upper.o
+       wordsplit.o str_ctype.o str_upper.o bucket.o
 
 obj/lib/libsh.a: $(addprefix obj/lib/,$(SHLIB_OBJS))
 

diff --git a/lib/bucket.c b/lib/bucket.c
new file mode 100644 (file)
index 0000000..2e6db98
--- /dev/null
+++ b/lib/bucket.c
@@ -0,0 +1,207 @@
+/*
+ *     Sherlock Library -- Object Buckets
+ *
+ *     (c) 2001 Martin Mares <mj@ucw.cz>
+ *
+ *     Warning: Touches internals of the fb-file module!
+ */
+
+#include "lib/lib.h"
+#include "lib/bucket.h"
+#include "lib/fastbuf.h"
+
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/file.h>
+
+static int obuck_fd;
+static struct fastbuf *obuck_fb;
+static struct obuck_header obuck_hdr;
+static sh_off_t start_of_this, start_of_next;
+static char *obuck_name = "db/objects";                /* FIXME */
+
+void
+obuck_init(int writeable)
+{
+  obuck_fb = bopen(obuck_name, (writeable ? O_RDWR | O_CREAT : O_RDONLY), 65536);
+  obuck_fd = obuck_fb->fd;
+}
+
+void
+obuck_cleanup(void)
+{
+  bclose(obuck_fb);
+}
+
+static void
+obuck_broken(char *msg)
+{
+  die("Object pool corrupted: %s", msg);       /* FIXME */
+}
+
+static inline void
+obuck_lock_read(void)
+{
+  flock(obuck_fd, LOCK_SH);
+}
+
+static inline void
+obuck_lock_write(void)
+{
+  flock(obuck_fd, LOCK_EX);
+}
+
+static inline void
+obuck_unlock(void)
+{
+  flock(obuck_fd, LOCK_UN);
+}
+
+static void
+obuck_fetch_header(oid_t oid)
+{
+  start_of_this = ((sh_off_t) oid) << OBUCK_SHIFT;
+  bsetpos(obuck_fb, start_of_this);
+  bread(obuck_fb, &obuck_hdr, sizeof(obuck_hdr));
+  if (obuck_hdr.magic != OBUCK_MAGIC)
+    obuck_broken("Missing magic number");
+  if (obuck_hdr.oid == OBUCK_OID_DELETED)
+    obuck_broken("Access to deleted bucket");
+  if (obuck_hdr.oid != oid)
+    obuck_broken("Invalid backlink");
+}
+
+struct fastbuf *
+obuck_fetch(struct obuck_header *hdrp)
+{
+  obuck_lock_read();
+  obuck_fetch_header(hdrp->oid);
+  memcpy(hdrp, &obuck_hdr, sizeof(obuck_hdr));
+  return obuck_fb;
+}
+
+void
+obuck_fetch_abort(struct fastbuf *b UNUSED)
+{
+  obuck_unlock();
+}
+
+void
+obuck_fetch_end(struct fastbuf *b UNUSED)
+{
+  if (bgetl(b) != OBUCK_TRAILER)
+    obuck_broken("Corrupted trailer");
+  obuck_unlock();
+}
+
+struct fastbuf *
+obuck_write(void)
+{
+  obuck_lock_write();
+  bseek(obuck_fb, 0, SEEK_END);
+  start_of_this = btell(obuck_fb);
+  if (start_of_this & (OBUCK_ALIGN - 1))
+    obuck_broken("Misaligned file");
+  obuck_hdr.magic = 0;
+  obuck_hdr.oid = start_of_this >> OBUCK_SHIFT;
+  obuck_hdr.length = obuck_hdr.orig_length = 0;
+  bwrite(obuck_fb, &obuck_hdr, sizeof(obuck_hdr));
+  return obuck_fb;
+}
+
+void
+obuck_write_end(struct fastbuf *b UNUSED, struct obuck_header *hdrp)
+{
+  int pad;
+  obuck_hdr.magic = OBUCK_MAGIC;
+  obuck_hdr.length = obuck_hdr.orig_length = btell(obuck_fb) - start_of_this - sizeof(obuck_hdr);
+  bputl(obuck_fb, OBUCK_TRAILER);
+  pad = (OBUCK_ALIGN - sizeof(obuck_hdr) - obuck_hdr.length - 4) & (OBUCK_ALIGN - 1);
+  while (pad--)
+    bputc(obuck_fb, 0);
+  bflush(obuck_fb);
+  bsetpos(obuck_fb, start_of_this);
+  /* FIXME: Can be replaced with single pwrite */
+  bwrite(obuck_fb, &obuck_hdr, sizeof(obuck_hdr));
+  bflush(obuck_fb);
+  obuck_unlock();
+  memcpy(hdrp, &obuck_hdr, sizeof(obuck_hdr));
+}
+
+void
+obuck_delete(oid_t oid)
+{
+  obuck_lock_write();
+  obuck_fetch_header(oid);
+  obuck_hdr.oid = OBUCK_OID_DELETED;
+  bflush(obuck_fb);
+  bsetpos(obuck_fb, start_of_this);
+  bwrite(obuck_fb, &obuck_hdr, sizeof(obuck_hdr));
+  bflush(obuck_fb);
+  obuck_unlock();
+}
+
+struct fastbuf *
+obuck_walk_init(void)
+{
+  start_of_this = start_of_next = 0;
+  obuck_lock_read();
+  return obuck_fb;
+}
+
+struct fastbuf *
+obuck_walk_next(struct fastbuf *b, struct obuck_header *hdrp)
+{
+  int c;
+
+restart:
+  start_of_this = start_of_next;
+  bsetpos(b, start_of_this);
+  c = bgetc(b);
+  if (c < 0)
+    return NULL;
+  bungetc(b, c);
+  bread(b, &obuck_hdr, sizeof(obuck_hdr));
+  if (obuck_hdr.magic != OBUCK_MAGIC)
+    obuck_broken("Missing magic number");
+  start_of_next = (start_of_this + sizeof(obuck_hdr) + obuck_hdr.orig_length +
+       4 + OBUCK_ALIGN - 1) & ~((sh_off_t)(OBUCK_ALIGN - 1));
+  if (obuck_hdr.oid == OBUCK_OID_DELETED)
+    goto restart;
+  memcpy(hdrp, &obuck_hdr, sizeof(obuck_hdr));
+  return b;
+}
+
+void
+obuck_walk_end(struct fastbuf *b UNUSED)
+{
+  obuck_unlock();
+}
+
+#ifdef TEST
+int main(void)
+{
+  int i, j;
+  struct obuck_header h;
+  struct fastbuf *b;
+  obuck_init(1);
+  for(j=0; j<100; j++)
+    {
+      b = obuck_write();
+      for(i=0; i<100*j; i++)
+        bputc(b, i);
+      obuck_write_end(b, &h);
+      printf("%d\t%08x\t%d\n", j, h.oid, h.orig_length);
+    }
+  obuck_delete(0);
+  b = obuck_walk_init();
+  while (b = obuck_walk_next(b, &h))
+    {
+      printf("<<< %08x\t%d\n", h.oid, h.orig_length);
+    }
+  obuck_walk_end(b);
+  obuck_cleanup();
+  return 0;
+}
+#endif

diff --git a/lib/bucket.h b/lib/bucket.h
new file mode 100644 (file)
index 0000000..a9b938b
--- /dev/null
+++ b/lib/bucket.h
@@ -0,0 +1,43 @@
+/*
+ *     Sherlock Library -- Object Buckets
+ *
+ *     (c) 2001 Martin Mares <mj@ucw.cz>
+ */
+
+/*
+ * Format: The object pool is merely a sequence of object buckets.
+ * Each bucket starts with struct obuck_header and it's padded
+ * by zeros to a multiple of OBUCK_ALIGN bytes.
+ *
+ * Locking: Each operation on the pool is protected by a flock.
+ *
+ * The buckets emulate non-seekable fastbuf streams.
+ */
+
+#define OBUCK_SHIFT 7
+#define OBUCK_ALIGN (1<<OBUCK_SHIFT)
+#define OBUCK_MAGIC 0xdeadf00d
+#define OBUCK_TRAILER 0xfeedcafe
+#define OBUCK_OID_DELETED (~(oid_t)0)
+
+struct obuck_header {
+  u32 magic;                   /* OBUCK_MAGIC should dwell here */
+  oid_t oid;                   /* ID of this object or OBUCK_OID_DELETED */
+  u32 length;                  /* Length of compressed data in the bucket */
+  u32 orig_length;             /* Length of uncompressed data */
+  /* Bucket data continue here */
+};
+
+struct fastbuf;
+
+void obuck_init(int writeable);
+void obuck_cleanup(void);
+struct fastbuf * obuck_fetch(struct obuck_header *hdrp);
+void obuck_fetch_abort(struct fastbuf *b);
+void obuck_fetch_end(struct fastbuf *b);
+struct fastbuf * obuck_write(void);
+void obuck_write_end(struct fastbuf *b, struct obuck_header *hdrp);
+void obuck_delete(oid_t oid);
+struct fastbuf * obuck_walk_init(void);
+struct fastbuf * obuck_walk_next(struct fastbuf *b, struct obuck_header *hdrp);
+void obuck_walk_end(struct fastbuf *b);