- buckettool updated to extract, cat, and create all kinds of buckets

- added parameter -r for raw (non-parsed) output

diff --git a/lib/Makefile b/lib/Makefile
index 54ebe46b0be783848f8780a844d749dd39d9ac11..9cee12776da053e8378aa45b2b60b4713b2c1f96 100644 (file)
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -26,7 +26,7 @@ obj/lib/hashfunc.o obj/lib/hashfunc.oo: CFLAGS += -funroll-loops
 
 obj/lib/db-test: obj/lib/db-test.o $(LIBSH)
 obj/lib/db-tool: obj/lib/db-tool.o $(LIBSH)
-obj/lib/buckettool: obj/lib/buckettool.o $(LIBSH)
+obj/lib/buckettool: obj/lib/buckettool.o $(LIBCHARSET) $(LIBSH)
 obj/lib/conf-test: obj/lib/conf-test.o $(LIBSH)
 obj/lib/sort-test: obj/lib/sort-test.o $(LIBSH)
 obj/lib/lfs-test: obj/lib/lfs-test.o $(LIBSH)

diff --git a/lib/buckettool.c b/lib/buckettool.c
index 179e3d98ea67e10844be6dfe4b33a8f94ed7d844..81b252f8eed6538cd6955eeae79cef735e9842eb 100644 (file)
--- a/lib/buckettool.c
+++ b/lib/buckettool.c
@@ -2,6 +2,7 @@
  *     Sherlock Library -- Bucket Manipulation Tool
  *
  *     (c) 2001 Martin Mares <mj@ucw.cz>
+ *     (c) 2004 Robert Spalek <robert@ucw.cz>
  *
  *     This software may be freely distributed and used according to the terms
  *     of the GNU Lesser General Public License.
@@ -12,6 +13,12 @@
 #include "lib/fastbuf.h"
 #include "lib/lfs.h"
 #include "lib/conf.h"
+#include "lib/pools.h"
+#include "lib/object.h"
+#include "lib/buck2obj.h"
+#include "lib/obj2buck.h"
+#include "lib/lizard.h"
+#include "charset/unistream.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -20,6 +27,8 @@
 #include <unistd.h>
 
 static int verbose;
+static struct mempool *pool;
+static struct buck2obj_buf *buck_buf;
 
 static void
 help(void)
@@ -41,6 +50,7 @@ CF_USAGE
 -q\t\tquick check of bucket file consistency\n\
 -s\t\tshake down bucket file (without updating other structures!!!)\n\
 -v\t\tbe verbose\n\
+-r\t\tdo not parse V33 buckets, but print the raw content\n\
 ");
   exit(1);
 }
@@ -82,10 +92,45 @@ delete(char *id)
   obuck_cleanup();
 }
 
+static inline void
+dump_oattr(struct fastbuf *out, struct oattr *oa)
+{
+  for (struct oattr *a = oa; a; a = a->same)
+    bprintf(out, "%c%s\n", a->attr, a->val);
+}
+
+static void
+dump_parsed_bucket(struct fastbuf *out, struct obuck_header *h, struct fastbuf *b)
+{
+  mp_flush(pool);
+  struct odes *o = obj_read_bucket(buck_buf, h->type, h->length, b, NULL);
+  if (!o)
+    bprintf(out, "Cannot parse bucket %x of type %x and length %d: %m\n", h->oid, h->type, h->length);
+  else
+  {
+    if (h->type < BUCKET_TYPE_V30)
+    {
+      for (struct oattr *oa = o->attrs; oa; oa = oa->next)
+       dump_oattr(out, oa);
+    }
+    else
+    {
+#define        IS_HEADER(x) (x=='O' || x=='U')
+      for (struct oattr *oa = o->attrs; oa; oa = oa->next)
+       if (IS_HEADER(oa->attr))
+         dump_oattr(out, oa);
+      bputc(out, '\n');
+      for (struct oattr *oa = o->attrs; oa; oa = oa->next)
+       if (!IS_HEADER(oa->attr))
+         dump_oattr(out, oa);
+    }
+  }
+}
+
 static void
 extract(char *id)
 {
-  struct fastbuf *b;
+  struct fastbuf *b, *out;
   byte buf[1024];
   int l;
   struct obuck_header h;
@@ -93,13 +138,24 @@ extract(char *id)
   h.oid = parse_id(id);
   obuck_init(0);
   obuck_find_by_oid(&h);
+  out = bfdopen_shared(1, 65536);
   b = obuck_fetch();
-  while ((l = bread(b, buf, sizeof(buf))))
-    fwrite(buf, 1, l, stdout);
+  if (h.type < BUCKET_TYPE_V33 || !buck_buf)
+  {
+    while ((l = bread(b, buf, sizeof(buf))))
+      bwrite(out, buf, l);
+  }
+  else
+    dump_parsed_bucket(out, &h, b);
   bclose(b);
+  bclose(out);
   obuck_cleanup();
 }
 
+#define        GBUF_TYPE       byte
+#define        GBUF_PREFIX(x)  bb_##x
+#include "lib/gbuf.h"
+
 static void
 insert(byte *arg)
 {
@@ -108,7 +164,10 @@ insert(byte *arg)
   struct obuck_header h;
   byte *e;
   u32 type;
+  bb_t lizard_buf, compressed_buf;
 
+  bb_init(&lizard_buf);
+  bb_init(&compressed_buf);
   if (!arg)
     type = BUCKET_TYPE_PLAIN;
   else if (sscanf(arg, "%x", &type) != 1)
@@ -118,14 +177,57 @@ insert(byte *arg)
   obuck_init(1);
   do
     {
+      uns lizard_filled = 0;
+      uns in_body = 0;
       b = NULL;
-      while ((e = bgets(in, buf, sizeof(buf))) && buf[0])
+      while ((e = bgets(in, buf, sizeof(buf))))
        {
-         *e++ = '\n';
+         if (!buf[0])
+         {
+           if (in_body || type < BUCKET_TYPE_V30)
+             break;
+           in_body = 1;
+         }
          if (!b)
            b = obuck_create(type);
-         bwrite(b, buf, e-buf);
+         if (type < BUCKET_TYPE_V33)
+         {
+           *e++ = '\n';
+           bwrite(b, buf, e-buf);
+         }
+         else if (in_body == 1)
+         {
+           bputc(b, 0);
+           in_body = 2;
+         }
+         else if (type == BUCKET_TYPE_V33 || !in_body)
+         {
+           bwrite_v33(b, buf[0], buf+1, e-buf-1);
+         }
+         else
+         {
+           uns want_len = lizard_filled + (e-buf) + 6 + LIZARD_NEEDS_CHARS;    // +6 is the maximum UTF-8 length
+           bb_grow(&lizard_buf, want_len);
+           byte *ptr = lizard_buf.ptr + lizard_filled;
+           WRITE_V33(ptr, buf[0], buf+1, e-buf-1);
+           lizard_filled = ptr - lizard_buf.ptr;
+         }
        }
+      if (in_body && type == BUCKET_TYPE_V33_LIZARD)
+      {
+       bputl(b, lizard_filled
+#if 0  //TEST error resilience: write wrong length
+           +1
+#endif
+           );
+       uns want_len = lizard_filled * LIZARD_MAX_MULTIPLY + LIZARD_MAX_ADD;
+       bb_grow(&compressed_buf, want_len);
+       want_len = lizard_compress(lizard_buf.ptr, lizard_filled, compressed_buf.ptr);
+#if 0  //TEST error resilience: tamper the compressed data by removing EOF
+       compressed_buf[want_len-1] = 1;
+#endif
+       bwrite(b, compressed_buf.ptr, want_len);
+      }
       if (b)
        {
          obuck_create_end(b, &h);
@@ -133,6 +235,8 @@ insert(byte *arg)
        }
     }
   while (e);
+  bb_done(&lizard_buf);
+  bb_done(&compressed_buf);
   obuck_cleanup();
   bclose(in);
 }
@@ -143,21 +247,25 @@ cat(void)
   struct obuck_header h;
   struct fastbuf *b, *out;
   byte buf[1024];
-  int l, lf;
 
   obuck_init(0);
   out = bfdopen_shared(1, 65536);
   while (b = obuck_slurp_pool(&h))
     {
       bprintf(out, "### %08x %6d %08x\n", h.oid, h.length, h.type);
-      lf = 1;
-      while ((l = bread(b, buf, sizeof(buf))))
+      if (h.type < BUCKET_TYPE_V33 || !buck_buf)
+      {
+       int lf = 1, l;
+       while ((l = bread(b, buf, sizeof(buf))))
        {
          bwrite(out, buf, l);
          lf = (buf[l-1] == '\n');
        }
-      if (!lf)
-       bprintf(out, "\n# <missing EOL>\n");
+       if (!lf)
+         bprintf(out, "\n# <missing EOL>\n");
+      }
+      else
+       dump_parsed_bucket(out, &h, b);
     }
   bclose(out);
   obuck_cleanup();
@@ -287,14 +395,17 @@ main(int argc, char **argv)
 {
   int i, op;
   char *arg = NULL;
+  uns raw = 0;
 
   log_init(NULL);
   op = 0;
-  while ((i = cf_getopt(argc, argv, CF_SHORT_OPTS "lLd:x:i::cfFqsv", CF_NO_LONG_OPTS, NULL)) != -1)
+  while ((i = cf_getopt(argc, argv, CF_SHORT_OPTS "lLd:x:i::cfFqsvr", CF_NO_LONG_OPTS, NULL)) != -1)
     if (i == '?' || op)
       help();
     else if (i == 'v')
       verbose++;
+    else if (i == 'r')
+      raw++;
     else
       {
        op = i;
@@ -303,6 +414,11 @@ main(int argc, char **argv)
   if (optind < argc)
     help();
 
+  if (!raw)
+  {
+    pool = mp_new(1<<14);
+    buck_buf = buck2obj_alloc(pool);
+  }
   switch (op)
     {
     case 'l':
@@ -338,6 +454,11 @@ main(int argc, char **argv)
     default:
       help();
     }
+  if (buck_buf)
+  {
+    buck2obj_free(buck_buf);
+    mp_delete(pool);
+  }
 
   return 0;
 }