Adapted to new unicode includes.

[libucw.git] / lib / buckettool.c

diff --git a/lib/buckettool.c b/lib/buckettool.c
index 609d4e03b933b71ee25f1560d4fda731e7cd4b18..f22f17f90691e53ab164dfb97970f99710fc570b 100644 (file)
--- a/lib/buckettool.c
+++ b/lib/buckettool.c
@@ -1,13 +1,23 @@
 /*
  *     Sherlock Library -- Bucket Manipulation Tool
  *
 /*
  *     Sherlock Library -- Bucket Manipulation Tool
  *

- *     (c) 2001 Martin Mares <mj@ucw.cz>
+ *     (c) 2001--2004 Martin Mares <mj@ucw.cz>
+ *     (c) 2004 Robert Spalek <robert@ucw.cz>
+ *
+ *     This software may be freely distributed and used according to the terms
+ *     of the GNU Lesser General Public License.

  */
 
 #include "lib/lib.h"
 #include "lib/bucket.h"
 #include "lib/fastbuf.h"
 #include "lib/lfs.h"
  */
 
 #include "lib/lib.h"
 #include "lib/bucket.h"
 #include "lib/fastbuf.h"
 #include "lib/lfs.h"

+#include "lib/conf.h"
+#include "lib/pools.h"
+#include "lib/object.h"
+#include "lib/lizard.h"
+#include "lib/bbuf.h"
+#include "lib/ff-utf8.h"

 
 #include <stdio.h>
 #include <stdlib.h>
 
 #include <stdio.h>
 #include <stdlib.h>


@@ -15,21 +25,31 @@
 #include <fcntl.h>
 #include <unistd.h>
 
 #include <fcntl.h>
 #include <unistd.h>
 

+static int verbose;
+static struct mempool *pool;
+static struct buck2obj_buf *buck_buf;
+

 static void
 help(void)
 {
   fprintf(stderr, "\
 static void
 help(void)
 {
   fprintf(stderr, "\

-Usage: buckettool <commands>\n\
+Usage: buckettool [<options>] <command>\n\

 \n\
 \n\

-Commands:\n\
+Options:\n"
+CF_USAGE
+"\nCommands:\n\

 -l\t\tlist all buckets\n\
 -L\t\tlist all buckets including deleted ones\n\
 -d <obj>\tdelete bucket\n\
 -x <obj>\textract bucket\n\
 -l\t\tlist all buckets\n\
 -L\t\tlist all buckets including deleted ones\n\
 -d <obj>\tdelete bucket\n\
 -x <obj>\textract bucket\n\

--i\t\tinsert bucket\n\
+-i[<type>]\tinsert buckets separated by blank lines\n\

 -c\t\tconcatenate and dump all buckets\n\
 -f\t\taudit bucket file structure\n\
 -F\t\taudit and fix bucket file structure\n\
 -c\t\tconcatenate and dump all buckets\n\
 -f\t\taudit bucket file structure\n\
 -F\t\taudit and fix bucket file structure\n\

+-q\t\tquick check of bucket file consistency\n\
+-r\t\tdo not parse V33 buckets, but print the raw content\n\
+-s\t\tshake down bucket file (without updating other structures!!!)\n\
+-v\t\tbe verbose\n\

 ");
   exit(1);
 }
 ");
   exit(1);
 }


@@ -56,7 +76,7 @@ list(int full)
        if (h.oid == OBUCK_OID_DELETED)
          printf("DELETED  %6d\n", h.length);
        else
        if (h.oid == OBUCK_OID_DELETED)
          printf("DELETED  %6d\n", h.length);
        else

-         printf("%08x %6d %6d\n", h.oid, h.length, h.orig_length);
+         printf("%08x %6d %08x\n", h.oid, h.length, h.type);

       }
     while (obuck_find_next(&h, full));
   obuck_cleanup();
       }
     while (obuck_find_next(&h, full));
   obuck_cleanup();


@@ -71,10 +91,34 @@ delete(char *id)
   obuck_cleanup();
 }
 
   obuck_cleanup();
 }
 

+static inline void
+dump_oattrs(struct fastbuf *out, struct oattr *a)
+{
+  for (; a; a = a->same)
+    bprintf(out, "%c%s\n", a->attr, a->val);
+}
+
+static void
+dump_parsed_bucket(struct fastbuf *out, struct obuck_header *h, struct fastbuf *b)
+{
+  struct odes *o_hdr, *o_body;
+  mp_flush(pool);
+  o_hdr = obj_new(pool);
+  o_body = obj_new(pool);
+  if (buck2obj_parse(buck_buf, h->type, h->length, b, o_hdr, NULL, o_body) < 0)
+    bprintf(out, ".Cannot parse bucket %x of type %x and length %d: %m\n", h->oid, h->type, h->length);
+  else
+    {
+      dump_oattrs(out, o_hdr->attrs);
+      bputc(out, '\n');
+      dump_oattrs(out, o_body->attrs);
+    }
+}
+

 static void
 extract(char *id)
 {
 static void
 extract(char *id)
 {

-  struct fastbuf *b;
+  struct fastbuf *b, *out;

   byte buf[1024];
   int l;
   struct obuck_header h;
   byte buf[1024];
   int l;
   struct obuck_header h;


@@ -82,55 +126,129 @@ extract(char *id)
   h.oid = parse_id(id);
   obuck_init(0);
   obuck_find_by_oid(&h);
   h.oid = parse_id(id);
   obuck_init(0);
   obuck_find_by_oid(&h);

+  out = bfdopen_shared(1, 65536);

   b = obuck_fetch();
   b = obuck_fetch();

-  while ((l = bread(b, buf, sizeof(buf))))
-    fwrite(buf, 1, l, stdout);
-  obuck_fetch_end(b);
+  if (h.type < BUCKET_TYPE_V33 || !buck_buf)
+  {
+    while ((l = bread(b, buf, sizeof(buf))))
+      bwrite(out, buf, l);
+  }
+  else
+    dump_parsed_bucket(out, &h, b);
+  bclose(b);
+  bclose(out);

   obuck_cleanup();
 }
 
 static void
   obuck_cleanup();
 }
 
 static void

-insert(void)
+insert(byte *arg)

 {
 {

-  struct fastbuf *b;
-  byte buf[1024];
-  int l;
+  struct fastbuf *b, *in;
+  byte buf[4096];

   struct obuck_header h;
   struct obuck_header h;

+  byte *e;
+  u32 type;
+  bb_t lizard_buf, compressed_buf;
+
+  bb_init(&lizard_buf);
+  bb_init(&compressed_buf);
+  if (!arg)
+    type = BUCKET_TYPE_PLAIN;
+  else if (sscanf(arg, "%x", &type) != 1)
+    die("Type `%s' is not a hexadecimal number");
+  attr_set_type(type);

 
 

+  in = bfdopen_shared(0, 4096);

   obuck_init(1);
   obuck_init(1);

-  b = obuck_create();
-  while ((l = fread(buf, 1, sizeof(buf), stdin)))
-    bwrite(b, buf, l);
-  obuck_create_end(b, &h);
+  do
+    {
+      uns lizard_filled = 0;
+      uns in_body = 0;
+      b = NULL;
+      while ((e = bgets(in, buf, sizeof(buf))))
+       {
+         if (!buf[0])
+         {
+           if (in_body || type < BUCKET_TYPE_V30)
+             break;
+           in_body = 1;
+         }
+         if (!b)
+           b = obuck_create(type);
+         if (in_body == 1)
+         {
+           bputc(b, 0);
+           in_body = 2;
+         }
+         else if (type <= BUCKET_TYPE_V33 || !in_body)
+         {
+           bput_attr(b, buf[0], buf+1, e-buf-1);
+         }
+         else
+         {
+           ASSERT(BUCKET_TYPE_V33_LIZARD);
+           uns want_len = lizard_filled + (e-buf) + 6 + LIZARD_NEEDS_CHARS;    // +6 is the maximum UTF-8 length
+           bb_grow(&lizard_buf, want_len);
+           byte *ptr = lizard_buf.ptr + lizard_filled;
+           ptr = put_attr(ptr, buf[0], buf+1, e-buf-1);
+           lizard_filled = ptr - lizard_buf.ptr;
+         }
+       }
+      if (in_body && type == BUCKET_TYPE_V33_LIZARD)
+      {
+       bputl(b, lizard_filled
+#if 0  //TEST error resilience: write wrong length
+           +1
+#endif
+           );
+       uns want_len = lizard_filled * LIZARD_MAX_MULTIPLY + LIZARD_MAX_ADD;
+       bb_grow(&compressed_buf, want_len);
+       want_len = lizard_compress(lizard_buf.ptr, lizard_filled, compressed_buf.ptr);
+#if 0  //TEST error resilience: tamper the compressed data by removing EOF
+       compressed_buf[want_len-1] = 1;
+#endif
+       bwrite(b, compressed_buf.ptr, want_len);
+      }
+      if (b)
+       {
+         obuck_create_end(b, &h);
+         printf("%08x %d %08x\n", h.oid, h.length, h.type);
+       }
+    }
+  while (e);
+  bb_done(&lizard_buf);
+  bb_done(&compressed_buf);

   obuck_cleanup();
   obuck_cleanup();

-  printf("%08x %d %d\n", h.oid, h.length, h.orig_length);
+  bclose(in);

 }
 
 static void
 cat(void)
 {
   struct obuck_header h;
 }
 
 static void
 cat(void)
 {
   struct obuck_header h;

-  struct fastbuf *b;
+  struct fastbuf *b, *out;

   byte buf[1024];
   byte buf[1024];

-  int l, lf;

 
   obuck_init(0);
 
   obuck_init(0);

-  if (obuck_find_first(&h, 0))
-    do
+  out = bfdopen_shared(1, 65536);
+  while (b = obuck_slurp_pool(&h))
+    {
+      bprintf(out, "### %08x %6d %08x\n", h.oid, h.length, h.type);
+      if (h.type < BUCKET_TYPE_V33 || !buck_buf)

       {
       {

-       printf("### %08x %6d %6d\n", h.oid, h.length, h.orig_length);
-       b = obuck_fetch();
-       lf = 1;
+       int lf = 1, l;

        while ((l = bread(b, buf, sizeof(buf))))
        while ((l = bread(b, buf, sizeof(buf))))

-         {
-           fwrite(buf, 1, l, stdout);
-           lf = (buf[l-1] == '\n');
-         }
-       obuck_fetch_end(b);
+       {
+         bwrite(out, buf, l);
+         lf = (buf[l-1] == '\n');
+       }

        if (!lf)
        if (!lf)

-         printf("\n# <missing EOL>\n");
+         bprintf(out, "\n# <missing EOL>\n");

       }
       }

-    while (obuck_find_next(&h, 0));
+      else
+       dump_parsed_bucket(out, &h, b);
+    }
+  bclose(out);

   obuck_cleanup();
 }
 
   obuck_cleanup();
 }
 


@@ -170,9 +288,12 @@ fsck(int fix)
            printf("%08x  missing trailer\n", oid);
          else if (chk != OBUCK_TRAILER)
            printf("%08x  mismatched trailer\n", oid);
            printf("%08x  missing trailer\n", oid);
          else if (chk != OBUCK_TRAILER)
            printf("%08x  mismatched trailer\n", oid);

-         /* OK */
-         pos = end;
-         continue;
+         else
+           {
+             /* OK */
+             pos = end;
+             continue;
+           }

        }
       errors++;
       end = pos;
        }
       errors++;
       end = pos;


@@ -191,7 +312,7 @@ fsck(int fix)
              if (fix)
                {
                  printf("*** truncating file\n");
              if (fix)
                {
                  printf("*** truncating file\n");

-                 ftruncate(fd, pos);
+                 sh_ftruncate(fd, pos);

                }
              else
                printf("*** would truncate the file here\n");
                }
              else
                printf("*** would truncate the file here\n");


@@ -200,17 +321,19 @@ fsck(int fix)
        }
       while (nh.magic != OBUCK_MAGIC ||
             (nh.oid != (oid_t)(end >> OBUCK_SHIFT) && nh.oid != OBUCK_OID_DELETED));
        }
       while (nh.magic != OBUCK_MAGIC ||
             (nh.oid != (oid_t)(end >> OBUCK_SHIFT) && nh.oid != OBUCK_OID_DELETED));

-      printf("*** match at oid %08x\n", end >> OBUCK_SHIFT);
+      printf("*** match at oid %08x\n", (uns)(end >> OBUCK_SHIFT));

       if (fix)
        {
          h.magic = OBUCK_MAGIC;
          h.oid = OBUCK_OID_DELETED;
       if (fix)
        {
          h.magic = OBUCK_MAGIC;
          h.oid = OBUCK_OID_DELETED;

-         h.length = h.orig_length = end - pos - sizeof(h) - 4;
+         h.length = end - pos - sizeof(h) - 4;

          sh_pwrite(fd, &h, sizeof(h), pos);
          chk = OBUCK_TRAILER;
          sh_pwrite(fd, &chk, 4, end-4);
          sh_pwrite(fd, &h, sizeof(h), pos);
          chk = OBUCK_TRAILER;
          sh_pwrite(fd, &chk, 4, end-4);

-         printf("*** replaced the invalid chunk by a DELETED bucket of size %d\n", end - pos);
+         printf("*** replaced the invalid chunk by a DELETED bucket of size %d\n", (uns)(end - pos));

        }
        }

+      else
+       printf("*** would mark %d bytes as DELETED\n", (uns)(end - pos));

       pos = end;
     }
  finish:
       pos = end;
     }
  finish:


@@ -219,52 +342,104 @@ fsck(int fix)
     exit(1);
 }
 
     exit(1);
 }
 

+static int
+shake_kibitz(struct obuck_header *old, oid_t new, byte *buck UNUSED)
+{
+  if (verbose)
+    {
+      printf("%08x -> ", old->oid);
+      if (new == OBUCK_OID_DELETED)
+       puts("DELETED");
+      else
+       printf("%08x\n", new);
+    }
+  return 1;
+}
+
+static void
+shake(void)
+{
+  obuck_init(1);
+  obuck_shakedown(shake_kibitz);
+  obuck_cleanup();
+}
+
+static void
+quickcheck(void)
+{
+  obuck_init(1);
+  obuck_cleanup();
+}
+

 int
 main(int argc, char **argv)
 {
   int i, op;
 int
 main(int argc, char **argv)
 {
   int i, op;

+  char *arg = NULL;
+  uns raw = 0;

 
 

+  log_init(NULL);

   op = 0;
   op = 0;

-  while ((i = getopt(argc, argv, "lLd:x:icfF")) != -1)
-    switch (i)
+  while ((i = cf_getopt(argc, argv, CF_SHORT_OPTS "lLd:x:i::cfFqrsv", CF_NO_LONG_OPTS, NULL)) != -1)
+    if (i == '?' || op)
+      help();
+    else if (i == 'v')
+      verbose++;
+    else if (i == 'r')
+      raw++;
+    else

       {
       {

-      case 'l':
-       list(0);
-       op++;
-       break;
-      case 'L':
-       list(1);
-       op++;
-       break;
-      case 'd':
-       delete(optarg);
-       op++;
-       break;
-      case 'x':
-       extract(optarg);
-       op++;
-       break;
-      case 'i':
-       insert();
-       op++;
-       break;
-      case 'c':
-       cat();
-       op++;
-       break;
-      case 'f':
-       fsck(0);
-       op++;
-       break;
-      case 'F':
-       fsck(1);
-       op++;
-       break;
-      default:
-       help();
+       op = i;
+       arg = optarg;

       }
       }

-  if (optind < argc || !op)
+  if (optind < argc)

     help();
 
     help();
 

+  if (!raw)
+  {
+    pool = mp_new(1<<14);
+    buck_buf = buck2obj_alloc();
+  }
+  switch (op)
+    {
+    case 'l':
+      list(0);
+      break;
+    case 'L':
+      list(1);
+      break;
+    case 'd':
+      delete(arg);
+      break;
+    case 'x':
+      extract(arg);
+      break;
+    case 'i':
+      insert(arg);
+      break;
+    case 'c':
+      cat();
+      break;
+    case 'f':
+      fsck(0);
+      break;
+    case 'F':
+      fsck(1);
+      break;
+    case 'q':
+      quickcheck();
+      break;
+    case 's':
+      shake();
+      break;
+    default:
+      help();
+    }
+  if (buck_buf)
+  {
+    buck2obj_free(buck_buf);
+    mp_delete(pool);
+  }
+

   return 0;
 }
   return 0;
 }