From: Robert Spalek Date: Mon, 28 Jun 2004 09:50:14 +0000 (+0000) Subject: - buckettool updated to extract, cat, and create all kinds of buckets X-Git-Tag: holmes-import~991 X-Git-Url: http://mj.ucw.cz/gitweb/?a=commitdiff_plain;h=95d43aff618ef84172e8893bae7727b496865007;p=libucw.git - buckettool updated to extract, cat, and create all kinds of buckets - added parameter -r for raw (non-parsed) output --- diff --git a/lib/Makefile b/lib/Makefile index 54ebe46b..9cee1277 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -26,7 +26,7 @@ obj/lib/hashfunc.o obj/lib/hashfunc.oo: CFLAGS += -funroll-loops obj/lib/db-test: obj/lib/db-test.o $(LIBSH) obj/lib/db-tool: obj/lib/db-tool.o $(LIBSH) -obj/lib/buckettool: obj/lib/buckettool.o $(LIBSH) +obj/lib/buckettool: obj/lib/buckettool.o $(LIBCHARSET) $(LIBSH) obj/lib/conf-test: obj/lib/conf-test.o $(LIBSH) obj/lib/sort-test: obj/lib/sort-test.o $(LIBSH) obj/lib/lfs-test: obj/lib/lfs-test.o $(LIBSH) diff --git a/lib/buckettool.c b/lib/buckettool.c index 179e3d98..81b252f8 100644 --- a/lib/buckettool.c +++ b/lib/buckettool.c @@ -2,6 +2,7 @@ * Sherlock Library -- Bucket Manipulation Tool * * (c) 2001 Martin Mares + * (c) 2004 Robert Spalek * * This software may be freely distributed and used according to the terms * of the GNU Lesser General Public License. @@ -12,6 +13,12 @@ #include "lib/fastbuf.h" #include "lib/lfs.h" #include "lib/conf.h" +#include "lib/pools.h" +#include "lib/object.h" +#include "lib/buck2obj.h" +#include "lib/obj2buck.h" +#include "lib/lizard.h" +#include "charset/unistream.h" #include #include @@ -20,6 +27,8 @@ #include static int verbose; +static struct mempool *pool; +static struct buck2obj_buf *buck_buf; static void help(void) @@ -41,6 +50,7 @@ CF_USAGE -q\t\tquick check of bucket file consistency\n\ -s\t\tshake down bucket file (without updating other structures!!!)\n\ -v\t\tbe verbose\n\ +-r\t\tdo not parse V33 buckets, but print the raw content\n\ "); exit(1); } @@ -82,10 +92,45 @@ delete(char *id) obuck_cleanup(); } +static inline void +dump_oattr(struct fastbuf *out, struct oattr *oa) +{ + for (struct oattr *a = oa; a; a = a->same) + bprintf(out, "%c%s\n", a->attr, a->val); +} + +static void +dump_parsed_bucket(struct fastbuf *out, struct obuck_header *h, struct fastbuf *b) +{ + mp_flush(pool); + struct odes *o = obj_read_bucket(buck_buf, h->type, h->length, b, NULL); + if (!o) + bprintf(out, "Cannot parse bucket %x of type %x and length %d: %m\n", h->oid, h->type, h->length); + else + { + if (h->type < BUCKET_TYPE_V30) + { + for (struct oattr *oa = o->attrs; oa; oa = oa->next) + dump_oattr(out, oa); + } + else + { +#define IS_HEADER(x) (x=='O' || x=='U') + for (struct oattr *oa = o->attrs; oa; oa = oa->next) + if (IS_HEADER(oa->attr)) + dump_oattr(out, oa); + bputc(out, '\n'); + for (struct oattr *oa = o->attrs; oa; oa = oa->next) + if (!IS_HEADER(oa->attr)) + dump_oattr(out, oa); + } + } +} + static void extract(char *id) { - struct fastbuf *b; + struct fastbuf *b, *out; byte buf[1024]; int l; struct obuck_header h; @@ -93,13 +138,24 @@ extract(char *id) h.oid = parse_id(id); obuck_init(0); obuck_find_by_oid(&h); + out = bfdopen_shared(1, 65536); b = obuck_fetch(); - while ((l = bread(b, buf, sizeof(buf)))) - fwrite(buf, 1, l, stdout); + if (h.type < BUCKET_TYPE_V33 || !buck_buf) + { + while ((l = bread(b, buf, sizeof(buf)))) + bwrite(out, buf, l); + } + else + dump_parsed_bucket(out, &h, b); bclose(b); + bclose(out); obuck_cleanup(); } +#define GBUF_TYPE byte +#define GBUF_PREFIX(x) bb_##x +#include "lib/gbuf.h" + static void insert(byte *arg) { @@ -108,7 +164,10 @@ insert(byte *arg) struct obuck_header h; byte *e; u32 type; + bb_t lizard_buf, compressed_buf; + bb_init(&lizard_buf); + bb_init(&compressed_buf); if (!arg) type = BUCKET_TYPE_PLAIN; else if (sscanf(arg, "%x", &type) != 1) @@ -118,14 +177,57 @@ insert(byte *arg) obuck_init(1); do { + uns lizard_filled = 0; + uns in_body = 0; b = NULL; - while ((e = bgets(in, buf, sizeof(buf))) && buf[0]) + while ((e = bgets(in, buf, sizeof(buf)))) { - *e++ = '\n'; + if (!buf[0]) + { + if (in_body || type < BUCKET_TYPE_V30) + break; + in_body = 1; + } if (!b) b = obuck_create(type); - bwrite(b, buf, e-buf); + if (type < BUCKET_TYPE_V33) + { + *e++ = '\n'; + bwrite(b, buf, e-buf); + } + else if (in_body == 1) + { + bputc(b, 0); + in_body = 2; + } + else if (type == BUCKET_TYPE_V33 || !in_body) + { + bwrite_v33(b, buf[0], buf+1, e-buf-1); + } + else + { + uns want_len = lizard_filled + (e-buf) + 6 + LIZARD_NEEDS_CHARS; // +6 is the maximum UTF-8 length + bb_grow(&lizard_buf, want_len); + byte *ptr = lizard_buf.ptr + lizard_filled; + WRITE_V33(ptr, buf[0], buf+1, e-buf-1); + lizard_filled = ptr - lizard_buf.ptr; + } } + if (in_body && type == BUCKET_TYPE_V33_LIZARD) + { + bputl(b, lizard_filled +#if 0 //TEST error resilience: write wrong length + +1 +#endif + ); + uns want_len = lizard_filled * LIZARD_MAX_MULTIPLY + LIZARD_MAX_ADD; + bb_grow(&compressed_buf, want_len); + want_len = lizard_compress(lizard_buf.ptr, lizard_filled, compressed_buf.ptr); +#if 0 //TEST error resilience: tamper the compressed data by removing EOF + compressed_buf[want_len-1] = 1; +#endif + bwrite(b, compressed_buf.ptr, want_len); + } if (b) { obuck_create_end(b, &h); @@ -133,6 +235,8 @@ insert(byte *arg) } } while (e); + bb_done(&lizard_buf); + bb_done(&compressed_buf); obuck_cleanup(); bclose(in); } @@ -143,21 +247,25 @@ cat(void) struct obuck_header h; struct fastbuf *b, *out; byte buf[1024]; - int l, lf; obuck_init(0); out = bfdopen_shared(1, 65536); while (b = obuck_slurp_pool(&h)) { bprintf(out, "### %08x %6d %08x\n", h.oid, h.length, h.type); - lf = 1; - while ((l = bread(b, buf, sizeof(buf)))) + if (h.type < BUCKET_TYPE_V33 || !buck_buf) + { + int lf = 1, l; + while ((l = bread(b, buf, sizeof(buf)))) { bwrite(out, buf, l); lf = (buf[l-1] == '\n'); } - if (!lf) - bprintf(out, "\n# \n"); + if (!lf) + bprintf(out, "\n# \n"); + } + else + dump_parsed_bucket(out, &h, b); } bclose(out); obuck_cleanup(); @@ -287,14 +395,17 @@ main(int argc, char **argv) { int i, op; char *arg = NULL; + uns raw = 0; log_init(NULL); op = 0; - while ((i = cf_getopt(argc, argv, CF_SHORT_OPTS "lLd:x:i::cfFqsv", CF_NO_LONG_OPTS, NULL)) != -1) + while ((i = cf_getopt(argc, argv, CF_SHORT_OPTS "lLd:x:i::cfFqsvr", CF_NO_LONG_OPTS, NULL)) != -1) if (i == '?' || op) help(); else if (i == 'v') verbose++; + else if (i == 'r') + raw++; else { op = i; @@ -303,6 +414,11 @@ main(int argc, char **argv) if (optind < argc) help(); + if (!raw) + { + pool = mp_new(1<<14); + buck_buf = buck2obj_alloc(pool); + } switch (op) { case 'l': @@ -338,6 +454,11 @@ main(int argc, char **argv) default: help(); } + if (buck_buf) + { + buck2obj_free(buck_buf); + mp_delete(pool); + } return 0; }