X-Git-Url: http://mj.ucw.cz/gitweb/?a=blobdiff_plain;f=lib%2Fbuckettool.c;h=4b7847a566a49ec7341807c724eb062686492fd2;hb=beac4661ae8230dd1747cb4ebbfddc016183b140;hp=c93aaf51d2f3c75c053b65f613f2a44dce2ad2f7;hpb=836e8e5166980faab8814e74543b09ef1574647d;p=libucw.git diff --git a/lib/buckettool.c b/lib/buckettool.c index c93aaf51..4b7847a5 100644 --- a/lib/buckettool.c +++ b/lib/buckettool.c @@ -2,12 +2,23 @@ * Sherlock Library -- Bucket Manipulation Tool * * (c) 2001 Martin Mares + * (c) 2004 Robert Spalek + * + * This software may be freely distributed and used according to the terms + * of the GNU Lesser General Public License. */ #include "lib/lib.h" #include "lib/bucket.h" #include "lib/fastbuf.h" #include "lib/lfs.h" +#include "lib/conf.h" +#include "lib/pools.h" +#include "lib/object.h" +#include "lib/buck2obj.h" +#include "lib/obj2buck.h" +#include "lib/lizard.h" +#include "charset/unistream.h" #include #include @@ -15,21 +26,31 @@ #include #include +static int verbose; +static struct mempool *pool; +static struct buck2obj_buf *buck_buf; + static void help(void) { fprintf(stderr, "\ -Usage: buckettool \n\ +Usage: buckettool [] \n\ \n\ -Commands:\n\ +Options:\n" +CF_USAGE +"\nCommands:\n\ -l\t\tlist all buckets\n\ -L\t\tlist all buckets including deleted ones\n\ -d \tdelete bucket\n\ -x \textract bucket\n\ --i\t\tinsert bucket\n\ +-i[]\tinsert buckets separated by blank lines\n\ -c\t\tconcatenate and dump all buckets\n\ -f\t\taudit bucket file structure\n\ -F\t\taudit and fix bucket file structure\n\ +-q\t\tquick check of bucket file consistency\n\ +-s\t\tshake down bucket file (without updating other structures!!!)\n\ +-v\t\tbe verbose\n\ +-r\t\tdo not parse V33 buckets, but print the raw content\n\ "); exit(1); } @@ -56,7 +77,7 @@ list(int full) if (h.oid == OBUCK_OID_DELETED) printf("DELETED %6d\n", h.length); else - printf("%08x %6d %6d\n", h.oid, h.length, h.orig_length); + printf("%08x %6d %08x\n", h.oid, h.length, h.type); } while (obuck_find_next(&h, full)); obuck_cleanup(); @@ -71,10 +92,36 @@ delete(char *id) obuck_cleanup(); } +static inline void +dump_oattr(struct fastbuf *out, struct oattr *oa) +{ + for (struct oattr *a = oa; a; a = a->same) + bprintf(out, "%c%s\n", a->attr, a->val); +} + +static void +dump_parsed_bucket(struct fastbuf *out, struct obuck_header *h, struct fastbuf *b) +{ + struct odes *o = obj_read_bucket(buck_buf, pool, h->type, h->length, b, NULL); + if (!o) + bprintf(out, "Cannot parse bucket %x of type %x and length %d: %m\n", h->oid, h->type, h->length); + else + { +#define IS_HEADER(x) (x=='O' || x=='U') + for (struct oattr *oa = o->attrs; oa; oa = oa->next) + if (IS_HEADER(oa->attr)) + dump_oattr(out, oa); + bputc(out, '\n'); + for (struct oattr *oa = o->attrs; oa; oa = oa->next) + if (!IS_HEADER(oa->attr)) + dump_oattr(out, oa); + } +} + static void extract(char *id) { - struct fastbuf *b; + struct fastbuf *b, *out; byte buf[1024]; int l; struct obuck_header h; @@ -82,55 +129,136 @@ extract(char *id) h.oid = parse_id(id); obuck_init(0); obuck_find_by_oid(&h); + out = bfdopen_shared(1, 65536); b = obuck_fetch(); - while ((l = bread(b, buf, sizeof(buf)))) - fwrite(buf, 1, l, stdout); - obuck_fetch_end(b); + if (h.type < BUCKET_TYPE_V33 || !buck_buf) + { + while ((l = bread(b, buf, sizeof(buf)))) + bwrite(out, buf, l); + } + else + dump_parsed_bucket(out, &h, b); + bclose(b); + bclose(out); obuck_cleanup(); } +#define GBUF_TYPE byte +#define GBUF_PREFIX(x) bb_##x +#include "lib/gbuf.h" + static void -insert(void) +insert(byte *arg) { - struct fastbuf *b; - byte buf[1024]; - int l; + struct fastbuf *b, *in; + byte buf[4096]; struct obuck_header h; + byte *e; + u32 type; + bb_t lizard_buf, compressed_buf; + bb_init(&lizard_buf); + bb_init(&compressed_buf); + if (!arg) + type = BUCKET_TYPE_PLAIN; + else if (sscanf(arg, "%x", &type) != 1) + die("Type `%s' is not a hexadecimal number"); + + in = bfdopen_shared(0, 4096); obuck_init(1); - b = obuck_create(); - while ((l = fread(buf, 1, sizeof(buf), stdin))) - bwrite(b, buf, l); - obuck_create_end(b, &h); + do + { + uns lizard_filled = 0; + uns in_body = 0; + b = NULL; + while ((e = bgets(in, buf, sizeof(buf)))) + { + if (!buf[0]) + { + if (in_body || type < BUCKET_TYPE_V30) + break; + in_body = 1; + } + if (!b) + b = obuck_create(type); + if (type < BUCKET_TYPE_V33) + { + *e++ = '\n'; + bwrite(b, buf, e-buf); + } + else if (in_body == 1) + { + bputc(b, 0); + in_body = 2; + } + else if (type == BUCKET_TYPE_V33 || !in_body) + { + bwrite_v33(b, buf[0], buf+1, e-buf-1); + } + else + { + uns want_len = lizard_filled + (e-buf) + 6 + LIZARD_NEEDS_CHARS; // +6 is the maximum UTF-8 length + bb_grow(&lizard_buf, want_len); + byte *ptr = lizard_buf.ptr + lizard_filled; + WRITE_V33(ptr, buf[0], buf+1, e-buf-1); + lizard_filled = ptr - lizard_buf.ptr; + } + } + if (in_body && type == BUCKET_TYPE_V33_LIZARD) + { + bputl(b, lizard_filled +#if 0 //TEST error resilience: write wrong length + +1 +#endif + ); + uns want_len = lizard_filled * LIZARD_MAX_MULTIPLY + LIZARD_MAX_ADD; + bb_grow(&compressed_buf, want_len); + want_len = lizard_compress(lizard_buf.ptr, lizard_filled, compressed_buf.ptr); +#if 0 //TEST error resilience: tamper the compressed data by removing EOF + compressed_buf[want_len-1] = 1; +#endif + bwrite(b, compressed_buf.ptr, want_len); + } + if (b) + { + obuck_create_end(b, &h); + printf("%08x %d %08x\n", h.oid, h.length, h.type); + } + } + while (e); + bb_done(&lizard_buf); + bb_done(&compressed_buf); obuck_cleanup(); - printf("%08x %d %d\n", h.oid, h.length, h.orig_length); + bclose(in); } static void cat(void) { struct obuck_header h; - struct fastbuf *b; + struct fastbuf *b, *out; byte buf[1024]; - int l, lf; obuck_init(0); - if (obuck_find_first(&h, 0)) - do + out = bfdopen_shared(1, 65536); + while (b = obuck_slurp_pool(&h)) + { + bprintf(out, "### %08x %6d %08x\n", h.oid, h.length, h.type); + if (h.type < BUCKET_TYPE_V33 || !buck_buf) { - printf("### %08x %6d %6d\n", h.oid, h.length, h.orig_length); - b = obuck_fetch(); - lf = 1; + int lf = 1, l; while ((l = bread(b, buf, sizeof(buf)))) - { - fwrite(buf, 1, l, stdout); - lf = (buf[l-1] == '\n'); - } - obuck_fetch_end(b); + { + bwrite(out, buf, l); + lf = (buf[l-1] == '\n'); + } if (!lf) - printf("\n# \n"); + bprintf(out, "\n# \n"); } - while (obuck_find_next(&h, 0)); + else + dump_parsed_bucket(out, &h, b); + } + bclose(out); obuck_cleanup(); } @@ -143,8 +271,10 @@ fsck(int fix) sh_off_t end; oid_t oid; u32 chk; + int errors = 0; + int fatal_errors = 0; - fd = open(obuck_name, O_RDWR); + fd = sh_open(obuck_name, O_RDWR); if (fd < 0) die("Unable to open the bucket file %s: %m", obuck_name); for(;;) @@ -168,16 +298,21 @@ fsck(int fix) printf("%08x missing trailer\n", oid); else if (chk != OBUCK_TRAILER) printf("%08x mismatched trailer\n", oid); - /* OK */ - pos = end; - continue; + else + { + /* OK */ + pos = end; + continue; + } } + errors++; end = pos; do { if (pos - end > 0x10000000) { printf("*** skipped for too long, giving up\n"); + fatal_errors++; goto finish; } end += OBUCK_ALIGN; @@ -187,7 +322,7 @@ fsck(int fix) if (fix) { printf("*** truncating file\n"); - ftruncate(fd, pos); + sh_ftruncate(fd, pos); } else printf("*** would truncate the file here\n"); @@ -196,69 +331,125 @@ fsck(int fix) } while (nh.magic != OBUCK_MAGIC || (nh.oid != (oid_t)(end >> OBUCK_SHIFT) && nh.oid != OBUCK_OID_DELETED)); - printf("*** match at oid %08x\n", end >> OBUCK_SHIFT); + printf("*** match at oid %08x\n", (uns)(end >> OBUCK_SHIFT)); if (fix) { h.magic = OBUCK_MAGIC; h.oid = OBUCK_OID_DELETED; - h.length = h.orig_length = end - pos - sizeof(h) - 4; + h.length = end - pos - sizeof(h) - 4; sh_pwrite(fd, &h, sizeof(h), pos); chk = OBUCK_TRAILER; sh_pwrite(fd, &chk, 4, end-4); - printf("*** replaced the invalid chunk by a DELETED bucket of size %d\n", end - pos); + printf("*** replaced the invalid chunk by a DELETED bucket of size %d\n", (uns)(end - pos)); } + else + printf("*** would mark %d bytes as DELETED\n", (uns)(end - pos)); pos = end; } finish: close(fd); + if (!fix && errors || fatal_errors) + exit(1); +} + +static int +shake_kibitz(struct obuck_header *old, oid_t new, byte *buck UNUSED) +{ + if (verbose) + { + printf("%08x -> ", old->oid); + if (new == OBUCK_OID_DELETED) + puts("DELETED"); + else + printf("%08x\n", new); + } + return 1; +} + +static void +shake(void) +{ + obuck_init(1); + obuck_shakedown(shake_kibitz); + obuck_cleanup(); +} + +static void +quickcheck(void) +{ + obuck_init(1); + obuck_cleanup(); } int main(int argc, char **argv) { int i, op; + char *arg = NULL; + uns raw = 0; + log_init(NULL); op = 0; - while ((i = getopt(argc, argv, "lLd:x:icfF")) != -1) - switch (i) + while ((i = cf_getopt(argc, argv, CF_SHORT_OPTS "lLd:x:i::cfFqsvr", CF_NO_LONG_OPTS, NULL)) != -1) + if (i == '?' || op) + help(); + else if (i == 'v') + verbose++; + else if (i == 'r') + raw++; + else { - case 'l': - list(0); - op++; - break; - case 'L': - list(1); - op++; - break; - case 'd': - delete(optarg); - op++; - break; - case 'x': - extract(optarg); - op++; - break; - case 'i': - insert(); - op++; - break; - case 'c': - cat(); - op++; - break; - case 'f': - fsck(0); - op++; - break; - case 'F': - fsck(1); - op++; - break; - default: - help(); + op = i; + arg = optarg; } - if (optind < argc || !op) + if (optind < argc) help(); + if (!raw) + { + pool = mp_new(1<<14); + buck_buf = buck2obj_alloc(); + } + switch (op) + { + case 'l': + list(0); + break; + case 'L': + list(1); + break; + case 'd': + delete(arg); + break; + case 'x': + extract(arg); + break; + case 'i': + insert(arg); + break; + case 'c': + cat(); + break; + case 'f': + fsck(0); + break; + case 'F': + fsck(1); + break; + case 'q': + quickcheck(); + break; + case 's': + shake(); + break; + default: + help(); + } + if (buck_buf) + { + buck2obj_free(buck_buf); + mp_delete(pool); + } + return 0; }