2 * Sherlock Library -- Bucket Manipulation Tool
4 * (c) 2001 Martin Mares <mj@ucw.cz>
5 * (c) 2004 Robert Spalek <robert@ucw.cz>
7 * This software may be freely distributed and used according to the terms
8 * of the GNU Lesser General Public License.
12 #include "lib/bucket.h"
13 #include "lib/fastbuf.h"
16 #include "lib/pools.h"
17 #include "lib/object.h"
18 #include "lib/buck2obj.h"
19 #include "lib/obj2buck.h"
20 #include "lib/lizard.h"
21 #include "charset/unistream.h"
30 static struct mempool *pool;
31 static struct buck2obj_buf *buck_buf;
37 Usage: buckettool [<options>] <command>\n\
42 -l\t\tlist all buckets\n\
43 -L\t\tlist all buckets including deleted ones\n\
44 -d <obj>\tdelete bucket\n\
45 -x <obj>\textract bucket\n\
46 -i[<type>]\tinsert buckets separated by blank lines\n\
47 -c\t\tconcatenate and dump all buckets\n\
48 -f\t\taudit bucket file structure\n\
49 -F\t\taudit and fix bucket file structure\n\
50 -q\t\tquick check of bucket file consistency\n\
51 -s\t\tshake down bucket file (without updating other structures!!!)\n\
53 -r\t\tdo not parse V33 buckets, but print the raw content\n\
62 oid_t o = strtoul(c, &e, 16);
64 die("Invalid object ID: %s", c);
71 struct obuck_header h;
74 if (obuck_find_first(&h, full))
77 if (h.oid == OBUCK_OID_DELETED)
78 printf("DELETED %6d\n", h.length);
80 printf("%08x %6d %08x\n", h.oid, h.length, h.type);
82 while (obuck_find_next(&h, full));
89 oid_t oid = parse_id(id);
96 dump_oattr(struct fastbuf *out, struct oattr *oa)
98 for (struct oattr *a = oa; a; a = a->same)
99 bprintf(out, "%c%s\n", a->attr, a->val);
103 dump_parsed_bucket(struct fastbuf *out, struct obuck_header *h, struct fastbuf *b)
105 struct odes *o = obj_read_bucket(buck_buf, pool, h->type, h->length, b, NULL);
107 bprintf(out, "Cannot parse bucket %x of type %x and length %d: %m\n", h->oid, h->type, h->length);
110 if (h->type < BUCKET_TYPE_V30)
112 for (struct oattr *oa = o->attrs; oa; oa = oa->next)
117 #define IS_HEADER(x) (x=='O' || x=='U')
118 for (struct oattr *oa = o->attrs; oa; oa = oa->next)
119 if (IS_HEADER(oa->attr))
122 for (struct oattr *oa = o->attrs; oa; oa = oa->next)
123 if (!IS_HEADER(oa->attr))
132 struct fastbuf *b, *out;
135 struct obuck_header h;
137 h.oid = parse_id(id);
139 obuck_find_by_oid(&h);
140 out = bfdopen_shared(1, 65536);
142 if (h.type < BUCKET_TYPE_V33 || !buck_buf)
144 while ((l = bread(b, buf, sizeof(buf))))
148 dump_parsed_bucket(out, &h, b);
154 #define GBUF_TYPE byte
155 #define GBUF_PREFIX(x) bb_##x
156 #include "lib/gbuf.h"
161 struct fastbuf *b, *in;
163 struct obuck_header h;
166 bb_t lizard_buf, compressed_buf;
168 bb_init(&lizard_buf);
169 bb_init(&compressed_buf);
171 type = BUCKET_TYPE_PLAIN;
172 else if (sscanf(arg, "%x", &type) != 1)
173 die("Type `%s' is not a hexadecimal number");
175 in = bfdopen_shared(0, 4096);
179 uns lizard_filled = 0;
182 while ((e = bgets(in, buf, sizeof(buf))))
186 if (in_body || type < BUCKET_TYPE_V30)
191 b = obuck_create(type);
192 if (type < BUCKET_TYPE_V33)
195 bwrite(b, buf, e-buf);
197 else if (in_body == 1)
202 else if (type == BUCKET_TYPE_V33 || !in_body)
204 bwrite_v33(b, buf[0], buf+1, e-buf-1);
208 uns want_len = lizard_filled + (e-buf) + 6 + LIZARD_NEEDS_CHARS; // +6 is the maximum UTF-8 length
209 bb_grow(&lizard_buf, want_len);
210 byte *ptr = lizard_buf.ptr + lizard_filled;
211 WRITE_V33(ptr, buf[0], buf+1, e-buf-1);
212 lizard_filled = ptr - lizard_buf.ptr;
215 if (in_body && type == BUCKET_TYPE_V33_LIZARD)
217 bputl(b, lizard_filled
218 #if 0 //TEST error resilience: write wrong length
222 uns want_len = lizard_filled * LIZARD_MAX_MULTIPLY + LIZARD_MAX_ADD;
223 bb_grow(&compressed_buf, want_len);
224 want_len = lizard_compress(lizard_buf.ptr, lizard_filled, compressed_buf.ptr);
225 #if 0 //TEST error resilience: tamper the compressed data by removing EOF
226 compressed_buf[want_len-1] = 1;
228 bwrite(b, compressed_buf.ptr, want_len);
232 obuck_create_end(b, &h);
233 printf("%08x %d %08x\n", h.oid, h.length, h.type);
237 bb_done(&lizard_buf);
238 bb_done(&compressed_buf);
246 struct obuck_header h;
247 struct fastbuf *b, *out;
251 out = bfdopen_shared(1, 65536);
252 while (b = obuck_slurp_pool(&h))
254 bprintf(out, "### %08x %6d %08x\n", h.oid, h.length, h.type);
255 if (h.type < BUCKET_TYPE_V33 || !buck_buf)
258 while ((l = bread(b, buf, sizeof(buf))))
261 lf = (buf[l-1] == '\n');
264 bprintf(out, "\n# <missing EOL>\n");
267 dump_parsed_bucket(out, &h, b);
277 struct obuck_header h, nh;
283 int fatal_errors = 0;
285 fd = sh_open(obuck_name, O_RDWR);
287 die("Unable to open the bucket file %s: %m", obuck_name);
290 oid = pos >> OBUCK_SHIFT;
291 i = sh_pread(fd, &h, sizeof(h), pos);
295 printf("%08x incomplete header\n", oid);
296 else if (h.magic == OBUCK_INCOMPLETE_MAGIC)
297 printf("%08x incomplete file\n", oid);
298 else if (h.magic != OBUCK_MAGIC)
299 printf("%08x invalid header magic\n", oid);
300 else if (h.oid != oid && h.oid != OBUCK_OID_DELETED)
301 printf("%08x invalid header backlink\n", oid);
304 end = (pos + sizeof(h) + h.length + 4 + OBUCK_ALIGN - 1) & ~(sh_off_t)(OBUCK_ALIGN - 1);
305 if (sh_pread(fd, &chk, 4, end-4) != 4)
306 printf("%08x missing trailer\n", oid);
307 else if (chk != OBUCK_TRAILER)
308 printf("%08x mismatched trailer\n", oid);
320 if (pos - end > 0x10000000)
322 printf("*** skipped for too long, giving up\n");
327 if (sh_pread(fd, &nh, sizeof(nh), end) != sizeof(nh))
329 printf("*** unable to find next header\n");
332 printf("*** truncating file\n");
333 sh_ftruncate(fd, pos);
336 printf("*** would truncate the file here\n");
340 while (nh.magic != OBUCK_MAGIC ||
341 (nh.oid != (oid_t)(end >> OBUCK_SHIFT) && nh.oid != OBUCK_OID_DELETED));
342 printf("*** match at oid %08x\n", (uns)(end >> OBUCK_SHIFT));
345 h.magic = OBUCK_MAGIC;
346 h.oid = OBUCK_OID_DELETED;
347 h.length = end - pos - sizeof(h) - 4;
348 sh_pwrite(fd, &h, sizeof(h), pos);
350 sh_pwrite(fd, &chk, 4, end-4);
351 printf("*** replaced the invalid chunk by a DELETED bucket of size %d\n", (uns)(end - pos));
354 printf("*** would mark %d bytes as DELETED\n", (uns)(end - pos));
359 if (!fix && errors || fatal_errors)
364 shake_kibitz(struct obuck_header *old, oid_t new, byte *buck UNUSED)
368 printf("%08x -> ", old->oid);
369 if (new == OBUCK_OID_DELETED)
372 printf("%08x\n", new);
381 obuck_shakedown(shake_kibitz);
393 main(int argc, char **argv)
401 while ((i = cf_getopt(argc, argv, CF_SHORT_OPTS "lLd:x:i::cfFqsvr", CF_NO_LONG_OPTS, NULL)) != -1)
418 pool = mp_new(1<<14);
419 buck_buf = buck2obj_alloc();
458 buck2obj_free(buck_buf);