2 * Sherlock Library -- Object Buckets
4 * (c) 2001 Martin Mares <mj@ucw.cz>
8 #include "lib/bucket.h"
9 #include "lib/fastbuf.h"
20 static unsigned int obuck_remains, obuck_check_pad;
21 static struct fastbuf *obuck_fb;
22 static struct obuck_header obuck_hdr;
23 static sh_off_t bucket_start;
25 /*** Configuration ***/
27 byte *obuck_name = "not/configured";
28 static int obuck_io_buflen = 65536;
29 static int obuck_shake_buflen = 1048576;
31 static struct cfitem obuck_config[] = {
32 { "Buckets", CT_SECTION, NULL },
33 { "BucketFile", CT_STRING, &obuck_name },
34 { "BufSize", CT_INT, &obuck_io_buflen },
35 { "ShakeBufSize", CT_INT, &obuck_shake_buflen },
36 { NULL, CT_STOP, NULL }
39 static void CONSTRUCTOR obuck_init_config(void)
41 cf_register(obuck_config);
44 /*** Internal operations ***/
47 obuck_broken(char *msg)
49 die("Object pool corrupted: %s (pos=%Lx)", msg, (long long) bucket_start);
53 * Unfortunately we cannot use flock() here since it happily permits
54 * locking a shared fd (e.g., after fork()) multiple times. The fcntl
55 * locks are very ugly and they don't support 64-bit offsets, but we
56 * can work around the problem by always locking the first header
61 obuck_do_lock(int type)
66 fl.l_whence = SEEK_SET;
68 fl.l_len = sizeof(struct obuck_header);
69 if (fcntl(obuck_fd, F_SETLKW, &fl) < 0)
70 die("fcntl lock: %m");
76 obuck_do_lock(F_RDLCK);
80 obuck_lock_write(void)
82 obuck_do_lock(F_WRLCK);
88 obuck_do_lock(F_UNLCK);
91 /*** FastIO emulation ***/
93 /* We need to use pread/pwrite since we work on fd's shared between processes */
96 obuck_fb_refill(struct fastbuf *f)
98 unsigned limit = (f->buflen < obuck_remains) ? f->buflen : obuck_remains;
99 unsigned size = (limit == obuck_remains) ? (limit+obuck_check_pad+4) : limit;
104 l = sh_pread(f->fd, f->buffer, size, f->fdpos);
106 die("Error reading bucket: %m");
107 if ((unsigned) l != size)
108 obuck_broken("Short read");
110 f->bstop = f->buffer + limit;
113 obuck_remains -= limit;
114 if (!obuck_remains) /* Should check the trailer */
116 if (GET_U32(f->buffer + size - 4) != OBUCK_TRAILER)
117 obuck_broken("Missing trailer");
123 obuck_fb_spout(struct fastbuf *f)
125 int l = f->bptr - f->buffer;
130 int z = sh_pwrite(f->fd, c, l, f->fdpos);
132 die("Error writing bucket: %m");
142 obuck_fb_close(struct fastbuf *f)
147 /*** Exported functions ***/
150 obuck_init(int writeable)
155 obuck_fd = sh_open(obuck_name, (writeable ? O_RDWR | O_CREAT : O_RDONLY), 0666);
157 die("Unable to open bucket file %s: %m", obuck_name);
158 obuck_fb = b = xmalloc_zero(sizeof(struct fastbuf) + obuck_io_buflen + OBUCK_ALIGN + 4);
159 b->buflen = obuck_io_buflen;
160 b->buffer = (char *)(b+1);
161 b->bptr = b->bstop = b->buffer;
162 b->bufend = b->buffer + obuck_io_buflen;
165 b->refill = obuck_fb_refill;
166 b->spout = obuck_fb_spout;
167 b->close = obuck_fb_close;
169 size = sh_seek(obuck_fd, 0, SEEK_END);
172 /* If the bucket pool is not empty, check consistency of its end */
174 bucket_start = size - 4; /* for error reporting */
175 if (sh_pread(obuck_fd, &check, 4, size-4) != 4 ||
176 check != OBUCK_TRAILER)
177 obuck_broken("Missing trailer of last object");
198 struct fastbuf *b = obuck_fb;
200 bucket_start = obuck_get_pos(oid);
202 if (sh_pread(obuck_fd, &obuck_hdr, sizeof(obuck_hdr), bucket_start) != sizeof(obuck_hdr))
203 obuck_broken("Short header read");
204 b->fdpos = bucket_start + sizeof(obuck_hdr);
205 if (obuck_hdr.magic != OBUCK_MAGIC)
206 obuck_broken("Missing magic number");
207 if (obuck_hdr.oid == OBUCK_OID_DELETED)
208 obuck_broken("Access to deleted bucket");
209 if (obuck_hdr.oid != oid)
210 obuck_broken("Invalid backlink");
214 obuck_find_by_oid(struct obuck_header *hdrp)
216 oid_t oid = hdrp->oid;
218 ASSERT(oid < OBUCK_OID_FIRST_SPECIAL);
222 memcpy(hdrp, &obuck_hdr, sizeof(obuck_hdr));
226 obuck_find_first(struct obuck_header *hdrp, int full)
230 return obuck_find_next(hdrp, full);
234 obuck_find_next(struct obuck_header *hdrp, int full)
237 struct fastbuf *b = obuck_fb;
242 bucket_start = (bucket_start + sizeof(obuck_hdr) + obuck_hdr.length +
243 4 + OBUCK_ALIGN - 1) & ~((sh_off_t)(OBUCK_ALIGN - 1));
246 c = sh_pread(obuck_fd, &obuck_hdr, sizeof(obuck_hdr), bucket_start);
250 if (c != sizeof(obuck_hdr))
251 obuck_broken("Short header read");
252 b->fdpos = bucket_start + sizeof(obuck_hdr);
253 if (obuck_hdr.magic != OBUCK_MAGIC)
254 obuck_broken("Missing magic number");
255 if (obuck_hdr.oid != OBUCK_OID_DELETED || full)
257 memcpy(hdrp, &obuck_hdr, sizeof(obuck_hdr));
266 obuck_remains = obuck_hdr.length;
267 obuck_check_pad = (OBUCK_ALIGN - sizeof(obuck_hdr) - obuck_hdr.length - 4) & (OBUCK_ALIGN - 1);
272 obuck_fetch_end(struct fastbuf *b UNUSED)
281 bucket_start = sh_seek(obuck_fd, 0, SEEK_END);
282 if (bucket_start & (OBUCK_ALIGN - 1))
283 obuck_broken("Misaligned file");
284 obuck_hdr.magic = OBUCK_INCOMPLETE_MAGIC;
285 obuck_hdr.oid = bucket_start >> OBUCK_SHIFT;
286 obuck_hdr.length = obuck_hdr.orig_length = 0;
287 obuck_fb->fdpos = obuck_fb->pos = bucket_start;
288 bwrite(obuck_fb, &obuck_hdr, sizeof(obuck_hdr));
293 obuck_create_end(struct fastbuf *b UNUSED, struct obuck_header *hdrp)
296 obuck_hdr.magic = OBUCK_MAGIC;
297 obuck_hdr.length = obuck_hdr.orig_length = btell(obuck_fb) - bucket_start - sizeof(obuck_hdr);
298 pad = (OBUCK_ALIGN - sizeof(obuck_hdr) - obuck_hdr.length - 4) & (OBUCK_ALIGN - 1);
301 bputl(obuck_fb, OBUCK_TRAILER);
303 ASSERT(!(btell(obuck_fb) & (OBUCK_ALIGN - 1)));
304 sh_pwrite(obuck_fd, &obuck_hdr, sizeof(obuck_hdr), bucket_start);
306 memcpy(hdrp, &obuck_hdr, sizeof(obuck_hdr));
310 obuck_delete(oid_t oid)
314 obuck_hdr.oid = OBUCK_OID_DELETED;
315 sh_pwrite(obuck_fd, &obuck_hdr, sizeof(obuck_hdr), bucket_start);
322 obuck_shakedown(int (*kibitz)(struct obuck_header *old, oid_t new, byte *buck))
325 sh_off_t rstart, wstart, w_bucket_start;
326 int roff, woff, rsize, l;
327 struct obuck_header *rhdr, *whdr;
329 rbuf = xmalloc(obuck_shake_buflen);
330 wbuf = xmalloc(obuck_shake_buflen);
332 roff = woff = rsize = 0;
334 /* We need to be the only accessor, all the object ID's are becoming invalid */
339 bucket_start = rstart + roff;
340 w_bucket_start = wstart + woff;
341 if (rsize - roff < OBUCK_ALIGN)
343 rhdr = (struct obuck_header *)(rbuf + roff);
344 if (rhdr->magic != OBUCK_MAGIC ||
345 rhdr->oid != OBUCK_OID_DELETED && rhdr->oid != (bucket_start >> OBUCK_SHIFT))
346 obuck_broken("header mismatch during shakedown");
347 l = (sizeof(struct obuck_header) + rhdr->length + 4 + OBUCK_ALIGN - 1) & ~(OBUCK_ALIGN-1);
348 if (rsize - roff < l)
350 if (GET_U32(rbuf + roff + l - 4) != OBUCK_TRAILER)
351 obuck_broken("missing trailer during shakedown");
352 if (rhdr->oid != OBUCK_OID_DELETED &&
353 kibitz(rhdr, w_bucket_start >> OBUCK_SHIFT, (byte *)(rhdr+1)))
355 if (bucket_start == w_bucket_start)
357 /* No copying needed now nor ever in the past, hence woff==0 */
362 if (obuck_shake_buflen - woff < l)
364 if (sh_pwrite(obuck_fd, wbuf, woff, wstart) != woff)
365 die("obuck_shakedown write failed: %m");
369 whdr = (struct obuck_header *)(wbuf+woff);
370 memcpy(whdr, rhdr, l);
371 whdr->oid = w_bucket_start >> OBUCK_SHIFT;
376 kibitz(rhdr, OBUCK_OID_DELETED, NULL);
383 memmove(rbuf, rbuf+roff, rsize-roff);
388 l = sh_pread(obuck_fd, rbuf+rsize, obuck_shake_buflen-rsize, rstart+rsize);
390 die("obuck_shakedown read error: %m");
395 obuck_broken("unexpected EOF during shakedown");
401 if (sh_pwrite(obuck_fd, wbuf, woff, wstart) != woff)
402 die("obuck_shakedown write failed: %m");
405 sh_ftruncate(obuck_fd, wstart);
419 #define LEN(i) ((259309*(i))%MAXLEN)
421 int main(int argc, char **argv)
424 unsigned int i, j, cnt;
425 struct obuck_header h;
429 if (cf_getopt(argc, argv, CF_SHORT_OPTS, CF_NO_LONG_OPTS, NULL) >= 0 ||
431 die("This program supports only the following command-line arguments:\n" CF_USAGE);
435 for(j=0; j<COUNT; j++)
438 for(i=0; i<LEN(j); i++)
439 bputc(b, (i+j) % 256);
440 obuck_create_end(b, &h);
441 printf("Writing %08x %d -> %d\n", h.oid, h.orig_length, h.length);
444 for(j=0; j<COUNT; j++)
445 if (j % 100 < KILLPERC)
447 printf("Deleting %08x\n", ids[j]);
448 obuck_delete(ids[j]);
451 for(j=0; j<COUNT; j++)
452 if (j % 100 >= KILLPERC)
456 obuck_find_by_oid(&h);
458 printf("Reading %08x %d -> %d\n", h.oid, h.orig_length, h.length);
459 if (h.orig_length != LEN(j))
460 die("Invalid length");
461 for(i=0; i<h.orig_length; i++)
462 if ((unsigned) bgetc(b) != (i+j) % 256)
463 die("Contents mismatch");
468 if (obuck_find_first(&h, 0))
471 printf("<<< %08x\t%d\n", h.oid, h.orig_length);
474 while (obuck_find_next(&h, 0));
476 die("Walk mismatch");