/*
* Sherlock Library -- Object Buckets
*
- * (c) 2001--2002 Martin Mares <mj@ucw.cz>
+ * (c) 2001--2003 Martin Mares <mj@ucw.cz>
*
* This software may be freely distributed and used according to the terms
* of the GNU Lesser General Public License.
byte *obuck_name = "not/configured";
static uns obuck_io_buflen = 65536;
static int obuck_shake_buflen = 1048576;
+static uns obuck_slurp_buflen = 65536;
static struct cfitem obuck_config[] = {
{ "Buckets", CT_SECTION, NULL },
{ "BucketFile", CT_STRING, &obuck_name },
{ "BufSize", CT_INT, &obuck_io_buflen },
{ "ShakeBufSize", CT_INT, &obuck_shake_buflen },
+ { "SlurpBufSize", CT_INT, &obuck_slurp_buflen },
{ NULL, CT_STOP, NULL }
};
{
}
+oid_t
+obuck_predict_last_oid(void)
+{
+ sh_off_t size = sh_seek(obuck_fd, 0, SEEK_END);
+ return size >> OBUCK_SHIFT;
+}
+
struct fastbuf *
-obuck_create(void)
+obuck_create(u32 type)
{
obuck_lock_write();
bflush(obuck_fb);
obuck_broken("Misaligned file");
obuck_hdr.magic = OBUCK_INCOMPLETE_MAGIC;
obuck_hdr.oid = bucket_start >> OBUCK_SHIFT;
- obuck_hdr.length = obuck_hdr.orig_length = 0;
+ obuck_hdr.length = 0;
+ obuck_hdr.type = type;
bucket_current = bucket_start;
bwrite(obuck_fb, &obuck_hdr, sizeof(obuck_hdr));
obuck_fb->pos = -sizeof(obuck_hdr);
{
int pad;
obuck_hdr.magic = OBUCK_MAGIC;
- obuck_hdr.length = obuck_hdr.orig_length = btell(obuck_fb);
+ obuck_hdr.length = btell(obuck_fb);
pad = (OBUCK_ALIGN - sizeof(obuck_hdr) - obuck_hdr.length - 4) & (OBUCK_ALIGN - 1);
while (pad--)
bputc(obuck_fb, 0);
obuck_unlock();
}
+/*** Fast reading of the whole pool ***/
+
+static struct fastbuf *obuck_rpf;
+
+static int
+obuck_slurp_refill(struct fastbuf *f)
+{
+ uns l;
+
+ if (!obuck_remains)
+ return 0;
+ l = bdirect_read_prepare(obuck_rpf, &f->buffer);
+ if (!l)
+ obuck_broken("Incomplete object");
+ l = MIN(l, obuck_remains);
+ bdirect_read_commit(obuck_rpf, f->buffer + l);
+ obuck_remains -= l;
+ f->bptr = f->buffer;
+ f->bufend = f->bstop = f->buffer + l;
+ return 1;
+}
+
+struct fastbuf *
+obuck_slurp_pool(struct obuck_header *hdrp)
+{
+ static struct fastbuf limiter;
+ uns l;
+
+ do
+ {
+ if (!obuck_rpf)
+ {
+ obuck_lock_read();
+ obuck_rpf = bopen(obuck_name, O_RDONLY, obuck_slurp_buflen);
+ }
+ else
+ {
+ bsetpos(obuck_rpf, bucket_current - 4);
+ if (bgetl(obuck_rpf) != OBUCK_TRAILER)
+ obuck_broken("Missing trailer");
+ }
+ bucket_start = btell(obuck_rpf);
+ l = bread(obuck_rpf, hdrp, sizeof(struct obuck_header));
+ if (!l)
+ {
+ bclose(obuck_rpf);
+ obuck_rpf = NULL;
+ obuck_unlock();
+ return NULL;
+ }
+ if (l != sizeof(struct obuck_header))
+ obuck_broken("Short header read");
+ if (hdrp->magic != OBUCK_MAGIC)
+ obuck_broken("Missing magic number");
+ bucket_current = (bucket_start + sizeof(obuck_hdr) + hdrp->length +
+ 4 + OBUCK_ALIGN - 1) & ~((sh_off_t)(OBUCK_ALIGN - 1));
+ }
+ while (hdrp->oid == OBUCK_OID_DELETED);
+ if (obuck_get_pos(hdrp->oid) != bucket_start)
+ obuck_broken("Invalid backlink");
+ obuck_remains = hdrp->length;
+ limiter.bptr = limiter.bstop = limiter.buffer = limiter.bufend = NULL;
+ limiter.name = "Bucket";
+ limiter.pos = 0;
+ limiter.refill = obuck_slurp_refill;
+ return &limiter;
+}
+
/*** Shakedown ***/
void
goto broken;
}
l = (sizeof(struct obuck_header) + rhdr->length + 4 + OBUCK_ALIGN - 1) & ~(OBUCK_ALIGN-1);
+ if (l > obuck_shake_buflen)
+ {
+ if (rhdr->oid != OBUCK_OID_DELETED)
+ {
+ msg = "bucket longer than ShakeBufSize";
+ goto broken;
+ }
+ rstart = bucket_start + l;
+ roff = 0;
+ rsize = 0;
+ goto reread;
+ }
if (rsize - roff < l)
goto reread;
if (GET_U32(rbuf + roff + l - 4) != OBUCK_TRAILER)
return;
broken:
- log(L_ERROR, "Error during object pool shakedown: %s (pos=%Ld), gathering debris", msg, (long long) bucket_start);
+ log(L_ERROR, "Error during object pool shakedown: %s (pos=%Ld, id=%x), gathering debris", msg, (long long) bucket_start, (uns)(bucket_start >> OBUCK_SHIFT));
if (woff)
{
sh_pwrite(obuck_fd, wbuf, woff, wstart);
obuck_hdr.length = bucket_start - wstart - sizeof(obuck_hdr) - 4;
else
obuck_hdr.length = 0x40000000 - sizeof(obuck_hdr) - 4;
- obuck_hdr.orig_length = obuck_hdr.length;
sh_pwrite(obuck_fd, &obuck_hdr, sizeof(obuck_hdr), wstart);
wstart += sizeof(obuck_hdr) + obuck_hdr.length + 4;
sh_pwrite(obuck_fd, &check, 4, wstart-4);
for(i=0; i<LEN(j); i++)
bputc(b, (i+j) % 256);
obuck_create_end(b, &h);
- printf("Writing %08x %d -> %d\n", h.oid, h.orig_length, h.length);
+ printf("Writing %08x %d\n", h.oid, h.length);
ids[j] = h.oid;
}
for(j=0; j<COUNT; j++)
h.oid = ids[j];
obuck_find_by_oid(&h);
b = obuck_fetch();
- printf("Reading %08x %d -> %d\n", h.oid, h.orig_length, h.length);
- if (h.orig_length != LEN(j))
+ printf("Reading %08x %d\n", h.oid, h.length);
+ if (h.length != LEN(j))
die("Invalid length");
- for(i=0; i<h.orig_length; i++)
+ for(i=0; i<h.length; i++)
if ((unsigned) bgetc(b) != (i+j) % 256)
die("Contents mismatch");
if (bgetc(b) != EOF)
if (obuck_find_first(&h, 0))
do
{
- printf("<<< %08x\t%d\n", h.oid, h.orig_length);
+ printf("<<< %08x\t%d\n", h.oid, h.length);
cnt--;
}
while (obuck_find_next(&h, 0));