/*
- * Bucket -> Object converter
+ * Generating Objects from Buckets
*
* (c) 2004, Robert Spalek <robert@ucw.cz>
+ * (c) 2004, Martin Mares <mj@ucw.cz>
*/
+#undef LOCAL_DEBUG
+
#include "lib/lib.h"
#include "lib/unaligned.h"
-#include "lib/pools.h"
+#include "lib/mempool.h"
#include "lib/fastbuf.h"
-#include "charset/unicode.h"
+#include "lib/unicode.h"
#include "lib/object.h"
#include "lib/bucket.h"
#include "lib/lizard.h"
-#include "lib/buck2obj.h"
+#include "lib/bbuf.h"
+#include "lib/ff-utf8.h"
#include <stdlib.h>
#include <errno.h>
#include <unistd.h>
-#define MAX_HEADER_SIZE 1024 // extra space for the header not counted in MaxObjSize
-#define RET_ERR(num) ({ errno = num; return NULL; })
+#define RET_ERR(num) ({ errno = num; return -1; })
struct buck2obj_buf
{
- uns max_len, raw_len;
- byte *raw;
+ bb_t bb;
struct lizard_buffer *lizard;
- struct mempool *mp;
};
-static void
-buck2obj_alloc_internal(struct buck2obj_buf *buf, uns max_len)
+static uns get_attr_type;
+
+void
+get_attr_set_type(uns type)
+{
+ if (type < BUCKET_TYPE_PLAIN || type > BUCKET_TYPE_V33_LIZARD)
+ die("Unknown buckettype %x", type);
+ get_attr_type = type;
+}
+
+int
+get_attr(byte **pos, byte *end, struct parsed_attr *attr)
{
- buf->max_len = max_len;
- if (!max_len)
+ byte *ptr = *pos;
+ if (ptr >= end)
+ return -1;
+ if (get_attr_type < BUCKET_TYPE_V33)
{
- buf->raw_len = 0;
- buf->raw = NULL;
- return;
+ if (get_attr_type == BUCKET_TYPE_PLAIN)
+ {
+ while (ptr < end && *ptr == '\n')
+ ptr++;
+ *pos = ptr;
+ if (ptr >= end)
+ return -1;
+ }
+ else if (*ptr == '\n')
+ {
+ *pos = ++ptr;
+ attr->attr = 0;
+ return 0;
+ }
+ attr->attr = *ptr++;
+ attr->val = ptr;
+ while (ptr < end && *ptr != '\n')
+ ptr++;
+ attr->len = ptr++ - attr->val;
}
- buf->raw_len = max_len * LIZARD_MAX_MULTIPLY + LIZARD_MAX_ADD + MAX_HEADER_SIZE;
- buf->raw = xmalloc(buf->raw_len);
+ else
+ {
+ uns len;
+ GET_UTF8_32(ptr, len);
+ if (!len--)
+ {
+ *pos = ptr;
+ attr->attr = 0;
+ return 0;
+ }
+ attr->attr = ptr[len];
+ attr->val = ptr;
+ attr->len = len;
+ ptr += len+1;
+ }
+ if (ptr > end)
+ die("Incomplete attribute %c", attr->attr);
+ *pos = ptr;
+ return attr->attr;
+}
+
+int
+bget_attr(struct fastbuf *b, struct parsed_attr *attr)
+{
+ static bb_t buf;
+ if (get_attr_type < BUCKET_TYPE_V33)
+ {
+ int c = bgetc(b);
+ if (c < 0)
+ return -1;
+ if (get_attr_type == BUCKET_TYPE_PLAIN)
+ {
+ while (c == '\n')
+ c = bgetc(b);
+ if (c < 0)
+ return -1;
+ }
+ else if (c == '\n')
+ {
+ attr->attr = 0;
+ return 0;
+ }
+ attr->attr = c;
+
+ byte *ptr, *end;
+ uns len = bdirect_read_prepare(b, &ptr);
+ end = ptr + len;
+ attr->val = ptr;
+ while (ptr < end && *ptr != '\n')
+ ptr++;
+ if (ptr < end)
+ {
+ bdirect_read_commit(b, ptr+1);
+ attr->len = ptr - attr->val;
+ return attr->attr;
+ }
+
+ len = 0;
+ c = bgetc(b);
+ while (c >= 0 && c != '\n')
+ {
+ bb_grow(&buf, len+1);
+ buf.ptr[len++] = c;
+ c = bgetc(b);
+ }
+ if (c < 0)
+ die("Incomplete attribute %c", attr->attr);
+ attr->val = buf.ptr;
+ attr->len = len;
+ }
+ else
+ {
+ int len = bget_utf8_32(b);
+ if (len < 0)
+ return -1;
+ if (!len)
+ {
+ attr->attr = 0;
+ return 0;
+ }
+ attr->len = len-1;
+
+ byte *ptr;
+ int avail = bdirect_read_prepare(b, &ptr);
+ if (avail >= len)
+ {
+ attr->val = ptr;
+ attr->attr = ptr[len-1];
+ bdirect_read_commit(b, ptr + len);
+ return attr->attr;
+ }
+ bb_grow(&buf, --len);
+ breadb(b, buf.ptr, len);
+ attr->val = buf.ptr;
+ attr->len = len;
+ attr->attr = bgetc(b);
+ if (attr->attr < 0)
+ die("Incomplete attribute %c", attr->attr);
+ }
+ return attr->attr;
}
struct buck2obj_buf *
-buck2obj_alloc(struct mempool *mp)
+buck2obj_alloc(void)
{
struct buck2obj_buf *buf = xmalloc(sizeof(struct buck2obj_buf));
- buck2obj_alloc_internal(buf, 0);
- buf->lizard = lizard_alloc(0);
- buf->mp = mp;
+ bb_init(&buf->bb);
+ buf->lizard = lizard_alloc();
return buf;
}
buck2obj_free(struct buck2obj_buf *buf)
{
lizard_free(buf->lizard);
- if (buf->raw)
- xfree(buf->raw);
+ bb_done(&buf->bb);
xfree(buf);
}
-static void
-buck2obj_realloc(struct buck2obj_buf *buf, uns max_len)
-{
- if (max_len <= buf->max_len)
- return;
- if (max_len < 2*buf->max_len) // to ensure amortized logarithmic complexity
- max_len = 2*buf->max_len;
- if (buf->raw)
- xfree(buf->raw);
- buck2obj_alloc_internal(buf, max_len);
-}
-
static inline byte *
decode_attributes(byte *ptr, byte *end, struct odes *o, uns can_overwrite)
{
while (ptr < end)
{
uns len;
- GET_UTF8(ptr, len);
+ GET_UTF8_32(ptr, len);
if (!len--)
break;
byte type = ptr[len];
ptr[len] = 0;
obj_add_attr_ref(o, type, ptr);
- ptr += len + 1;
- }
- else if (can_overwrite == 1)
- while (ptr < end)
- {
- uns len;
- GET_UTF8(ptr, len);
- if (!len--)
- break;
- byte type = ptr[len];
-
- ptr[len] = 0;
- obj_add_attr(o, type, ptr);
- ptr[len] = type;
-
ptr += len + 1;
}
else
while (ptr < end)
{
uns len;
- GET_UTF8(ptr, len);
+ GET_UTF8_32(ptr, len);
if (!len--)
break;
byte type = ptr[len];
return ptr;
}
-struct odes *
-obj_read_bucket(struct buck2obj_buf *buf, uns buck_type, uns buck_len, struct fastbuf *body, uns want_body)
+int
+buck2obj_parse(struct buck2obj_buf *buf, uns buck_type, uns buck_len, struct fastbuf *body, struct odes *o_hdr, uns *body_start, struct odes *o_body)
{
- struct odes *o = obj_new(buf->mp);
-
- if (buck_type < BUCKET_TYPE_V33)
+ if (buck_type <= BUCKET_TYPE_PLAIN)
{
- if (want_body) // ignore empty lines, read until NUL or EOF
- {
- obj_read_multi(body, o);
- bgetc(body);
- }
- else // end on EOF or the first empty line
- obj_read(body, o);
+ if (body_start) // there is no header part
+ *body_start = 0;
+ // ignore empty lines and read until the end of the bucket
+ sh_off_t end = btell(body) + buck_len;
+ byte buf[MAX_ATTR_SIZE];
+ while (btell(body) < end && bgets(body, buf, sizeof(buf)))
+ if (buf[0])
+ obj_add_attr(o_hdr, buf[0], buf+1);
+ ASSERT(btell(body) == end);
}
- else
+ else if (buck_type == BUCKET_TYPE_V30)
{
+ sh_off_t start = btell(body);
+ sh_off_t end = start + buck_len;
+ byte buf[MAX_ATTR_SIZE];
+ while (btell(body) < end && bgets(body, buf, sizeof(buf)) && buf[0])
+ obj_add_attr(o_hdr, buf[0], buf+1);
+ if (body_start)
+ *body_start = btell(body) - start;
+ else
+ {
+ while (btell(body) < end && bgets(body, buf, sizeof(buf)))
+ if (buf[0])
+ obj_add_attr(o_body, buf[0], buf+1);
+ ASSERT(btell(body) == end);
+ }
+ }
+ else if (buck_type == BUCKET_TYPE_V33 || buck_type == BUCKET_TYPE_V33_LIZARD)
+ {
+ /* Avoid reading the whole bucket if only its header is needed. */
+ if (body_start)
+ {
+ sh_off_t start = btell(body);
+ sh_off_t end = start + buck_len;
+ while (btell(body) < end)
+ {
+ uns len = bget_utf8_32(body);
+ if (!len)
+ break;
+ byte *buf = mp_alloc_fast_noalign(o_hdr->pool, len);
+ bread(body, buf, len);
+ uns type = buf[--len];
+ buf[len] = 0;
+ obj_add_attr_ref(o_hdr, type, buf);
+ }
+ *body_start = btell(body) - start;
+ return 0;
+ }
+
/* Read all the bucket into 1 buffer, 0-copy if possible. */
- /* FIXME: This could be cached in buck2obj_buf */
- int can_overwrite = bconfig(body, BCONFIG_CAN_OVERWRITE, 0);
- if (can_overwrite < 0)
- can_overwrite = 0;
- uns overwritten;
byte *ptr, *end;
uns len = bdirect_read_prepare(body, &ptr);
+ uns copied = 0;
if (len < buck_len
- || (can_overwrite < 2 && buck_type == BUCKET_TYPE_V33))
+ || (body->can_overwrite_buffer < 2 && buck_type == BUCKET_TYPE_V33))
{
/* Copy if the original buffer is too small.
* If it is write-protected, copy it also if it is uncompressed. */
- if (buck_len > buf->raw_len)
- buck2obj_realloc(buf, buck_len);
- len = bread(body, buf->raw, buck_len);
- ptr = buf->raw;
- can_overwrite = 2;
- overwritten = 0;
+ DBG("NO ZC: %d < %d, %d %08x", len, buck_len, body->can_overwrite_buffer, buck_type);
+ bb_grow(&buf->bb, buck_len);
+ len = bread(body, buf->bb.ptr, buck_len);
+ ptr = buf->bb.ptr;
+ copied = 1;
}
else
- overwritten = can_overwrite > 1;
- end = ptr + len;
+ DBG("ZC (%d >= %d, %d %08x)", len, buck_len, body->can_overwrite_buffer, buck_type);
+ end = ptr + buck_len;
- ptr = decode_attributes(ptr, end, o, can_overwrite);// header
- if (!want_body)
- return o;
- if (buck_type == BUCKET_TYPE_V33)
- ;
- else if (buck_type == BUCKET_TYPE_V33_LIZARD) // decompression
+ ptr = decode_attributes(ptr, end, o_hdr, 0); // header
+ if (buck_type == BUCKET_TYPE_V33_LIZARD) // decompression
{
- len = GET_U32(ptr);
- ptr += 4;
- int res;
-decompress:
- res = lizard_decompress_safe(ptr, buf->lizard, len);
- if (res != (int) len)
- {
- if (res >= 0)
- errno = EINVAL;
- else if (errno == EFBIG)
+ if (ptr + 8 > end)
{
- lizard_realloc(buf->lizard, len);
- goto decompress;
+ if (ptr == end) // truncated bucket
+ goto commit;
+ RET_ERR(EINVAL);
}
- else
- return NULL;
- }
- ptr = buf->lizard->ptr;
+ len = GET_U32(ptr);
+ ptr += 4;
+ uns adler = GET_U32(ptr);
+ ptr += 4;
+ byte *new_ptr = lizard_decompress_safe(ptr, buf->lizard, len);
+ if (!new_ptr)
+ return -1;
+ if (adler32(new_ptr, len) != adler)
+ RET_ERR(EINVAL);
+ if (!copied)
+ bdirect_read_commit(body, end);
+ ptr = new_ptr;
end = ptr + len;
- can_overwrite = 2;
+ copied = 1;
}
- else // unknown bucket type
- RET_ERR(EINVAL);
- ASSERT(can_overwrite == 2); // because of the policy and decompression
- ptr = decode_attributes(ptr, end, o, 2); // body
-
+ ptr = decode_attributes(ptr, end, o_body, 2); // body
if (ptr != end)
RET_ERR(EINVAL);
- /* If (overwritten), bflush(body) might be needed. */
+ commit:
+ if (!copied)
+ bdirect_read_commit_modified(body, ptr);
}
- return o;
+ else
+ {
+ bskip(body, buck_len);
+ RET_ERR(EINVAL);
+ }
+ return 0;
}
-byte *
-obj_attr_to_bucket(byte *buf, uns buck_type, uns attr, byte *val)
+struct odes *
+obj_read_bucket(struct buck2obj_buf *buf, struct mempool *pool, uns buck_type, uns buck_len, struct fastbuf *body, uns *body_start)
{
- uns l;
-
- switch (buck_type)
- {
- case BUCKET_TYPE_PLAIN:
- case BUCKET_TYPE_V30:
- buf += sprintf(buf, "%c%s\n", attr, val);
- break;
- case BUCKET_TYPE_V33:
- case BUCKET_TYPE_V33_LIZARD:
- l = strlen(val) + 1;
- PUT_UTF8(buf, l);
- l--;
- memcpy(buf, val, l);
- buf += l;
- *buf++ = attr;
- break;
- default:
- die("obj_attr_to_bucket called for unknown type %08x", buck_type);
- }
- return buf;
+ struct odes *o = obj_new(pool);
+ if (buck2obj_parse(buf, buck_type, buck_len, body, o, body_start, o) < 0)
+ return NULL;
+ else
+ return o;
}
-byte *
-obj_attr_to_bucket_num(byte *buf, uns buck_type, uns attr, uns val)
+int
+obj_read(struct fastbuf *f, struct odes *o)
{
- byte vbuf[16];
- sprintf(vbuf, "%d", val);
- return obj_attr_to_bucket(buf, buck_type, attr, vbuf);
+ byte buf[MAX_ATTR_SIZE];
+
+ while (bgets(f, buf, sizeof(buf)))
+ {
+ if (!buf[0])
+ return 1;
+ obj_add_attr(o, buf[0], buf+1);
+ }
+ return 0;
}
projects / libucw.git / blobdiff