X-Git-Url: http://mj.ucw.cz/gitweb/?a=blobdiff_plain;f=lib%2Fbuck2obj.c;h=d75b48ab19e212b2d65e8996981db3b6dda85325;hb=c673da72822d7704572716de80402829921efdc3;hp=4ea6a5cca1e96d76762c9d87468b219edd817197;hpb=f02995ab2dfba8b686a155118a35ae20a3328b94;p=libucw.git diff --git a/lib/buck2obj.c b/lib/buck2obj.c index 4ea6a5cc..d75b48ab 100644 --- a/lib/buck2obj.c +++ b/lib/buck2obj.c @@ -1,49 +1,39 @@ /* - * Bucket -> Object converter + * Generating Objects from Buckets * * (c) 2004, Robert Spalek + * (c) 2004, Martin Mares */ #include "lib/lib.h" #include "lib/unaligned.h" -#include "lib/pools.h" +#include "lib/mempool.h" #include "lib/fastbuf.h" -#include "charset/unicode.h" +#include "lib/unicode.h" #include "lib/object.h" #include "lib/bucket.h" #include "lib/lizard.h" -#include "lib/buck2obj.h" +#include "lib/bbuf.h" +#include "lib/ff-utf8.h" #include #include #include -#define MAX_HEADER_SIZE 1024 // extra space for the header not counted in MaxObjSize -#define RET_ERR(num) ({ errno = num; return NULL; }) +#define RET_ERR(num) ({ errno = num; return -1; }) struct buck2obj_buf { - uns max_len, raw_len; - byte *raw; + bb_t bb; struct lizard_buffer *lizard; - struct mempool *mp; }; -static void -buck2obj_alloc_internal(struct buck2obj_buf *buf, uns max_len) -{ - buf->max_len = max_len; - buf->raw_len = max_len * LIZARD_MAX_MULTIPLY + LIZARD_MAX_ADD + MAX_HEADER_SIZE; - buf->raw = xmalloc(buf->raw_len); -} - struct buck2obj_buf * -buck2obj_alloc(uns max_len, struct mempool *mp) +buck2obj_alloc(void) { struct buck2obj_buf *buf = xmalloc(sizeof(struct buck2obj_buf)); - buck2obj_alloc_internal(buf, max_len); - buf->lizard = lizard_alloc(max_len); - buf->mp = mp; + bb_init(&buf->bb); + buf->lizard = lizard_alloc(); return buf; } @@ -51,22 +41,10 @@ void buck2obj_free(struct buck2obj_buf *buf) { lizard_free(buf->lizard); - xfree(buf->raw); + bb_done(&buf->bb); xfree(buf); } -void -buck2obj_realloc(struct buck2obj_buf *buf, uns max_len) -{ - if (max_len <= buf->max_len) - return; - if (max_len < 2*buf->max_len + 1) // to ensure amortized logarithmic complexity - max_len = 2*buf->max_len + 1; - xfree(buf->raw); - buck2obj_alloc_internal(buf, max_len); - lizard_realloc(buf->lizard, max_len); -} - static inline byte * decode_attributes(byte *ptr, byte *end, struct odes *o, uns can_overwrite) { @@ -82,21 +60,6 @@ decode_attributes(byte *ptr, byte *end, struct odes *o, uns can_overwrite) ptr[len] = 0; obj_add_attr_ref(o, type, ptr); - ptr += len + 1; - } - else if (can_overwrite == 1) - while (ptr < end) - { - uns len; - GET_UTF8(ptr, len); - if (!len--) - break; - byte type = ptr[len]; - - ptr[len] = 0; - obj_add_attr(o, type, ptr); - ptr[len] = type; - ptr += len + 1; } else @@ -118,78 +81,130 @@ decode_attributes(byte *ptr, byte *end, struct odes *o, uns can_overwrite) return ptr; } -struct odes * -buck2obj_convert(struct buck2obj_buf *buf, uns buck_type, struct fastbuf *body, uns want_body) +int +buck2obj_parse(struct buck2obj_buf *buf, uns buck_type, uns buck_len, struct fastbuf *body, struct odes *o_hdr, uns *body_start, struct odes *o_body) { - mp_flush(buf->mp); - struct odes *o = obj_new(buf->mp); - - if (buck_type < BUCKET_TYPE_V33) + if (buck_type <= BUCKET_TYPE_PLAIN) { - if (want_body) // ignore empty lines, read until EOF - obj_read_multi(body, o); - else // end on EOF or the first empty line - obj_read(body, o); + if (body_start) // there is no header part + *body_start = 0; + // ignore empty lines and read until the end of the bucket + sh_off_t end = btell(body) + buck_len; + byte buf[MAX_ATTR_SIZE]; + while (btell(body) < end && bgets(body, buf, sizeof(buf))) + if (buf[0]) + obj_add_attr(o_hdr, buf[0], buf+1); + ASSERT(btell(body) == end); } - else + else if (buck_type == BUCKET_TYPE_V30) + { + sh_off_t start = btell(body); + sh_off_t end = start + buck_len; + byte buf[MAX_ATTR_SIZE]; + while (btell(body) < end && bgets(body, buf, sizeof(buf)) && buf[0]) + obj_add_attr(o_hdr, buf[0], buf+1); + if (body_start) + *body_start = btell(body) - start; + else + { + while (btell(body) < end && bgets(body, buf, sizeof(buf))) + if (buf[0]) + obj_add_attr(o_body, buf[0], buf+1); + ASSERT(btell(body) == end); + } + } + else if (buck_type == BUCKET_TYPE_V33 || buck_type == BUCKET_TYPE_V33_LIZARD) { - /* Compute the length of the bucket. We cannot fetch this attribute - * directly due to remote indexing. */ - bseek(body, 0, SEEK_END); - sh_off_t buck_len = btell(body); - bsetpos(body, 0); + /* Avoid reading the whole bucket if only its header is needed. */ + if (body_start) + { + sh_off_t start = btell(body); + sh_off_t end = start + buck_len; + while (btell(body) < end) + { + uns len = bget_utf8(body); + if (!len) + break; + byte *buf = mp_alloc_fast_noalign(o_hdr->pool, len); + bread(body, buf, len); + uns type = buf[--len]; + buf[len] = 0; + obj_add_attr_ref(o_hdr, type, buf); + } + *body_start = btell(body) - start; + return 0; + } /* Read all the bucket into 1 buffer, 0-copy if possible. */ - int can_overwrite = bconfig(body, BCONFIG_CAN_OVERWRITE, 0); - if (can_overwrite < 0) - can_overwrite = 0; - uns overwritten; byte *ptr, *end; uns len = bdirect_read_prepare(body, &ptr); + uns copied = 0; if (len < buck_len - || (can_overwrite < 2 && buck_type == BUCKET_TYPE_V33)) + || (body->can_overwrite_buffer < 2 && buck_type == BUCKET_TYPE_V33)) { /* Copy if the original buffer is too small. * If it is write-protected, copy it also if it is uncompressed. */ - if (buck_len > buf->raw_len) - buck2obj_realloc(buf, buck_len); - len = bread(body, buf->raw, buck_len); - ptr = buf->raw; - can_overwrite = 2; - overwritten = 0; + bb_grow(&buf->bb, buck_len); + len = bread(body, buf->bb.ptr, buck_len); + ptr = buf->bb.ptr; + copied = 1; } - else - overwritten = can_overwrite > 1; - end = ptr + len; - - ptr = decode_attributes(ptr, end, o, can_overwrite);// header - if (!want_body) - return o; - if (buck_type == BUCKET_TYPE_V33) - ; - else if (buck_type == BUCKET_TYPE_V33_LIZARD) // decompression + end = ptr + buck_len; + + ptr = decode_attributes(ptr, end, o_hdr, 0); // header + if (buck_type == BUCKET_TYPE_V33_LIZARD) // decompression { + if (ptr + 8 > end) + RET_ERR(EINVAL); len = GET_U32(ptr); ptr += 4; - int res = lizard_decompress_safe(ptr, buf->lizard, len); - if (res != (int) len) - { - if (res >= 0) - errno = EINVAL; - return NULL; - } - ptr = buf->lizard->ptr; + uns adler = GET_U32(ptr); + ptr += 4; + byte *new_ptr = lizard_decompress_safe(ptr, buf->lizard, len); + if (!new_ptr) + return -1; + if (adler32(new_ptr, len) != adler) + RET_ERR(EINVAL); + if (!copied) + bdirect_read_commit(body, end); + ptr = new_ptr; end = ptr + len; - can_overwrite = 2; + copied = 1; } - else // unknown bucket type - RET_ERR(EINVAL); - ASSERT(can_overwrite == 2); // because of the policy and decompression - ptr = decode_attributes(ptr, end, o, 2); // body - + ptr = decode_attributes(ptr, end, o_body, 2); // body if (ptr != end) RET_ERR(EINVAL); - /* If (overwritten), bflush(body) might be needed. */ + if (!copied) + bdirect_read_commit_modified(body, ptr); } - return o; + else + { + bskip(body, buck_len); + RET_ERR(EINVAL); + } + return 0; +} + +struct odes * +obj_read_bucket(struct buck2obj_buf *buf, struct mempool *pool, uns buck_type, uns buck_len, struct fastbuf *body, uns *body_start) +{ + struct odes *o = obj_new(pool); + if (buck2obj_parse(buf, buck_type, buck_len, body, o, body_start, o) < 0) + return NULL; + else + return o; +} + +int +obj_read(struct fastbuf *f, struct odes *o) +{ + byte buf[MAX_ATTR_SIZE]; + + while (bgets(f, buf, sizeof(buf))) + { + if (!buf[0]) + return 1; + obj_add_attr(o, buf[0], buf+1); + } + return 0; }