X-Git-Url: http://mj.ucw.cz/gitweb/?a=blobdiff_plain;f=lib%2Fbuck2obj.c;h=dacdeba4fd6276350c426007c58ced8ae30c454e;hb=b1fd07e17d8b9041fd026385e51f273adb2bee5a;hp=92f1478b10ac4e926b4a5f16829ba6729a731b18;hpb=1aa9512b50adeef4a04577ecf52d8ecf6b29c2d0;p=libucw.git

diff --git a/lib/buck2obj.c b/lib/buck2obj.c
index 92f1478b..dacdeba4 100644
--- a/lib/buck2obj.c
+++ b/lib/buck2obj.c
@@ -1,55 +1,181 @@
 /*
- *	Bucket -> Object converter
+ *	Generating Objects from Buckets
  *
  *	(c) 2004, Robert Spalek <robert@ucw.cz>
+ *	(c) 2004, Martin Mares <mj@ucw.cz>
  */
 
+#undef LOCAL_DEBUG
+
 #include "lib/lib.h"
 #include "lib/unaligned.h"
-#include "lib/pools.h"
+#include "lib/mempool.h"
 #include "lib/fastbuf.h"
-#include "charset/unicode.h"
+#include "lib/unicode.h"
 #include "lib/object.h"
 #include "lib/bucket.h"
 #include "lib/lizard.h"
-#include "lib/buck2obj.h"
+#include "lib/bbuf.h"
+#include "lib/ff-utf8.h"
 
 #include <stdlib.h>
 #include <errno.h>
 #include <unistd.h>
 
-#define	MAX_HEADER_SIZE	1024		// extra space for the header not counted in MaxObjSize
-#define	RET_ERR(num)	({ errno = num; return NULL; })
+#define	RET_ERR(num)	({ errno = num; return -1; })
 
 struct buck2obj_buf
 {
-  uns max_len, raw_len;
-  byte *raw;
+  bb_t bb;
   struct lizard_buffer *lizard;
-  struct mempool *mp;
 };
 
-static void
-buck2obj_alloc_internal(struct buck2obj_buf *buf, uns max_len)
+static uns get_attr_type;
+
+void
+get_attr_set_type(uns type)
+{
+  if (type < BUCKET_TYPE_PLAIN || type > BUCKET_TYPE_V33_LIZARD)
+    die("Unknown buckettype %x", type);
+  get_attr_type = type;
+}
+
+int
+get_attr(byte **pos, byte *end, struct parsed_attr *attr)
 {
-  buf->max_len = max_len;
-  if (!max_len)
+  byte *ptr = *pos;
+  if (ptr >= end)
+    return -1;
+  if (get_attr_type < BUCKET_TYPE_V33)
   {
-    buf->raw_len = 0;
-    buf->raw = NULL;
-    return;
+    if (get_attr_type == BUCKET_TYPE_PLAIN)
+    {
+      while (ptr < end && *ptr == '\n')
+	ptr++;
+      *pos = ptr;
+      if (ptr >= end)
+	return -1;
+    }
+    else if (*ptr == '\n')
+    {
+      *pos = ++ptr;
+      attr->attr = 0;
+      return 0;
+    }
+    attr->attr = *ptr++;
+    attr->val = ptr;
+    while (ptr < end && *ptr != '\n')
+      ptr++;
+    attr->len = ptr++ - attr->val;
   }
-  buf->raw_len = max_len * LIZARD_MAX_MULTIPLY + LIZARD_MAX_ADD + MAX_HEADER_SIZE;
-  buf->raw = xmalloc(buf->raw_len);
+  else
+  {
+    uns len;
+    GET_UTF8_32(ptr, len);
+    if (!len--)
+    {
+      *pos = ptr;
+      attr->attr = 0;
+      return 0;
+    }
+    attr->attr = ptr[len];
+    attr->val = ptr;
+    attr->len = len;
+    ptr += len+1;
+  }
+  if (ptr > end)
+    die("Incomplete attribute %c", attr->attr);
+  *pos = ptr;
+  return attr->attr;
+}
+
+int
+bget_attr(struct fastbuf *b, struct parsed_attr *attr)
+{
+  static bb_t buf;
+  if (get_attr_type < BUCKET_TYPE_V33)
+  {
+    int c = bgetc(b);
+    if (c < 0)
+      return -1;
+    if (get_attr_type == BUCKET_TYPE_PLAIN)
+    {
+      while (c == '\n')
+	c = bgetc(b);
+      if (c < 0)
+	return -1;
+    }
+    else if (c == '\n')
+    {
+      attr->attr = 0;
+      return 0;
+    }
+    attr->attr = c;
+
+    byte *ptr, *end;
+    uns len = bdirect_read_prepare(b, &ptr);
+    end = ptr + len;
+    attr->val = ptr;
+    while (ptr < end && *ptr != '\n')
+      ptr++;
+    if (ptr < end)
+    {
+      bdirect_read_commit(b, ptr+1);
+      attr->len = ptr - attr->val;
+      return attr->attr;
+    }
+
+    len = 0;
+    c = bgetc(b);
+    while (c >= 0 && c != '\n')
+    {
+      bb_grow(&buf, len+1);
+      buf.ptr[len++] = c;
+      c = bgetc(b);
+    }
+    if (c < 0)
+      die("Incomplete attribute %c", attr->attr);
+    attr->val = buf.ptr;
+    attr->len = len;
+  }
+  else
+  {
+    int len = bget_utf8_32(b);
+    if (len < 0)
+      return -1;
+    if (!len)
+    {
+      attr->attr = 0;
+      return 0;
+    }
+    attr->len = len-1;
+
+    byte *ptr;
+    int avail = bdirect_read_prepare(b, &ptr);
+    if (avail >= len)
+    {
+      attr->val = ptr;
+      attr->attr = ptr[len-1];
+      bdirect_read_commit(b, ptr + len);
+      return attr->attr;
+    }
+    bb_grow(&buf, --len);
+    breadb(b, buf.ptr, len);
+    attr->val = buf.ptr;
+    attr->len = len;
+    attr->attr = bgetc(b);
+    if (attr->attr < 0)
+      die("Incomplete attribute %c", attr->attr);
+  }
+  return attr->attr;
 }
 
 struct buck2obj_buf *
-buck2obj_alloc(struct mempool *mp)
+buck2obj_alloc(void)
 {
   struct buck2obj_buf *buf = xmalloc(sizeof(struct buck2obj_buf));
-  buck2obj_alloc_internal(buf, 0);
-  buf->lizard = lizard_alloc(0);
-  buf->mp = mp;
+  bb_init(&buf->bb);
+  buf->lizard = lizard_alloc();
   return buf;
 }
 
@@ -57,23 +183,10 @@ void
 buck2obj_free(struct buck2obj_buf *buf)
 {
   lizard_free(buf->lizard);
-  if (buf->raw)
-    xfree(buf->raw);
+  bb_done(&buf->bb);
   xfree(buf);
 }
 
-static void
-buck2obj_realloc(struct buck2obj_buf *buf, uns max_len)
-{
-  if (max_len <= buf->max_len)
-    return;
-  if (max_len < 2*buf->max_len)		// to ensure amortized logarithmic complexity
-    max_len = 2*buf->max_len;
-  if (buf->raw)
-    xfree(buf->raw);
-  buck2obj_alloc_internal(buf, max_len);
-}
-
 static inline byte *
 decode_attributes(byte *ptr, byte *end, struct odes *o, uns can_overwrite)
 {
@@ -81,7 +194,7 @@ decode_attributes(byte *ptr, byte *end, struct odes *o, uns can_overwrite)
     while (ptr < end)
     {
       uns len;
-      GET_UTF8(ptr, len);
+      GET_UTF8_32(ptr, len);
       if (!len--)
 	break;
       byte type = ptr[len];
@@ -89,28 +202,13 @@ decode_attributes(byte *ptr, byte *end, struct odes *o, uns can_overwrite)
       ptr[len] = 0;
       obj_add_attr_ref(o, type, ptr);
 
-      ptr += len + 1;
-    }
-  else if (can_overwrite == 1)
-    while (ptr < end)
-    {
-      uns len;
-      GET_UTF8(ptr, len);
-      if (!len--)
-	break;
-      byte type = ptr[len];
-
-      ptr[len] = 0;
-      obj_add_attr(o, type, ptr);
-      ptr[len] = type;
-
       ptr += len + 1;
     }
   else
     while (ptr < end)
     {
       uns len;
-      GET_UTF8(ptr, len);
+      GET_UTF8_32(ptr, len);
       if (!len--)
 	break;
       byte type = ptr[len];
@@ -125,117 +223,138 @@ decode_attributes(byte *ptr, byte *end, struct odes *o, uns can_overwrite)
   return ptr;
 }
 
-struct odes *
-obj_read_bucket(struct buck2obj_buf *buf, uns buck_type, uns buck_len, struct fastbuf *body, uns want_body)
+int
+buck2obj_parse(struct buck2obj_buf *buf, uns buck_type, uns buck_len, struct fastbuf *body, struct odes *o_hdr, uns *body_start, struct odes *o_body)
 {
-  struct odes *o = obj_new(buf->mp);
-
-  if (buck_type < BUCKET_TYPE_V33)
+  if (buck_type <= BUCKET_TYPE_PLAIN)
   {
-    if (want_body)			// ignore empty lines, read until NUL or EOF
-      {
-	obj_read_multi(body, o);
-	bgetc(body);
-      }
-    else				// end on EOF or the first empty line
-      obj_read(body, o);
+    if (body_start)			// there is no header part
+      *body_start = 0;
+    // ignore empty lines and read until the end of the bucket
+    sh_off_t end = btell(body) + buck_len;
+    byte buf[MAX_ATTR_SIZE];
+    while (btell(body) < end && bgets(body, buf, sizeof(buf)))
+      if (buf[0])
+	obj_add_attr(o_hdr, buf[0], buf+1);
+    ASSERT(btell(body) == end);
   }
-  else
+  else if (buck_type == BUCKET_TYPE_V30)
   {
+    sh_off_t start = btell(body);
+    sh_off_t end = start + buck_len;
+    byte buf[MAX_ATTR_SIZE];
+    while (btell(body) < end && bgets(body, buf, sizeof(buf)) && buf[0])
+      obj_add_attr(o_hdr, buf[0], buf+1);
+    if (body_start)
+      *body_start = btell(body) - start;
+    else
+    {
+      while (btell(body) < end && bgets(body, buf, sizeof(buf)))
+	if (buf[0])
+	  obj_add_attr(o_body, buf[0], buf+1);
+      ASSERT(btell(body) == end);
+    }
+  }
+  else if (buck_type == BUCKET_TYPE_V33 || buck_type == BUCKET_TYPE_V33_LIZARD)
+  {
+    /* Avoid reading the whole bucket if only its header is needed.  */
+    if (body_start)
+    {
+      sh_off_t start = btell(body);
+      sh_off_t end = start + buck_len;
+      while (btell(body) < end)
+      {
+	uns len = bget_utf8_32(body);
+	if (!len)
+	  break;
+	byte *buf = mp_alloc_fast_noalign(o_hdr->pool, len);
+	bread(body, buf, len);
+	uns type = buf[--len];
+	buf[len] = 0;
+	obj_add_attr_ref(o_hdr, type, buf);
+      }
+      *body_start = btell(body) - start;
+      return 0;
+    }
+
     /* Read all the bucket into 1 buffer, 0-copy if possible.  */
-    /* FIXME: This could be cached in buck2obj_buf */
-    int can_overwrite = bconfig(body, BCONFIG_CAN_OVERWRITE, 0);
-    if (can_overwrite < 0)
-      can_overwrite = 0;
-    uns overwritten;
     byte *ptr, *end;
     uns len = bdirect_read_prepare(body, &ptr);
+    uns copied = 0;
     if (len < buck_len
-    || (can_overwrite < 2 && buck_type == BUCKET_TYPE_V33))
+    || (body->can_overwrite_buffer < 2 && buck_type == BUCKET_TYPE_V33))
     {
       /* Copy if the original buffer is too small.
        * If it is write-protected, copy it also if it is uncompressed.  */
-      if (buck_len > buf->raw_len)
-	buck2obj_realloc(buf, buck_len);
-      len = bread(body, buf->raw, buck_len);
-      ptr = buf->raw;
-      can_overwrite = 2;
-      overwritten = 0;
+      DBG("NO ZC: %d < %d, %d %08x", len, buck_len, body->can_overwrite_buffer, buck_type);
+      bb_grow(&buf->bb, buck_len);
+      len = bread(body, buf->bb.ptr, buck_len);
+      ptr = buf->bb.ptr;
+      copied = 1;
     }
     else
-      overwritten = can_overwrite > 1;
-    end = ptr + len;
+      DBG("ZC (%d >= %d, %d %08x)", len, buck_len, body->can_overwrite_buffer, buck_type);
+    end = ptr + buck_len;
 
-    ptr = decode_attributes(ptr, end, o, can_overwrite);// header
-    if (!want_body)
-      return o;
-    if (buck_type == BUCKET_TYPE_V33)
-      ;
-    else if (buck_type == BUCKET_TYPE_V33_LIZARD)	// decompression
+    ptr = decode_attributes(ptr, end, o_hdr, 0);		// header
+    if (buck_type == BUCKET_TYPE_V33_LIZARD)		// decompression
     {
-      len = GET_U32(ptr);
-      ptr += 4;
-      int res;
-decompress:
-      res = lizard_decompress_safe(ptr, buf->lizard, len);
-      if (res != (int) len)
-      {
-	if (res >= 0)
-	  errno = EINVAL;
-	else if (errno == EFBIG)
+      if (ptr + 8 > end)
 	{
-	  lizard_realloc(buf->lizard, len);
-	  goto decompress;
+	  if (ptr == end)				// truncated bucket
+	    goto commit;
+	  RET_ERR(EINVAL);
 	}
-	else
-	  return NULL;
-      }
-      ptr = buf->lizard->ptr;
+      len = GET_U32(ptr);
+      ptr += 4;
+      uns adler = GET_U32(ptr);
+      ptr += 4;
+      byte *new_ptr = lizard_decompress_safe(ptr, buf->lizard, len);
+      if (!new_ptr)
+	return -1;
+      if (adler32(new_ptr, len) != adler)
+	RET_ERR(EINVAL);
+      if (!copied)
+	bdirect_read_commit(body, end);
+      ptr = new_ptr;
       end = ptr + len;
-      can_overwrite = 2;
+      copied = 1;
     }
-    else						// unknown bucket type
-      RET_ERR(EINVAL);
-    ASSERT(can_overwrite == 2);				// because of the policy and decompression
-    ptr = decode_attributes(ptr, end, o, 2);		// body
-
+    ptr = decode_attributes(ptr, end, o_body, 2);	// body
     if (ptr != end)
       RET_ERR(EINVAL);
-    /* If (overwritten), bflush(body) might be needed.  */
+  commit:
+    if (!copied)
+      bdirect_read_commit_modified(body, ptr);
   }
-  return o;
+  else
+    {
+      bskip(body, buck_len);
+      RET_ERR(EINVAL);
+    }
+  return 0;
 }
 
-byte *
-obj_attr_to_bucket(byte *buf, uns buck_type, uns attr, byte *val)
+struct odes *
+obj_read_bucket(struct buck2obj_buf *buf, struct mempool *pool, uns buck_type, uns buck_len, struct fastbuf *body, uns *body_start)
 {
-  uns l;
-
-  switch (buck_type)
-    {
-    case BUCKET_TYPE_PLAIN:
-    case BUCKET_TYPE_V30:
-      buf += sprintf(buf, "%c%s\n", attr, val);
-      break;
-    case BUCKET_TYPE_V33:
-    case BUCKET_TYPE_V33_LIZARD:
-      l = strlen(val) + 1;
-      PUT_UTF8(buf, l);
-      l--;
-      memcpy(buf, val, l);
-      buf += l;
-      *buf++ = attr;
-      break;
-    default:
-      die("obj_attr_to_bucket called for unknown type %08x", buck_type);
-    }
-  return buf;
+  struct odes *o = obj_new(pool);
+  if (buck2obj_parse(buf, buck_type, buck_len, body, o, body_start, o) < 0)
+    return NULL;
+  else
+    return o;
 }
 
-byte *
-obj_attr_to_bucket_num(byte *buf, uns buck_type, uns attr, uns val)
+int
+obj_read(struct fastbuf *f, struct odes *o)
 {
-  byte vbuf[16];
-  sprintf(vbuf, "%d", val);
-  return obj_attr_to_bucket(buf, buck_type, attr, vbuf);
+  byte buf[MAX_ATTR_SIZE];
+
+  while (bgets(f, buf, sizeof(buf)))
+    {
+      if (!buf[0])
+	return 1;
+      obj_add_attr(o, buf[0], buf+1);
+    }
+  return 0;
 }