From: Martin Mares <mj@ucw.cz>
Date: Sat, 28 Oct 2000 14:01:28 +0000 (+0000)
Subject: New I/O library.
X-Git-Tag: holmes-import~1625
X-Git-Url: http://mj.ucw.cz/gitweb/?a=commitdiff_plain;h=168c1f2ddcb8ce07248b4079b6398afe8c2d792a;p=libucw.git

New I/O library.
---

diff --git a/lib/fastbuf.c b/lib/fastbuf.c
index c3b8b80d..717372fd 100644
--- a/lib/fastbuf.c
+++ b/lib/fastbuf.c
@@ -1,97 +1,22 @@
 /*
- *	Sherlock Library -- Fast File Buffering
+ *	Sherlock Library -- Fast Buffered I/O
  *
- *	(c) 1997--1999 Martin Mares <mj@atrey.karlin.mff.cuni.cz>
+ *	(c) 1997--2000 Martin Mares <mj@ucw.cz>
  */
 
 #include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
-#include <fcntl.h>
-#include <unistd.h>
 
 #include "lib.h"
 #include "fastbuf.h"
-#include "lfs.h"
-
-struct fastbuf *__bfdopen(int fd, uns buffer, byte *name)
-{
-  struct fastbuf *b = xmalloc(sizeof(struct fastbuf));
-
-  b->buflen = buffer;
-  b->buffer = xmalloc(buffer);
-  b->bptr = b->bstop = b->buffer;
-  b->bufend = b->buffer + buffer;
-  b->name = stralloc(name);
-  b->pos = b->fdpos = 0;
-  b->fd = fd;
-  return b;
-}
-
-struct fastbuf *
-bopen(byte *name, uns mode, uns buffer)
-{
-  int fd = sh_open(name, mode, 0666);
-  if (fd < 0)
-    die("Unable to %s file %s: %m",
-	(mode & O_CREAT) ? "create" : "open", name);
-  return __bfdopen(fd, buffer, name);
-}
-
-struct fastbuf *
-bfdopen(int fd, uns buffer)
-{
-  byte x[32];
-
-  sprintf(x, "fd%d", fd);
-  return __bfdopen(fd, buffer, x);
-}
 
 void bclose(struct fastbuf *f)
 {
   bflush(f);
-  close(f->fd);
-  free(f->name);
-  free(f->buffer);
+  f->close(f);
   free(f);
 }
 
-static int
-rdbuf(struct fastbuf *f)
-{
-  int l = read(f->fd, f->buffer, f->buflen);
-
-  if (l < 0)
-    die("Error reading %s: %m", f->name);
-  f->bptr = f->buffer;
-  f->bstop = f->buffer + l;
-  f->pos = f->fdpos;
-  f->fdpos += l;
-  return l;
-}
-
-static void
-wrbuf(struct fastbuf *f)
-{
-  int l = f->bptr - f->buffer;
-  char *c = f->buffer;
-
-  while (l)
-    {
-      int z = write(f->fd, c, l);
-      if (z <= 0)
-	die("Error writing %s: %m", f->name);
-      /* FIXME */
-      if (z != l)
-	log(L_ERROR "wrbuf: %d != %d (pos %Ld)", z, l, sh_seek(f->fd, 0, SEEK_CUR));
-      f->fdpos += z;
-      l -= z;
-      c += z;
-    }
-  f->bptr = f->buffer;
-  f->pos = f->fdpos;
-}
-
 void bflush(struct fastbuf *f)
 {
   if (f->bptr != f->buffer)
@@ -102,7 +27,7 @@ void bflush(struct fastbuf *f)
 	  f->pos = f->fdpos;
 	}
       else				/* Write data... */
-	wrbuf(f);
+	f->spout(f);
     }
 }
 
@@ -113,9 +38,7 @@ inline void bsetpos(struct fastbuf *f, sh_off_t pos)
   else
     {
       bflush(f);
-      if (f->fdpos != pos && sh_seek(f->fd, pos, SEEK_SET) < 0)
-	die("lseek on %s: %m", f->name);
-      f->fdpos = f->pos = pos;
+      f->seek(f, pos, SEEK_SET);
     }
 }
 
@@ -131,10 +54,7 @@ void bseek(struct fastbuf *f, sh_off_t pos, int whence)
       return bsetpos(f, btell(f) + pos);
     case SEEK_END:
       bflush(f);
-      l = sh_seek(f->fd, pos, whence);
-      if (l < 0)
-	die("lseek on %s: %m", f->name);
-      f->fdpos = f->pos = l;
+      f->seek(f, pos, SEEK_END);
       break;
     default:
       die("bseek: invalid whence=%d", whence);
@@ -145,7 +65,7 @@ int bgetc_slow(struct fastbuf *f)
 {
   if (f->bptr < f->bstop)
     return *f->bptr++;
-  if (!rdbuf(f))
+  if (!f->refill(f))
     return EOF;
   return *f->bptr++;
 }
@@ -154,7 +74,7 @@ int bpeekc_slow(struct fastbuf *f)
 {
   if (f->bptr < f->bstop)
     return *f->bptr;
-  if (!rdbuf(f))
+  if (!f->refill(f))
     return EOF;
   return *f->bptr;
 }
@@ -162,7 +82,7 @@ int bpeekc_slow(struct fastbuf *f)
 void bputc_slow(struct fastbuf *f, byte c)
 {
   if (f->bptr >= f->bufend)
-    wrbuf(f);
+    f->spout(f);
   *f->bptr++ = c;
 }
 
@@ -274,7 +194,7 @@ void bread_slow(struct fastbuf *f, void *b, uns l)
 
       if (!k)
 	{
-	  rdbuf(f);
+	  f->refill(f);
 	  k = f->bstop - f->bptr;
 	  if (!k)
 	    die("bread on %s: file exhausted", f->name);
@@ -296,7 +216,7 @@ void bwrite_slow(struct fastbuf *f, void *b, uns l)
 
       if (!k)
 	{
-	  wrbuf(f);
+	  f->spout(f);
 	  k = f->bufend - f->bptr;
 	}
       if (k > l)
@@ -329,60 +249,3 @@ bgets(struct fastbuf *f, byte *b, uns l)
     }
   die("%s: Line too long", f->name);
 }
-
-void bbcopy(struct fastbuf *f, struct fastbuf *t, uns l)
-{
-  uns rf = f->bstop - f->bptr;
-
-  if (!l)
-    return;
-  if (rf)
-    {
-      uns k = (rf <= l) ? rf : l;
-      bwrite(t, f->bptr, k);
-      f->bptr += k;
-      l -= k;
-    }
-  while (l >= t->buflen)
-    {
-      wrbuf(t);
-      if ((uns) read(f->fd, t->buffer, t->buflen) != t->buflen)
-	die("bbcopy: %s exhausted", f->name);
-      f->pos = f->fdpos;
-      f->fdpos += t->buflen;
-      f->bstop = f->bptr = f->buffer;
-      t->bptr = t->bufend;
-      l -= t->buflen;
-    }
-  while (l)
-    {
-      uns k = t->bufend - t->bptr;
-
-      if (!k)
-	{
-	  wrbuf(t);
-	  k = t->bufend - t->bptr;
-	}
-      if (k > l)
-	k = l;
-      bread(f, t->bptr, k);
-      t->bptr += k;
-      l -= k;
-    }
-}
-
-#ifdef TEST
-
-int main(int argc, char **argv)
-{
-  struct fastbuf *f, *t;
-  int c;
-
-  f = bopen("/etc/profile", O_RDONLY, 16);
-  t = bfdopen(1, 13);
-  bbcopy(f, t, 100);
-  bclose(f);
-  bclose(t);
-}
-
-#endif
diff --git a/lib/fastbuf.h b/lib/fastbuf.h
index c367a1ab..d5ca3e71 100644
--- a/lib/fastbuf.h
+++ b/lib/fastbuf.h
@@ -1,28 +1,68 @@
 /*
- *	Sherlock Library -- Fast File Buffering
+ *	Sherlock Library -- Fast Buffered I/O
  *
- *	(c) 1997--1999 Martin Mares <mj@atrey.karlin.mff.cuni.cz>
+ *	(c) 1997--2000 Martin Mares <mj@ucw.cz>
  */
 
 #ifndef EOF
 #include <stdio.h>
 #endif
 
+/*
+ *  Generic buffered I/O on a top of buffer swapping functions.
+ *
+ *  Buffer layout when reading:
+ *
+ *  +----------------+---------------------------+
+ *  | read data      | free space                |
+ *  +----------------+---------------------------+
+ *  ^        ^        ^                           ^
+ *  buffer   bptr     bstop                       bufend
+ *
+ *  After the last character is read, bptr == bstop and buffer refill
+ *  is deferred to the next read attempt. This gives us an easy way
+ *  how to implement bungetc().
+ *
+ *  When writing:
+ *
+ *  +----------------+---------------------------+
+ *  | written data   | free space                |
+ *  +----------------+---------------------------+
+ *  ^                 ^                           ^
+ *  buffer=bstop      bptr                        bufend
+ */
+
 struct fastbuf {
   byte *bptr, *bstop;			/* Access pointers */
   byte *buffer, *bufend;		/* Start and end of the buffer */
   byte *name;				/* File name for error messages */
-  uns buflen;				/* Size of standard portion of the buffer */
-  sh_off_t pos;				/* Position of bptr in the file */
-  sh_off_t fdpos;			/* Current position in the file */
-  int fd;				/* File descriptor */
+  uns buflen;				/* Size of the buffer */
+  sh_off_t pos;				/* Position of buffer start in the file */
+  sh_off_t fdpos;			/* Current position in the non-buffered file */
+  int fd;				/* File descriptor, -1 if not a real file */
+  void *lldata;				/* Data private to access functions below */
+  void *llpos;				/* ... continued ... */
+  int (*refill)(struct fastbuf *);	/* Get a buffer with new data */
+  void (*spout)(struct fastbuf *);	/* Write buffer data to the file */
+  void (*seek)(struct fastbuf *, sh_off_t, int);  /* Slow path for bseek(), buffer already flushed */
+  void (*close)(struct fastbuf *);	/* Close the stream */
 };
 
+/* FastIO on standard files */
+
 struct fastbuf *bopen(byte *name, uns mode, uns buffer);
 struct fastbuf *bfdopen(int fd, uns buffer);
+void bbcopy(struct fastbuf *f, struct fastbuf *t, uns l);
+
+/* FastIO on in-memory streams */
+
+struct fastbuf *fbmem_create(unsigned blocksize);	/* Create stream and return its writing fastbuf */
+struct fastbuf *fbmem_clone_read(struct fastbuf *);	/* Create reading fastbuf */
+
+/* Universal functions working on all fastbuf's */
+
 void bclose(struct fastbuf *f);
 void bflush(struct fastbuf *f);
-
 void bseek(struct fastbuf *f, sh_off_t pos, int whence);
 void bsetpos(struct fastbuf *f, sh_off_t pos);
 
@@ -248,7 +288,6 @@ extern inline void bwrite(struct fastbuf *f, void *b, uns l)
     bwrite_slow(f, b, l);
 }
 
-void bbcopy(struct fastbuf *f, struct fastbuf *t, uns l);
 byte *bgets(struct fastbuf *f, byte *b, uns l);	/* Non-std */
 
 extern inline void
@@ -264,6 +303,8 @@ bputsn(struct fastbuf *f, byte *b)
   bputc(f, '\n');
 }
 
+/* Depending on compile-time configuration, we select the right function for reading/writing of file offsets */
+
 #ifdef SHERLOCK_CONFIG_LARGE_DB
 #define bgeto(f) bget5(f)
 #define bputo(f,l) bput5(f,l)
diff --git a/lib/fb-file.c b/lib/fb-file.c
new file mode 100644
index 00000000..5fca69da
--- /dev/null
+++ b/lib/fb-file.c
@@ -0,0 +1,164 @@
+/*
+ *	Sherlock Library -- Fast Buffered I/O on Files
+ *
+ *	(c) 1997--2000 Martin Mares <mj@ucw.cz>
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "lib.h"
+#include "fastbuf.h"
+#include "lfs.h"
+
+static int
+bfd_refill(struct fastbuf *f)
+{
+  int l = read(f->fd, f->buffer, f->buflen);
+
+  if (l < 0)
+    die("Error reading %s: %m", f->name);
+  f->bptr = f->buffer;
+  f->bstop = f->buffer + l;
+  f->pos = f->fdpos;
+  f->fdpos += l;
+  return l;
+}
+
+static void
+bfd_spout(struct fastbuf *f)
+{
+  int l = f->bptr - f->buffer;
+  char *c = f->buffer;
+
+  while (l)
+    {
+      int z = write(f->fd, c, l);
+      if (z <= 0)
+	die("Error writing %s: %m", f->name);
+      f->fdpos += z;
+      l -= z;
+      c += z;
+    }
+  f->bptr = f->buffer;
+  f->pos = f->fdpos;
+}
+
+static void
+bfd_seek(struct fastbuf *f, sh_off_t pos, int whence)
+{
+  sh_off_t l;
+
+  if (whence == SEEK_SET && pos == f->fdpos)
+    return;
+
+  l = sh_seek(f->fd, pos, whence);
+  if (l < 0)
+    die("lseek on %s: %m", f->name);
+  f->fdpos = f->pos = l;
+}
+
+static void
+bfd_close(struct fastbuf *f)
+{
+  close(f->fd);
+}
+
+struct fastbuf *
+bfdopen_internal(int fd, uns buflen, byte *name)
+{
+  int namelen = strlen(name) + 1;
+  struct fastbuf *b = xmalloc(sizeof(struct fastbuf) + buflen + namelen);
+
+  b->buflen = buflen;
+  b->buffer = (char *)(b+1);
+  b->bptr = b->bstop = b->buffer;
+  b->bufend = b->buffer + buflen;
+  b->name = b->bufend;
+  strcpy(b->name, name);
+  b->pos = b->fdpos = 0;
+  b->fd = fd;
+  b->refill = bfd_refill;
+  b->spout = bfd_spout;
+  b->seek = bfd_seek;
+  b->close = bfd_close;
+  return b;
+}
+
+struct fastbuf *
+bopen(byte *name, uns mode, uns buffer)
+{
+  int fd = sh_open(name, mode, 0666);
+  if (fd < 0)
+    die("Unable to %s file %s: %m",
+	(mode & O_CREAT) ? "create" : "open", name);
+  return bfdopen_internal(fd, buffer, name);
+}
+
+struct fastbuf *
+bfdopen(int fd, uns buffer)
+{
+  byte x[32];
+
+  sprintf(x, "fd%d", fd);
+  return bfdopen_internal(fd, buffer, x);
+}
+
+void bbcopy(struct fastbuf *f, struct fastbuf *t, uns l)
+{
+  uns rf = f->bstop - f->bptr;
+
+  if (!l)
+    return;
+  if (rf)
+    {
+      uns k = (rf <= l) ? rf : l;
+      bwrite(t, f->bptr, k);
+      f->bptr += k;
+      l -= k;
+    }
+  while (l >= t->buflen)
+    {
+      t->spout(t);
+      if ((uns) read(f->fd, t->buffer, t->buflen) != t->buflen)
+	die("bbcopy: %s exhausted", f->name);
+      f->pos = f->fdpos;
+      f->fdpos += t->buflen;
+      f->bstop = f->bptr = f->buffer;
+      t->bptr = t->bufend;
+      l -= t->buflen;
+    }
+  while (l)
+    {
+      uns k = t->bufend - t->bptr;
+
+      if (!k)
+	{
+	  t->spout(t);
+	  k = t->bufend - t->bptr;
+	}
+      if (k > l)
+	k = l;
+      bread(f, t->bptr, k);
+      t->bptr += k;
+      l -= k;
+    }
+}
+
+#ifdef TEST
+
+int main(int argc, char **argv)
+{
+  struct fastbuf *f, *t;
+  int c;
+
+  f = bopen("/etc/profile", O_RDONLY, 16);
+  t = bfdopen(1, 13);
+  bbcopy(f, t, 100);
+  bclose(f);
+  bclose(t);
+}
+
+#endif
diff --git a/lib/fb-mem.c b/lib/fb-mem.c
new file mode 100644
index 00000000..2000a35f
--- /dev/null
+++ b/lib/fb-mem.c
@@ -0,0 +1,198 @@
+/*
+ *	Sherlock Library -- Fast Buffered I/O on Memory Streams
+ *
+ *	(c) 1997--2000 Martin Mares <mj@ucw.cz>
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "lib.h"
+#include "fastbuf.h"
+
+struct memstream {
+  unsigned blocksize;
+  unsigned uc;
+  struct msblock *first;
+};
+
+struct msblock {
+  struct msblock *next;
+  unsigned size;
+  byte data[0];
+};
+
+int
+fbmem_refill(struct fastbuf *f)
+{
+  struct memstream *s = f->lldata;
+  struct msblock *b = f->llpos;
+
+  if (!b)
+    {
+      b = s->first;
+      if (!b)
+	return 0;
+    }
+  else if (f->bstop < b->data + b->size)
+    {
+      f->bstop = b->data + b->size;
+      return 1;
+    }
+  else
+    {
+      if (!b->next)
+	return 0;
+      f->pos += b->size;
+      b = b->next;
+    }
+  f->buffer = f->bptr = b->data;
+  f->bufend = f->bstop = b->data + b->size;
+  f->llpos = b;
+  return 1;
+}
+
+void
+fbmem_spout(struct fastbuf *f)
+{
+  struct memstream *s = f->lldata;
+  struct msblock *b = f->llpos;
+  struct msblock *bb;
+
+  if (b)
+    {
+      b->size = f->bptr - b->data;
+      if (b->size < s->blocksize)
+	return;
+      f->pos += b->size;
+    }
+  bb = xmalloc(sizeof(struct msblock) + s->blocksize);
+  if (b)
+    b->next = bb;
+  else
+    s->first = bb;
+  bb->next = NULL;
+  bb->size = 0;
+  f->buffer = f->bptr = f->bstop = bb->data;
+  f->bufend = bb->data + s->blocksize;
+  f->llpos = bb;
+}
+
+void
+fbmem_seek(struct fastbuf *f, sh_off_t pos, int whence)
+{
+  struct memstream *m = f->lldata;
+  struct msblock *b;
+  unsigned int p = 0;
+
+  if (whence != SEEK_SET)
+    die("fbmem_seek: only SEEK_SET supported");
+  for (b=m->first; b; b=b->next)
+    {
+      if (pos <= p + b->size)	/* <=, because we need to be able to seek just after file end */
+	{
+	  f->pos = p;
+	  f->buffer = b->data;
+	  f->bptr = b->data + (pos - p);
+	  f->bufend = f->bstop = b->data + b->size;
+	  f->llpos = b;
+	  return;
+	}
+      p += b->size;
+    }
+  die("fbmem_seek to invalid offset");
+}
+
+void
+fbmem_close(struct fastbuf *f)
+{
+  struct memstream *m = f->lldata;
+  struct msblock *b;
+
+  if (--m->uc)
+    return;
+
+  while (b = m->first)
+    {
+      m->first = b->next;
+      free(b);
+    }
+  free(m);
+}
+
+struct fastbuf *
+fbmem_create(unsigned blocksize)
+{
+  struct fastbuf *f = xmalloc(sizeof(struct fastbuf));
+  struct memstream *m = xmalloc(sizeof(struct memstream));
+
+  m->blocksize = blocksize;
+  m->uc = 1;
+  m->first = NULL;
+
+  f->bptr = f->bstop = f->buffer = f->bufend = NULL;
+  f->pos = f->fdpos = 0;
+  f->name = "<fbmem-write>";
+  f->lldata = m;
+  f->refill = NULL;
+  f->spout = fbmem_spout;
+  f->seek = NULL;
+  f->close = fbmem_close;
+  return f;
+}
+
+struct fastbuf *
+fbmem_clone_read(struct fastbuf *b)
+{
+  struct fastbuf *f = xmalloc(sizeof(struct fastbuf));
+  struct memstream *s = b->lldata;
+
+  s->uc++;
+
+  f->bptr = f->bstop = f->buffer = f->bufend = NULL;
+  f->pos = f->fdpos = 0;
+  f->name = "<fbmem-read>";
+  f->lldata = s;
+  f->refill = fbmem_refill;
+  f->spout = NULL;
+  f->seek = fbmem_seek;
+  f->close = fbmem_close;
+  return f;
+}
+
+#ifdef TEST
+
+int main(void)
+{
+  struct fastbuf *w, *r;
+  int t;
+
+  w = fbmem_create(7);
+  r = fbmem_clone_read(w);
+  bwrite(w, "12345", 5);
+  bwrite(w, "12345", 5);
+  printf("<%d>", btell(w));
+  bflush(w);
+  printf("<%d>", btell(w));
+  printf("<%d>", btell(r));
+  while ((t = bgetc(r)) >= 0)
+    putchar(t);
+  printf("<%d>", btell(r));
+  bwrite(w, "12345", 5);
+  bwrite(w, "12345", 5);
+  printf("<%d>", btell(w));
+  bclose(w);
+  bsetpos(r, 0);
+  printf("<!%d>", btell(r));
+  while ((t = bgetc(r)) >= 0)
+    putchar(t);
+  bsetpos(r, 3);
+  printf("<!%d>", btell(r));
+  while ((t = bgetc(r)) >= 0)
+    putchar(t);
+  fflush(stdout);
+  bclose(r);
+  return 0;
+}
+
+#endif