From 168c1f2ddcb8ce07248b4079b6398afe8c2d792a Mon Sep 17 00:00:00 2001 From: Martin Mares Date: Sat, 28 Oct 2000 14:01:28 +0000 Subject: [PATCH] New I/O library. --- lib/fastbuf.c | 159 +++------------------------------------- lib/fastbuf.h | 57 +++++++++++++-- lib/fb-file.c | 164 +++++++++++++++++++++++++++++++++++++++++ lib/fb-mem.c | 198 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 422 insertions(+), 156 deletions(-) create mode 100644 lib/fb-file.c create mode 100644 lib/fb-mem.c diff --git a/lib/fastbuf.c b/lib/fastbuf.c index c3b8b80d..717372fd 100644 --- a/lib/fastbuf.c +++ b/lib/fastbuf.c @@ -1,97 +1,22 @@ /* - * Sherlock Library -- Fast File Buffering + * Sherlock Library -- Fast Buffered I/O * - * (c) 1997--1999 Martin Mares + * (c) 1997--2000 Martin Mares */ #include #include -#include -#include -#include #include "lib.h" #include "fastbuf.h" -#include "lfs.h" - -struct fastbuf *__bfdopen(int fd, uns buffer, byte *name) -{ - struct fastbuf *b = xmalloc(sizeof(struct fastbuf)); - - b->buflen = buffer; - b->buffer = xmalloc(buffer); - b->bptr = b->bstop = b->buffer; - b->bufend = b->buffer + buffer; - b->name = stralloc(name); - b->pos = b->fdpos = 0; - b->fd = fd; - return b; -} - -struct fastbuf * -bopen(byte *name, uns mode, uns buffer) -{ - int fd = sh_open(name, mode, 0666); - if (fd < 0) - die("Unable to %s file %s: %m", - (mode & O_CREAT) ? "create" : "open", name); - return __bfdopen(fd, buffer, name); -} - -struct fastbuf * -bfdopen(int fd, uns buffer) -{ - byte x[32]; - - sprintf(x, "fd%d", fd); - return __bfdopen(fd, buffer, x); -} void bclose(struct fastbuf *f) { bflush(f); - close(f->fd); - free(f->name); - free(f->buffer); + f->close(f); free(f); } -static int -rdbuf(struct fastbuf *f) -{ - int l = read(f->fd, f->buffer, f->buflen); - - if (l < 0) - die("Error reading %s: %m", f->name); - f->bptr = f->buffer; - f->bstop = f->buffer + l; - f->pos = f->fdpos; - f->fdpos += l; - return l; -} - -static void -wrbuf(struct fastbuf *f) -{ - int l = f->bptr - f->buffer; - char *c = f->buffer; - - while (l) - { - int z = write(f->fd, c, l); - if (z <= 0) - die("Error writing %s: %m", f->name); - /* FIXME */ - if (z != l) - log(L_ERROR "wrbuf: %d != %d (pos %Ld)", z, l, sh_seek(f->fd, 0, SEEK_CUR)); - f->fdpos += z; - l -= z; - c += z; - } - f->bptr = f->buffer; - f->pos = f->fdpos; -} - void bflush(struct fastbuf *f) { if (f->bptr != f->buffer) @@ -102,7 +27,7 @@ void bflush(struct fastbuf *f) f->pos = f->fdpos; } else /* Write data... */ - wrbuf(f); + f->spout(f); } } @@ -113,9 +38,7 @@ inline void bsetpos(struct fastbuf *f, sh_off_t pos) else { bflush(f); - if (f->fdpos != pos && sh_seek(f->fd, pos, SEEK_SET) < 0) - die("lseek on %s: %m", f->name); - f->fdpos = f->pos = pos; + f->seek(f, pos, SEEK_SET); } } @@ -131,10 +54,7 @@ void bseek(struct fastbuf *f, sh_off_t pos, int whence) return bsetpos(f, btell(f) + pos); case SEEK_END: bflush(f); - l = sh_seek(f->fd, pos, whence); - if (l < 0) - die("lseek on %s: %m", f->name); - f->fdpos = f->pos = l; + f->seek(f, pos, SEEK_END); break; default: die("bseek: invalid whence=%d", whence); @@ -145,7 +65,7 @@ int bgetc_slow(struct fastbuf *f) { if (f->bptr < f->bstop) return *f->bptr++; - if (!rdbuf(f)) + if (!f->refill(f)) return EOF; return *f->bptr++; } @@ -154,7 +74,7 @@ int bpeekc_slow(struct fastbuf *f) { if (f->bptr < f->bstop) return *f->bptr; - if (!rdbuf(f)) + if (!f->refill(f)) return EOF; return *f->bptr; } @@ -162,7 +82,7 @@ int bpeekc_slow(struct fastbuf *f) void bputc_slow(struct fastbuf *f, byte c) { if (f->bptr >= f->bufend) - wrbuf(f); + f->spout(f); *f->bptr++ = c; } @@ -274,7 +194,7 @@ void bread_slow(struct fastbuf *f, void *b, uns l) if (!k) { - rdbuf(f); + f->refill(f); k = f->bstop - f->bptr; if (!k) die("bread on %s: file exhausted", f->name); @@ -296,7 +216,7 @@ void bwrite_slow(struct fastbuf *f, void *b, uns l) if (!k) { - wrbuf(f); + f->spout(f); k = f->bufend - f->bptr; } if (k > l) @@ -329,60 +249,3 @@ bgets(struct fastbuf *f, byte *b, uns l) } die("%s: Line too long", f->name); } - -void bbcopy(struct fastbuf *f, struct fastbuf *t, uns l) -{ - uns rf = f->bstop - f->bptr; - - if (!l) - return; - if (rf) - { - uns k = (rf <= l) ? rf : l; - bwrite(t, f->bptr, k); - f->bptr += k; - l -= k; - } - while (l >= t->buflen) - { - wrbuf(t); - if ((uns) read(f->fd, t->buffer, t->buflen) != t->buflen) - die("bbcopy: %s exhausted", f->name); - f->pos = f->fdpos; - f->fdpos += t->buflen; - f->bstop = f->bptr = f->buffer; - t->bptr = t->bufend; - l -= t->buflen; - } - while (l) - { - uns k = t->bufend - t->bptr; - - if (!k) - { - wrbuf(t); - k = t->bufend - t->bptr; - } - if (k > l) - k = l; - bread(f, t->bptr, k); - t->bptr += k; - l -= k; - } -} - -#ifdef TEST - -int main(int argc, char **argv) -{ - struct fastbuf *f, *t; - int c; - - f = bopen("/etc/profile", O_RDONLY, 16); - t = bfdopen(1, 13); - bbcopy(f, t, 100); - bclose(f); - bclose(t); -} - -#endif diff --git a/lib/fastbuf.h b/lib/fastbuf.h index c367a1ab..d5ca3e71 100644 --- a/lib/fastbuf.h +++ b/lib/fastbuf.h @@ -1,28 +1,68 @@ /* - * Sherlock Library -- Fast File Buffering + * Sherlock Library -- Fast Buffered I/O * - * (c) 1997--1999 Martin Mares + * (c) 1997--2000 Martin Mares */ #ifndef EOF #include #endif +/* + * Generic buffered I/O on a top of buffer swapping functions. + * + * Buffer layout when reading: + * + * +----------------+---------------------------+ + * | read data | free space | + * +----------------+---------------------------+ + * ^ ^ ^ ^ + * buffer bptr bstop bufend + * + * After the last character is read, bptr == bstop and buffer refill + * is deferred to the next read attempt. This gives us an easy way + * how to implement bungetc(). + * + * When writing: + * + * +----------------+---------------------------+ + * | written data | free space | + * +----------------+---------------------------+ + * ^ ^ ^ + * buffer=bstop bptr bufend + */ + struct fastbuf { byte *bptr, *bstop; /* Access pointers */ byte *buffer, *bufend; /* Start and end of the buffer */ byte *name; /* File name for error messages */ - uns buflen; /* Size of standard portion of the buffer */ - sh_off_t pos; /* Position of bptr in the file */ - sh_off_t fdpos; /* Current position in the file */ - int fd; /* File descriptor */ + uns buflen; /* Size of the buffer */ + sh_off_t pos; /* Position of buffer start in the file */ + sh_off_t fdpos; /* Current position in the non-buffered file */ + int fd; /* File descriptor, -1 if not a real file */ + void *lldata; /* Data private to access functions below */ + void *llpos; /* ... continued ... */ + int (*refill)(struct fastbuf *); /* Get a buffer with new data */ + void (*spout)(struct fastbuf *); /* Write buffer data to the file */ + void (*seek)(struct fastbuf *, sh_off_t, int); /* Slow path for bseek(), buffer already flushed */ + void (*close)(struct fastbuf *); /* Close the stream */ }; +/* FastIO on standard files */ + struct fastbuf *bopen(byte *name, uns mode, uns buffer); struct fastbuf *bfdopen(int fd, uns buffer); +void bbcopy(struct fastbuf *f, struct fastbuf *t, uns l); + +/* FastIO on in-memory streams */ + +struct fastbuf *fbmem_create(unsigned blocksize); /* Create stream and return its writing fastbuf */ +struct fastbuf *fbmem_clone_read(struct fastbuf *); /* Create reading fastbuf */ + +/* Universal functions working on all fastbuf's */ + void bclose(struct fastbuf *f); void bflush(struct fastbuf *f); - void bseek(struct fastbuf *f, sh_off_t pos, int whence); void bsetpos(struct fastbuf *f, sh_off_t pos); @@ -248,7 +288,6 @@ extern inline void bwrite(struct fastbuf *f, void *b, uns l) bwrite_slow(f, b, l); } -void bbcopy(struct fastbuf *f, struct fastbuf *t, uns l); byte *bgets(struct fastbuf *f, byte *b, uns l); /* Non-std */ extern inline void @@ -264,6 +303,8 @@ bputsn(struct fastbuf *f, byte *b) bputc(f, '\n'); } +/* Depending on compile-time configuration, we select the right function for reading/writing of file offsets */ + #ifdef SHERLOCK_CONFIG_LARGE_DB #define bgeto(f) bget5(f) #define bputo(f,l) bput5(f,l) diff --git a/lib/fb-file.c b/lib/fb-file.c new file mode 100644 index 00000000..5fca69da --- /dev/null +++ b/lib/fb-file.c @@ -0,0 +1,164 @@ +/* + * Sherlock Library -- Fast Buffered I/O on Files + * + * (c) 1997--2000 Martin Mares + */ + +#include +#include +#include +#include + +#include "lib.h" +#include "fastbuf.h" +#include "lfs.h" + +static int +bfd_refill(struct fastbuf *f) +{ + int l = read(f->fd, f->buffer, f->buflen); + + if (l < 0) + die("Error reading %s: %m", f->name); + f->bptr = f->buffer; + f->bstop = f->buffer + l; + f->pos = f->fdpos; + f->fdpos += l; + return l; +} + +static void +bfd_spout(struct fastbuf *f) +{ + int l = f->bptr - f->buffer; + char *c = f->buffer; + + while (l) + { + int z = write(f->fd, c, l); + if (z <= 0) + die("Error writing %s: %m", f->name); + f->fdpos += z; + l -= z; + c += z; + } + f->bptr = f->buffer; + f->pos = f->fdpos; +} + +static void +bfd_seek(struct fastbuf *f, sh_off_t pos, int whence) +{ + sh_off_t l; + + if (whence == SEEK_SET && pos == f->fdpos) + return; + + l = sh_seek(f->fd, pos, whence); + if (l < 0) + die("lseek on %s: %m", f->name); + f->fdpos = f->pos = l; +} + +static void +bfd_close(struct fastbuf *f) +{ + close(f->fd); +} + +struct fastbuf * +bfdopen_internal(int fd, uns buflen, byte *name) +{ + int namelen = strlen(name) + 1; + struct fastbuf *b = xmalloc(sizeof(struct fastbuf) + buflen + namelen); + + b->buflen = buflen; + b->buffer = (char *)(b+1); + b->bptr = b->bstop = b->buffer; + b->bufend = b->buffer + buflen; + b->name = b->bufend; + strcpy(b->name, name); + b->pos = b->fdpos = 0; + b->fd = fd; + b->refill = bfd_refill; + b->spout = bfd_spout; + b->seek = bfd_seek; + b->close = bfd_close; + return b; +} + +struct fastbuf * +bopen(byte *name, uns mode, uns buffer) +{ + int fd = sh_open(name, mode, 0666); + if (fd < 0) + die("Unable to %s file %s: %m", + (mode & O_CREAT) ? "create" : "open", name); + return bfdopen_internal(fd, buffer, name); +} + +struct fastbuf * +bfdopen(int fd, uns buffer) +{ + byte x[32]; + + sprintf(x, "fd%d", fd); + return bfdopen_internal(fd, buffer, x); +} + +void bbcopy(struct fastbuf *f, struct fastbuf *t, uns l) +{ + uns rf = f->bstop - f->bptr; + + if (!l) + return; + if (rf) + { + uns k = (rf <= l) ? rf : l; + bwrite(t, f->bptr, k); + f->bptr += k; + l -= k; + } + while (l >= t->buflen) + { + t->spout(t); + if ((uns) read(f->fd, t->buffer, t->buflen) != t->buflen) + die("bbcopy: %s exhausted", f->name); + f->pos = f->fdpos; + f->fdpos += t->buflen; + f->bstop = f->bptr = f->buffer; + t->bptr = t->bufend; + l -= t->buflen; + } + while (l) + { + uns k = t->bufend - t->bptr; + + if (!k) + { + t->spout(t); + k = t->bufend - t->bptr; + } + if (k > l) + k = l; + bread(f, t->bptr, k); + t->bptr += k; + l -= k; + } +} + +#ifdef TEST + +int main(int argc, char **argv) +{ + struct fastbuf *f, *t; + int c; + + f = bopen("/etc/profile", O_RDONLY, 16); + t = bfdopen(1, 13); + bbcopy(f, t, 100); + bclose(f); + bclose(t); +} + +#endif diff --git a/lib/fb-mem.c b/lib/fb-mem.c new file mode 100644 index 00000000..2000a35f --- /dev/null +++ b/lib/fb-mem.c @@ -0,0 +1,198 @@ +/* + * Sherlock Library -- Fast Buffered I/O on Memory Streams + * + * (c) 1997--2000 Martin Mares + */ + +#include +#include + +#include "lib.h" +#include "fastbuf.h" + +struct memstream { + unsigned blocksize; + unsigned uc; + struct msblock *first; +}; + +struct msblock { + struct msblock *next; + unsigned size; + byte data[0]; +}; + +int +fbmem_refill(struct fastbuf *f) +{ + struct memstream *s = f->lldata; + struct msblock *b = f->llpos; + + if (!b) + { + b = s->first; + if (!b) + return 0; + } + else if (f->bstop < b->data + b->size) + { + f->bstop = b->data + b->size; + return 1; + } + else + { + if (!b->next) + return 0; + f->pos += b->size; + b = b->next; + } + f->buffer = f->bptr = b->data; + f->bufend = f->bstop = b->data + b->size; + f->llpos = b; + return 1; +} + +void +fbmem_spout(struct fastbuf *f) +{ + struct memstream *s = f->lldata; + struct msblock *b = f->llpos; + struct msblock *bb; + + if (b) + { + b->size = f->bptr - b->data; + if (b->size < s->blocksize) + return; + f->pos += b->size; + } + bb = xmalloc(sizeof(struct msblock) + s->blocksize); + if (b) + b->next = bb; + else + s->first = bb; + bb->next = NULL; + bb->size = 0; + f->buffer = f->bptr = f->bstop = bb->data; + f->bufend = bb->data + s->blocksize; + f->llpos = bb; +} + +void +fbmem_seek(struct fastbuf *f, sh_off_t pos, int whence) +{ + struct memstream *m = f->lldata; + struct msblock *b; + unsigned int p = 0; + + if (whence != SEEK_SET) + die("fbmem_seek: only SEEK_SET supported"); + for (b=m->first; b; b=b->next) + { + if (pos <= p + b->size) /* <=, because we need to be able to seek just after file end */ + { + f->pos = p; + f->buffer = b->data; + f->bptr = b->data + (pos - p); + f->bufend = f->bstop = b->data + b->size; + f->llpos = b; + return; + } + p += b->size; + } + die("fbmem_seek to invalid offset"); +} + +void +fbmem_close(struct fastbuf *f) +{ + struct memstream *m = f->lldata; + struct msblock *b; + + if (--m->uc) + return; + + while (b = m->first) + { + m->first = b->next; + free(b); + } + free(m); +} + +struct fastbuf * +fbmem_create(unsigned blocksize) +{ + struct fastbuf *f = xmalloc(sizeof(struct fastbuf)); + struct memstream *m = xmalloc(sizeof(struct memstream)); + + m->blocksize = blocksize; + m->uc = 1; + m->first = NULL; + + f->bptr = f->bstop = f->buffer = f->bufend = NULL; + f->pos = f->fdpos = 0; + f->name = ""; + f->lldata = m; + f->refill = NULL; + f->spout = fbmem_spout; + f->seek = NULL; + f->close = fbmem_close; + return f; +} + +struct fastbuf * +fbmem_clone_read(struct fastbuf *b) +{ + struct fastbuf *f = xmalloc(sizeof(struct fastbuf)); + struct memstream *s = b->lldata; + + s->uc++; + + f->bptr = f->bstop = f->buffer = f->bufend = NULL; + f->pos = f->fdpos = 0; + f->name = ""; + f->lldata = s; + f->refill = fbmem_refill; + f->spout = NULL; + f->seek = fbmem_seek; + f->close = fbmem_close; + return f; +} + +#ifdef TEST + +int main(void) +{ + struct fastbuf *w, *r; + int t; + + w = fbmem_create(7); + r = fbmem_clone_read(w); + bwrite(w, "12345", 5); + bwrite(w, "12345", 5); + printf("<%d>", btell(w)); + bflush(w); + printf("<%d>", btell(w)); + printf("<%d>", btell(r)); + while ((t = bgetc(r)) >= 0) + putchar(t); + printf("<%d>", btell(r)); + bwrite(w, "12345", 5); + bwrite(w, "12345", 5); + printf("<%d>", btell(w)); + bclose(w); + bsetpos(r, 0); + printf("", btell(r)); + while ((t = bgetc(r)) >= 0) + putchar(t); + bsetpos(r, 3); + printf("", btell(r)); + while ((t = bgetc(r)) >= 0) + putchar(t); + fflush(stdout); + bclose(r); + return 0; +} + +#endif -- 2.39.5