From: Jan 'Moskyt' Matejka Date: Wed, 4 Jul 2012 14:17:59 +0000 (+0200) Subject: Fastbuf: fbmulti backend X-Git-Tag: v5.99~130 X-Git-Url: http://mj.ucw.cz/gitweb/?a=commitdiff_plain;h=c2b6e2fa0eb287e4eb6c7bab1489efc6b41dba3d;p=libucw.git Fastbuf: fbmulti backend --- diff --git a/ucw/Makefile b/ucw/Makefile index 742f09ac..7d7b82fe 100644 --- a/ucw/Makefile +++ b/ucw/Makefile @@ -16,7 +16,7 @@ LIBUCW_MODS= \ conf-context conf-alloc conf-dump conf-input conf-intr conf-journal conf-parse conf-section conf-getopt \ ipaccess \ fastbuf ff-binary ff-string ff-printf ff-unicode ff-stkstring \ - fb-file fb-mem fb-temp tempfile fb-mmap fb-limfd fb-buffer fb-grow fb-pool fb-atomic fb-param fb-socket \ + fb-file fb-mem fb-temp tempfile fb-mmap fb-limfd fb-buffer fb-grow fb-pool fb-atomic fb-param fb-socket fb-multi \ char-cat char-upper char-lower unicode stkstring \ wildmatch regex \ prime primetable random \ @@ -116,7 +116,7 @@ TESTS+=$(addprefix $(o)/ucw/,regex.test unicode.test hash-test.test mempool.test slists.test bbuf.test kmp-test.test getopt.test ff-unicode.test eltpool.test \ fb-socket.test trie-test.test string.test sha1.test asort-test.test binheap-test.test \ redblack-test.test fb-file.test fb-grow.test fb-pool.test fb-atomic.test \ - fb-limfd.test fb-temp.test fb-mem.test fb-buffer.test fb-mmap.test url.test strtonum-test.test \ + fb-limfd.test fb-temp.test fb-mem.test fb-buffer.test fb-mmap.test fb-multi.test url.test strtonum-test.test \ gary.test time.test crc.test) $(o)/ucw/regex.test: $(o)/ucw/regex-t @@ -139,7 +139,7 @@ $(o)/ucw/binheap-test.test: $(o)/ucw/binheap-test $(o)/ucw/redblack-test.test: $(o)/ucw/redblack-test $(o)/ucw/strtonum-test.test: $(o)/ucw/strtonum-test $(addprefix $(o)/ucw/fb-,file.test grow.test pool.test socket.test atomic.test \ - limfd.test temp.test mem.test buffer.test mmap.test): %.test: %-t + limfd.test temp.test mem.test buffer.test mmap.test multi.test): %.test: %-t $(o)/ucw/url.test: $(o)/ucw/url-t $(o)/ucw/gary.test: $(o)/ucw/gary-t $(o)/ucw/time.test: $(o)/ucw/time-conf-t diff --git a/ucw/fastbuf.h b/ucw/fastbuf.h index 6d02ac03..aeda6da2 100644 --- a/ucw/fastbuf.h +++ b/ucw/fastbuf.h @@ -473,6 +473,31 @@ static inline void fbatomic_commit(struct fastbuf *b) fbatomic_internal_write(b); } +/*** + * === Fastbufs atop other fastbufs [[fbmulti]] + * + * Imagine some code which does massive string processing. It takes an input + * buffer, writes a part of it into an output buffer, then some other string + * and then the remaining part of the input buffer. Or anything else where you + * copy all the data at each stage of the complicated process. + * + * This backend takes multiple fastbufs and concatenates them formally into + * one. You may then read them consecutively as they were one fastbuf at all. + * + * This backend is read-only. + * + * This backend is seekable iff all of the supplied fastbufs are seekable. + * + * Please note that no cleanup of underlying fastbufs is provided. + * + * Also, please be aware of direct operations on the underlying buffers. The + * fbmulti backend doesn't expect it. + * + * The last parameter must be NULL. + ***/ + +struct fastbuf* fbmulti_create(uns bufsize, ...) SENTINEL_CHECK; + /*** === Configuring stream parameters [[bconfig]] ***/ enum bconfig_type { /** Parameters that could be configured. **/ diff --git a/ucw/fb-multi.c b/ucw/fb-multi.c new file mode 100644 index 00000000..20d8590e --- /dev/null +++ b/ucw/fb-multi.c @@ -0,0 +1,229 @@ +/* + * UCW Library -- Fast Buffered I/O on itself + * + * (c) 2012 Jan Moskyto Matejka + * + * This software may be freely distributed and used according to the terms + * of the GNU Lesser General Public License. + */ + +#include +#include +#include +#include + +#include + +struct fb_multi { + struct fastbuf fb; + struct mempool* mp; + struct subbuf* cur; + ucw_off_t len; + clist* subbufs; +}; +#define FB_MULTI(f) ((struct fb_multi *)(f)) + +struct subbuf { + cnode n; + ucw_off_t begin, end; + struct fastbuf* fb; +}; +#define SUBBUF(f) ((struct subbuf *)(f)) + +static inline void +fbmulti_subbuf_get_end(struct subbuf *s) +{ + if (s->fb->seek) { + bseek(s->fb, 0, SEEK_END); + s->end = s->begin + btell(s->fb); + } +} + +static inline int +fbmulti_subbuf_next(struct fastbuf *f) +{ + struct subbuf* next = clist_next(FB_MULTI(f)->subbufs, &FB_MULTI(f)->cur->n); + if (next == NULL) + return 0; + + if (f->seek) + next->begin = FB_MULTI(f)->cur->end; + FB_MULTI(f)->cur = next; + return 1; +} + +static int +fbmulti_refill(struct fastbuf *f) +{ + if (f->bufend == f->bstop) + f->bptr = f->bstop = f->buffer; + uns len = bread(FB_MULTI(f)->cur->fb, f->bstop, (f->bufend - f->bstop)); + f->bstop += len; + f->pos += len; + if (len) + return len; + + // Current buf returned EOF + // Update the information on end of this buffer + fbmulti_subbuf_get_end(FB_MULTI(f)->cur); + + // Take the next one if exists + if (fbmulti_subbuf_next(f)) + return fbmulti_refill(f); + else + return 0; +} + +static void +fbmulti_get_len(struct fastbuf *f) +{ + ASSERT (f->seek); + FB_MULTI(f)->len = 0; + + CLIST_FOR_EACH(struct subbuf *, n, *(FB_MULTI(f)->subbufs)) + { + n->begin = FB_MULTI(f)->len; + fbmulti_subbuf_get_end(n); + FB_MULTI(f)->len = n->end; + } +} + +static int +fbmulti_seek(struct fastbuf *f, ucw_off_t pos, int whence) +{ + switch(whence) + { + case SEEK_SET: + if (f->pos > pos) { + FB_MULTI(f)->cur = clist_head(FB_MULTI(f)->subbufs); + FB_MULTI(f)->cur->begin = 0; + f->pos = 0; + return fbmulti_seek(f, pos, SEEK_SET); + } + + do { + fbmulti_subbuf_get_end(FB_MULTI(f)->cur); + if (pos < FB_MULTI(f)->cur->end) + break; + + if (!fbmulti_subbuf_next(f)) + bthrow(f, "seek", "Seek out of range"); + + } while (1); + + bseek(FB_MULTI(f)->cur->fb, (pos - FB_MULTI(f)->cur->begin), SEEK_SET); + f->pos = pos; + f->bptr = f->bstop = f->buffer; + return 1; + break; + + case SEEK_END: + fbmulti_get_len(f); + return fbmulti_seek(f, FB_MULTI(f)->len+pos, SEEK_CUR); + break; + + default: + ASSERT(0); + } +} + +static void +fbmulti_update_capability(struct fastbuf *f) { + // FB Multi is only a proxy to other fastbufs ... if any of them lacks + // support of any feature, FB Multi also provides no support of that feature + f->refill = fbmulti_refill; + f->seek = fbmulti_seek; + + CLIST_FOR_EACH(struct subbuf *, n, *(FB_MULTI(f)->subbufs)) { + if (!n->fb->refill) + f->refill = NULL; + + if (!n->fb->seek) + f->seek = NULL; + } +} + +struct fastbuf* +fbmulti_create(uns bufsize, ...) +{ + struct mempool *mp = mp_new(bufsize); + struct fastbuf *fb_out = mp_alloc(mp, sizeof(struct fb_multi)); + FB_MULTI(fb_out)->mp = mp; + + struct fastbuf *fb_in; + clist* subbufs = mp_alloc(mp, sizeof(clist)); + clist_init(subbufs); + FB_MULTI(fb_out)->subbufs = subbufs; + + va_list args; + va_start(args, bufsize); + while (fb_in = va_arg(args, struct fastbuf *)) { + struct subbuf *sb = mp_alloc(mp, sizeof(struct subbuf)); + sb->fb = fb_in; + clist_add_tail(subbufs, &(sb->n)); + } + va_end(args); + + FB_MULTI(fb_out)->cur = clist_head(subbufs); + + fb_out->buffer = mp_alloc(mp, bufsize); + fb_out->bptr = fb_out->bstop = fb_out->buffer; + fb_out->bufend = fb_out->buffer + bufsize; + fb_out->name = ""; + + fbmulti_update_capability(fb_out); + + return fb_out; +} + +#ifdef TEST + +int main(int argc, char ** argv) +{ + if (argc < 2) + { + fprintf(stderr, "You must specify a test (r, w, o)\n"); + return 1; + } + switch (*argv[1]) + { + case 'r': + { + char *data[] = { "One\nLine", "Two\nLines", "Th\nreeLi\nnes\n" }; + struct fastbuf fb[ARRAY_SIZE(data)]; + for (uns i=0;i