From: Martin Mares Date: Tue, 28 Jan 2014 20:50:04 +0000 (+0100) Subject: Introduced generic allocators X-Git-Tag: v5.99~8 X-Git-Url: http://mj.ucw.cz/gitweb/?a=commitdiff_plain;h=a1fd264e82dc24343c8129a3e8e611f976f262c9;p=libucw.git Introduced generic allocators The concept of generic allocators, previously used locally in gary's, turned out to be very promising and applicable to other data structures in the future. So I decided to decouple allocators from gary.[ch] and introduce them to the society of libucw primitives. now provides allocators for malloc and zeroed malloc, each mempool can also serve as an allocator (the current implementation has an instance of struct ucw_allocator in each struct mempool, but it is always accessed through an inline function, so it can be changed if the overhead is deemed too heavy). Growing arrays have been redefined in terms of generic allocators. The realloc module has been merged to alloc.c to keep our malloc wrappers together. --- diff --git a/ucw/Makefile b/ucw/Makefile index faef00e9..9f050505 100644 --- a/ucw/Makefile +++ b/ucw/Makefile @@ -1,4 +1,4 @@ -# Makefile for the UCW Library (c) 1997--2010 Martin Mares +# Makefile for the UCW Library (c) 1997--2014 Martin Mares DIRS+=ucw LIBUCW=$(o)/ucw/libucw.pc @@ -9,7 +9,8 @@ endif LIBUCW_MODS= \ threads \ - alloc alloc_str realloc bigalloc mempool mempool-str mempool-fmt eltpool \ + alloc alloc_str alloc-std \ + bigalloc mempool mempool-str mempool-fmt eltpool \ partmap hashfunc \ slists simple-lists bitsig \ log log-stream log-file log-syslog log-conf tbf \ @@ -30,7 +31,7 @@ LIBUCW_MODS= \ base64 base224 \ io-careful io-sync io-mmap io-size \ string str-esc str-split str-match str-imatch str-hex str-fix \ - bbuf gary gary-mp \ + bbuf gary \ getopt \ strtonum \ resource trans res-fd res-mem res-subpool res-mempool res-eltpool \ @@ -40,7 +41,7 @@ LIBUCW_MODS= \ LIBUCW_MAIN_INCLUDES= \ lib.h log.h threads.h time.h \ - mempool.h eltpool.h \ + alloc.h mempool.h eltpool.h \ clists.h slists.h simple-lists.h \ string.h stkstring.h unicode.h varint.h chartype.h regex.h \ wildmatch.h \ @@ -165,7 +166,7 @@ $(o)/ucw/strtonum-test.test: $(o)/ucw/strtonum-test $(addprefix $(o)/ucw/fb-,file.test grow.test pool.test socket.test atomic.test \ limfd.test temp.test mem.test buffer.test mmap.test multi.test): %.test: %-t $(o)/ucw/url.test: $(o)/ucw/url-t -$(o)/ucw/gary.test: $(o)/ucw/gary-t $(o)/ucw/gary-mp-t +$(o)/ucw/gary.test: $(o)/ucw/gary-t $(o)/ucw/time.test: $(o)/ucw/time-conf-t $(o)/ucw/crc.test: $(o)/ucw/crc-t $(o)/ucw/signames.test: $(o)/ucw/signames-t diff --git a/ucw/alloc-std.c b/ucw/alloc-std.c new file mode 100644 index 00000000..f3b1d462 --- /dev/null +++ b/ucw/alloc-std.c @@ -0,0 +1,57 @@ +/* + * UCW Library -- Generic Allocator Using Malloc + * + * (c) 2014 Martin Mares + * + * This software may be freely distributed and used according to the terms + * of the GNU Lesser General Public License. + */ + +#include +#include + +#include + +/* Default allocator */ + +static void *ucw_std_alloc(struct ucw_allocator *a UNUSED, size_t size) +{ + return xmalloc(size); +} + +static void *ucw_std_realloc(struct ucw_allocator *a UNUSED, void *ptr, size_t old_size UNUSED, size_t new_size) +{ + return xrealloc(ptr, new_size); +} + +static void ucw_std_free(struct ucw_allocator *a UNUSED, void *ptr) +{ + xfree(ptr); +} + +struct ucw_allocator ucw_allocator_std = { + .alloc = ucw_std_alloc, + .realloc = ucw_std_realloc, + .free = ucw_std_free, +}; + +/* Zeroing allocator */ + +static void *ucw_zeroed_alloc(struct ucw_allocator *a UNUSED, size_t size) +{ + return xmalloc_zero(size); +} + +static void *ucw_zeroed_realloc(struct ucw_allocator *a UNUSED, void *ptr, size_t old_size, size_t new_size) +{ + ptr = xrealloc(ptr, new_size); + if (old_size < new_size) + bzero((byte *) ptr + old_size, new_size - old_size); + return ptr; +} + +struct ucw_allocator ucw_allocator_zeroed = { + .alloc = ucw_zeroed_alloc, + .realloc = ucw_zeroed_realloc, + .free = ucw_std_free, +}; diff --git a/ucw/alloc.c b/ucw/alloc.c index c4e40640..816da2a0 100644 --- a/ucw/alloc.c +++ b/ucw/alloc.c @@ -39,3 +39,13 @@ xfree(void *ptr) */ free(ptr); } + +void * +xrealloc(void *old, size_t size) +{ + /* We assume that realloc(NULL, x) works like malloc(x), which is true with the glibc. */ + void *x = realloc(old, size); + if (!x && size) + die("Cannot reallocate %zu bytes of memory", size); + return x; +} diff --git a/ucw/alloc.h b/ucw/alloc.h new file mode 100644 index 00000000..62924719 --- /dev/null +++ b/ucw/alloc.h @@ -0,0 +1,21 @@ +/* + * UCW Library -- Generic allocators + * + * (c) 2014 Martin Mares + */ + +#ifndef _UCW_ALLOC_H +#define _UCW_ALLOC_H + +struct ucw_allocator { + void * (*alloc)(struct ucw_allocator *alloc, size_t size); + void * (*realloc)(struct ucw_allocator *alloc, void *ptr, size_t old_size, size_t new_size); + void (*free)(struct ucw_allocator *alloc, void *ptr); +}; + +/* alloc-std.c */ + +extern struct ucw_allocator ucw_allocator_std; +extern struct ucw_allocator ucw_allocator_zeroed; + +#endif diff --git a/ucw/conf-intr.c b/ucw/conf-intr.c index 79a9e8da..509dc8b6 100644 --- a/ucw/conf-intr.c +++ b/ucw/conf-intr.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -110,8 +111,7 @@ interpret_set_dynamic(struct cf_item *item, int number, char **pars, void **ptr) uns size = cf_type_size(type, item->u.utype); cf_journal_block(ptr, sizeof(void*)); // boundary checks done by the caller - struct gary_allocator *a = gary_new_allocator_mp(cf_get_pool()); // FIXME: One copy should be enough - *ptr = gary_init(size, number, a); + *ptr = gary_init(size, number, mp_get_allocator(cf_get_pool())); return cf_parse_ary(number, pars, *ptr, type, &item->u); } @@ -126,8 +126,7 @@ interpret_add_dynamic(struct cf_item *item, int number, char **pars, int *proces int taken = MIN(number, ABS(item->number)-old_nr); *processed = taken; // stretch the dynamic array - struct gary_allocator *a = gary_new_allocator_mp(cf_get_pool()); // FIXME: One copy should be enough - void *new_p = gary_init(size, old_nr + taken, a); + void *new_p = gary_init(size, old_nr + taken, mp_get_allocator(cf_get_pool())); cf_journal_block(ptr, sizeof(void*)); *ptr = new_p; if (op == OP_APPEND) { diff --git a/ucw/gary-mp.c b/ucw/gary-mp.c deleted file mode 100644 index 77e30aab..00000000 --- a/ucw/gary-mp.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * UCW Library -- Growing arrays over mempools - * - * (c) 2014 Martin Mares - */ - -#include -#include -#include - -#include - -static void *gary_mp_alloc(struct gary_allocator *a, size_t size) -{ - return mp_alloc(a->data, size); -} - -static void *gary_mp_realloc(struct gary_allocator *a, void *ptr, size_t old_size, size_t new_size) -{ - if (new_size <= old_size) - return ptr; - - /* - * In the future, we might want to do something like mp_realloc(), - * but we have to check that it is indeed the last block in the pool. - */ - void *new = mp_alloc(a->data, new_size); - memcpy(new, ptr, old_size); - return new; -} - -static void gary_mp_free(struct gary_allocator *a UNUSED, void *ptr UNUSED) -{ -} - -struct gary_allocator *gary_new_allocator_mp(struct mempool *mp) -{ - struct gary_allocator *a = mp_alloc(mp, sizeof(*a)); - *a = (struct gary_allocator) { - .alloc = gary_mp_alloc, - .realloc = gary_mp_realloc, - .free = gary_mp_free, - .data = mp, - }; - return a; -} - -#ifdef TEST - -#include - -int main(void) -{ - struct mempool *mp = mp_new(4096); - struct gary_allocator *alloc = gary_new_allocator_mp(mp); - int *a; - GARY_INIT_ALLOC(a, 5, alloc); - - for (int i=0; i<5; i++) - a[i] = i+1; - - GARY_PUSH(a); - *GARY_PUSH(a) = 10; - *GARY_PUSH(a) = 20; - *GARY_PUSH(a) = 30; - GARY_POP(a); - GARY_FIX(a); - - for (int i=0; i<(int)GARY_SIZE(a); i++) - printf("%d\n", a[i]); - - GARY_FREE(a); - mp_delete(mp); - return 0; -} - -#endif diff --git a/ucw/gary.c b/ucw/gary.c index a4a45706..8207de39 100644 --- a/ucw/gary.c +++ b/ucw/gary.c @@ -14,7 +14,7 @@ struct gary_hdr gary_empty_hdr; void * -gary_init(size_t elt_size, size_t num_elts, struct gary_allocator *allocator) +gary_init(size_t elt_size, size_t num_elts, struct ucw_allocator *allocator) { DBG("GARY: Init to %zd elements", num_elts); struct gary_hdr *h = allocator->alloc(allocator, GARY_HDR_SIZE + elt_size * num_elts); @@ -69,62 +69,28 @@ gary_fix(void *array) return GARY_BODY(h); } -/* Default allocator */ - -static void *gary_default_alloc(struct gary_allocator *a UNUSED, size_t size) -{ - return xmalloc(size); -} - -static void *gary_default_realloc(struct gary_allocator *a UNUSED, void *ptr, size_t old_size UNUSED, size_t new_size) -{ - return xrealloc(ptr, new_size); -} - -static void gary_default_free(struct gary_allocator *a UNUSED, void *ptr) -{ - xfree(ptr); -} - -struct gary_allocator gary_allocator_default = { - .alloc = gary_default_alloc, - .realloc = gary_default_realloc, - .free = gary_default_free, -}; - -/* Zeroing allocator */ - -static void *gary_zeroed_alloc(struct gary_allocator *a UNUSED, size_t size) -{ - return xmalloc_zero(size); -} - -static void *gary_zeroed_realloc(struct gary_allocator *a UNUSED, void *ptr, size_t old_size, size_t new_size) -{ - ptr = xrealloc(ptr, new_size); - if (old_size < new_size) - bzero((byte *) ptr + old_size, new_size - old_size); - return ptr; -} - -struct gary_allocator gary_allocator_zeroed = { - .alloc = gary_zeroed_alloc, - .realloc = gary_zeroed_realloc, - .free = gary_default_free, -}; - #ifdef TEST +#include + #include -int main(void) +int main(int argc, char **argv UNUSED) { int *a; - GARY_INIT_ZERO(a, 5); + struct mempool *mp = NULL; + + if (argc < 2) + GARY_INIT_ZERO(a, 5); + else + { + mp = mp_new(4096); + GARY_INIT_ALLOC(a, 5, mp_get_allocator(mp)); + } for (int i=0; i<5; i++) { - ASSERT(!a[i]); + ASSERT(!a[i] || mp); a[i] = i+1; } @@ -139,6 +105,8 @@ int main(void) printf("%d\n", a[i]); GARY_FREE(a); + if (mp) + mp_delete(mp); return 0; } diff --git a/ucw/gary.h b/ucw/gary.h index aa82855c..6610e799 100644 --- a/ucw/gary.h +++ b/ucw/gary.h @@ -7,9 +7,9 @@ #ifndef _UCW_GARY_H #define _UCW_GARY_H +#include + #ifdef CONFIG_UCW_CLEAN_ABI -#define gary_allocator_default ucw_gary_allocator_default -#define gary_allocator_zeroed ucw_gary_allocator_zeroed #define gary_fix ucw_gary_fix #define gary_init ucw_gary_init #define gary_push_helper ucw_gary_push_helper @@ -20,25 +20,15 @@ struct gary_hdr { size_t num_elts; size_t have_space; size_t elt_size; - struct gary_allocator *allocator; -}; - -struct gary_allocator { - void * (*alloc)(struct gary_allocator *alloc, size_t size); - void * (*realloc)(struct gary_allocator *alloc, void *ptr, size_t old_size, size_t new_size); - void (*free)(struct gary_allocator *alloc, void *ptr); - void *data; + struct ucw_allocator *allocator; }; -extern struct gary_allocator gary_allocator_default; -extern struct gary_allocator gary_allocator_zeroed; - #define GARY_HDR_SIZE ALIGN_TO(sizeof(struct gary_hdr), CPU_STRUCT_ALIGN) #define GARY_HDR(ptr) ((struct gary_hdr *)((byte*)(ptr) - GARY_HDR_SIZE)) #define GARY_BODY(ptr) ((byte *)(ptr) + GARY_HDR_SIZE) -#define GARY_INIT(ptr, n) (ptr) = gary_init(sizeof(*(ptr)), (n), &gary_allocator_default) -#define GARY_INIT_ZERO(ptr, n) (ptr) = gary_init(sizeof(*(ptr)), (n), &gary_allocator_zeroed) +#define GARY_INIT(ptr, n) (ptr) = gary_init(sizeof(*(ptr)), (n), &ucw_allocator_std) +#define GARY_INIT_ZERO(ptr, n) (ptr) = gary_init(sizeof(*(ptr)), (n), &ucw_allocator_zeroed) #define GARY_INIT_ALLOC(ptr, n, a) (ptr) = gary_init(sizeof(*(ptr)), (n), (a)) #define GARY_INIT_SPACE(ptr, n) do { GARY_INIT(ptr, n); (GARY_HDR(ptr))->num_elts = 0; } while (0) #define GARY_INIT_SPACE_ZERO(ptr, n) do { GARY_INIT_ZERO(ptr, n); (GARY_HDR(ptr))->num_elts = 0; } while (0) @@ -63,7 +53,7 @@ extern struct gary_allocator gary_allocator_zeroed; #define GARY_FIX(ptr) (ptr) = gary_fix((ptr)) /* Internal functions */ -void *gary_init(size_t elt_size, size_t num_elts, struct gary_allocator *allocator); +void *gary_init(size_t elt_size, size_t num_elts, struct ucw_allocator *allocator); void *gary_set_size(void *array, size_t n); void *gary_push_helper(void *array, size_t n, byte **cptr); void *gary_fix(void *array); @@ -91,9 +81,4 @@ extern struct gary_hdr gary_empty_hdr; (ptr) = gary_push_helper((ptr), 1, (byte **) &_c); \ _c; }) -/* gary-mp.c */ - -struct mempool; -struct gary_allocator *gary_new_allocator_mp(struct mempool *mp); - #endif diff --git a/ucw/gary.t b/ucw/gary.t index 3df10f52..08da6f10 100644 --- a/ucw/gary.t +++ b/ucw/gary.t @@ -10,7 +10,7 @@ Out: 1 10 20 -Run: ../obj/ucw/gary-mp-t +Run: ../obj/ucw/gary-t pool Out: 1 2 3 diff --git a/ucw/mempool.c b/ucw/mempool.c index 521724a8..b55ba925 100644 --- a/ucw/mempool.c +++ b/ucw/mempool.c @@ -1,7 +1,7 @@ /* * UCW Library -- Memory Pools (One-Time Allocation) * - * (c) 1997--2001 Martin Mares + * (c) 1997--2014 Martin Mares * (c) 2007 Pavel Charvat * * This software may be freely distributed and used according to the terms @@ -11,6 +11,7 @@ #undef LOCAL_DEBUG #include +#include #include #include @@ -33,14 +34,46 @@ mp_align_size(uns size) #endif } +static void *mp_allocator_alloc(struct ucw_allocator *a, size_t size) +{ + struct mempool *mp = (struct mempool *) a; + return mp_alloc_fast(mp, size); +} + +static void *mp_allocator_realloc(struct ucw_allocator *a, void *ptr, size_t old_size, size_t new_size) +{ + if (new_size <= old_size) + return ptr; + + /* + * In the future, we might want to do something like mp_realloc(), + * but we have to check that it is indeed the last block in the pool. + */ + struct mempool *mp = (struct mempool *) a; + void *new = mp_alloc_fast(mp, new_size); + memcpy(new, ptr, old_size); + return new; +} + +static void mp_allocator_free(struct ucw_allocator *a UNUSED, void *ptr UNUSED) +{ + // Does nothing +} + void mp_init(struct mempool *pool, uns chunk_size) { chunk_size = mp_align_size(MAX(sizeof(struct mempool), chunk_size)); *pool = (struct mempool) { + .allocator = { + .alloc = mp_allocator_alloc, + .realloc = mp_allocator_realloc, + .free = mp_allocator_free, + }, .chunk_size = chunk_size, .threshold = chunk_size >> 1, - .last_big = &pool->last_big }; + .last_big = &pool->last_big + }; } static void * @@ -90,6 +123,11 @@ mp_new(uns chunk_size) DBG("Creating mempool %p with %u bytes long chunks", pool, chunk_size); chunk->next = NULL; *pool = (struct mempool) { + .allocator = { + .alloc = mp_allocator_alloc, + .realloc = mp_allocator_realloc, + .free = mp_allocator_free, + }, .state = { .free = { chunk_size - sizeof(*pool) }, .last = { chunk } }, .chunk_size = chunk_size, .threshold = chunk_size >> 1, diff --git a/ucw/mempool.h b/ucw/mempool.h index f5aff6f5..a36086ea 100644 --- a/ucw/mempool.h +++ b/ucw/mempool.h @@ -1,7 +1,7 @@ /* * UCW Library -- Memory Pools * - * (c) 1997--2005 Martin Mares + * (c) 1997--2014 Martin Mares * (c) 2007 Pavel Charvat * * This software may be freely distributed and used according to the terms @@ -11,6 +11,8 @@ #ifndef _UCW_POOLS_H #define _UCW_POOLS_H +#include + #ifdef CONFIG_UCW_CLEAN_ABI #define mp_alloc ucw_mp_alloc #define mp_alloc_internal ucw_mp_alloc_internal @@ -65,6 +67,7 @@ struct mempool_state { * You should use this one as an opaque handle only, the insides are internal. **/ struct mempool { + struct ucw_allocator allocator; struct mempool_state state; void *unused, *last_big; uns chunk_size, threshold, idx; @@ -186,6 +189,14 @@ static inline void *mp_alloc_fast_noalign(struct mempool *pool, uns size) return mp_alloc_internal(pool, size); } +/** + * Return a generic allocator representing the given mempool. + **/ +static inline struct ucw_allocator *mp_get_allocator(struct mempool *mp) +{ + return &mp->allocator; +} + /*** * [[gbuf]] * Growing buffers diff --git a/ucw/realloc.c b/ucw/realloc.c deleted file mode 100644 index 4a15e3aa..00000000 --- a/ucw/realloc.c +++ /dev/null @@ -1,22 +0,0 @@ -/* - * UCW Library -- Memory Re-allocation - * - * (c) 1997 Martin Mares - * - * This software may be freely distributed and used according to the terms - * of the GNU Lesser General Public License. - */ - -#include - -#include - -void * -xrealloc(void *old, size_t size) -{ - /* We assume that realloc(NULL, x) works like malloc(x), which is true with the glibc. */ - void *x = realloc(old, size); - if (!x && size) - die("Cannot reallocate %zu bytes of memory", size); - return x; -}