2 * UCW Library -- Memory Pools
4 * (c) 1997--2014 Martin Mares <mj@ucw.cz>
5 * (c) 2007 Pavel Charvat <pchar@ucw.cz>
7 * This software may be freely distributed and used according to the terms
8 * of the GNU Lesser General Public License.
14 #include <ucw/alloc.h>
17 #ifdef CONFIG_UCW_CLEAN_ABI
18 #define mp_alloc ucw_mp_alloc
19 #define mp_alloc_internal ucw_mp_alloc_internal
20 #define mp_alloc_noalign ucw_mp_alloc_noalign
21 #define mp_alloc_zero ucw_mp_alloc_zero
22 #define mp_append_printf ucw_mp_append_printf
23 #define mp_append_vprintf ucw_mp_append_vprintf
24 #define mp_delete ucw_mp_delete
25 #define mp_flush ucw_mp_flush
26 #define mp_grow_internal ucw_mp_grow_internal
27 #define mp_init ucw_mp_init
28 #define mp_memdup ucw_mp_memdup
29 #define mp_multicat ucw_mp_multicat
30 #define mp_new ucw_mp_new
31 #define mp_open ucw_mp_open
32 #define mp_pop ucw_mp_pop
33 #define mp_printf ucw_mp_printf
34 #define mp_push ucw_mp_push
35 #define mp_realloc ucw_mp_realloc
36 #define mp_realloc_zero ucw_mp_realloc_zero
37 #define mp_restore ucw_mp_restore
38 #define mp_spread_internal ucw_mp_spread_internal
39 #define mp_start ucw_mp_start
40 #define mp_start_internal ucw_mp_start_internal
41 #define mp_start_noalign ucw_mp_start_noalign
42 #define mp_stats ucw_mp_stats
43 #define mp_str_from_mem ucw_mp_str_from_mem
44 #define mp_strdup ucw_mp_strdup
45 #define mp_strjoin ucw_mp_strjoin
46 #define mp_total_size ucw_mp_total_size
47 #define mp_vprintf ucw_mp_vprintf
57 * Memory pool state (see @mp_push(), ...).
58 * You should use this one as an opaque handle only, the insides are internal.
60 struct mempool_state {
63 struct mempool_state *next;
68 * You should use this one as an opaque handle only, the insides are internal.
71 struct ucw_allocator allocator;
72 struct mempool_state state;
73 void *unused, *last_big;
74 uns chunk_size, threshold, idx;
78 struct mempool_stats { /** Mempool statistics. See @mp_stats(). **/
79 u64 total_size; /* Real allocated size in bytes */
80 u64 used_size; /* Estimated size allocated from mempool to application */
81 uns chain_count[3]; /* Number of allocated chunks in small/big/unused chains */
82 uns chain_size[3]; /* Size of allocated chunks in small/big/unused chains */
92 * Initialize a given mempool structure.
93 * @chunk_size must be in the interval `[1, UINT_MAX / 2]`.
94 * It will allocate memory by this large chunks and take
95 * memory to satisfy requests from them.
97 * Memory pools can be treated as <<trans:respools,resources>>, see <<trans:res_mempool()>>.
99 void mp_init(struct mempool *pool, uns chunk_size);
102 * Allocate and initialize a new memory pool.
103 * See @mp_init() for @chunk_size limitations.
105 * The new mempool structure is allocated on the new mempool.
107 * Memory pools can be treated as <<trans:respools,resources>>, see <<trans:res_mempool()>>.
109 struct mempool *mp_new(uns chunk_size);
112 * Cleanup mempool initialized by mp_init or mp_new.
113 * Frees all the memory allocated by this mempool and,
114 * if created by @mp_new(), the @pool itself.
116 void mp_delete(struct mempool *pool);
119 * Frees all data on a memory pool, but leaves it working.
120 * It can keep some of the chunks allocated to serve
121 * further allocation requests. Leaves the @pool alive,
122 * even if it was created with @mp_new().
124 void mp_flush(struct mempool *pool);
127 * Compute some statistics for debug purposes.
128 * See the definition of the <<struct_mempool_stats,mempool_stats structure>>.
129 * This function scans the chunk list, so it can be slow. If you are interested
130 * in total memory consumption only, mp_total_size() is faster.
132 void mp_stats(struct mempool *pool, struct mempool_stats *stats);
135 * Return how many bytes were allocated by the pool, including unused parts
136 * of chunks. This function runs in constant time.
138 u64 mp_total_size(struct mempool *pool);
143 * Allocation routines
144 * -------------------
147 /* For internal use only, do not call directly */
148 void *mp_alloc_internal(struct mempool *pool, uns size) LIKE_MALLOC;
151 * The function allocates new @size bytes on a given memory pool.
152 * If the @size is zero, the resulting pointer is undefined,
153 * but it may be safely reallocated or used as the parameter
154 * to other functions below.
156 * The resulting pointer is always aligned to a multiple of
157 * `CPU_STRUCT_ALIGN` bytes and this condition remains true also
158 * after future reallocations.
160 void *mp_alloc(struct mempool *pool, uns size);
163 * The same as @mp_alloc(), but the result may be unaligned.
165 void *mp_alloc_noalign(struct mempool *pool, uns size);
168 * The same as @mp_alloc(), but fills the newly allocated memory with zeroes.
170 void *mp_alloc_zero(struct mempool *pool, uns size);
173 * Inlined version of @mp_alloc().
175 static inline void *mp_alloc_fast(struct mempool *pool, uns size)
177 uns avail = pool->state.free[0] & ~(CPU_STRUCT_ALIGN - 1);
180 pool->state.free[0] = avail - size;
181 return (byte *)pool->state.last[0] - avail;
184 return mp_alloc_internal(pool, size);
188 * Inlined version of @mp_alloc_noalign().
190 static inline void *mp_alloc_fast_noalign(struct mempool *pool, uns size)
192 if (size <= pool->state.free[0])
194 void *ptr = (byte *)pool->state.last[0] - pool->state.free[0];
195 pool->state.free[0] -= size;
199 return mp_alloc_internal(pool, size);
203 * Return a generic allocator representing the given mempool.
205 static inline struct ucw_allocator *mp_get_allocator(struct mempool *mp)
207 return &mp->allocator;
215 * You do not need to know, how a buffer will need to be large,
216 * you can grow it incrementally to needed size. You can grow only
217 * one buffer at a time on a given mempool.
219 * Similar functionality is provided by <<growbuf:,growing buffes>> module.
222 /* For internal use only, do not call directly */
223 void *mp_start_internal(struct mempool *pool, uns size) LIKE_MALLOC;
224 void *mp_grow_internal(struct mempool *pool, uns size);
225 void *mp_spread_internal(struct mempool *pool, void *p, uns size);
227 static inline uns mp_idx(struct mempool *pool, void *ptr)
229 return ptr == pool->last_big;
233 * Open a new growing buffer (at least @size bytes long).
234 * If the @size is zero, the resulting pointer is undefined,
235 * but it may be safely reallocated or used as the parameter
236 * to other functions below.
238 * The resulting pointer is always aligned to a multiple of
239 * `CPU_STRUCT_ALIGN` bytes and this condition remains true also
240 * after future reallocations. There is an unaligned version as well.
242 * Keep in mind that you can't make any other pool allocations
243 * before you "close" the growing buffer with @mp_end().
245 void *mp_start(struct mempool *pool, uns size);
246 void *mp_start_noalign(struct mempool *pool, uns size);
249 * Inlined version of @mp_start().
251 static inline void *mp_start_fast(struct mempool *pool, uns size)
253 uns avail = pool->state.free[0] & ~(CPU_STRUCT_ALIGN - 1);
257 pool->state.free[0] = avail;
258 return (byte *)pool->state.last[0] - avail;
261 return mp_start_internal(pool, size);
265 * Inlined version of @mp_start_noalign().
267 static inline void *mp_start_fast_noalign(struct mempool *pool, uns size)
269 if (size <= pool->state.free[0])
272 return (byte *)pool->state.last[0] - pool->state.free[0];
275 return mp_start_internal(pool, size);
279 * Return start pointer of the growing buffer allocated by latest @mp_start() or a similar function.
281 static inline void *mp_ptr(struct mempool *pool)
283 return (byte *)pool->state.last[pool->idx] - pool->state.free[pool->idx];
287 * Return the number of bytes available for extending the growing buffer.
288 * (Before a reallocation will be needed).
290 static inline uns mp_avail(struct mempool *pool)
292 return pool->state.free[pool->idx];
296 * Grow the buffer allocated by @mp_start() to be at least @size bytes long
297 * (@size may be less than @mp_avail(), even zero). Reallocated buffer may
298 * change its starting position. The content will be unchanged to the minimum
299 * of the old and new sizes; newly allocated memory will be uninitialized.
300 * Multiple calls to mp_grow() have amortized linear cost wrt. the maximum value of @size. */
301 static inline void *mp_grow(struct mempool *pool, uns size)
303 return (size <= mp_avail(pool)) ? mp_ptr(pool) : mp_grow_internal(pool, size);
307 * Grow the buffer by at least one byte -- equivalent to <<mp_grow(),`mp_grow`>>`(@pool, @mp_avail(pool) + 1)`.
309 static inline void *mp_expand(struct mempool *pool)
311 return mp_grow_internal(pool, mp_avail(pool) + 1);
315 * Ensure that there is at least @size bytes free after @p,
316 * if not, reallocate and adjust @p.
318 static inline void *mp_spread(struct mempool *pool, void *p, uns size)
320 return (((uns)((byte *)pool->state.last[pool->idx] - (byte *)p) >= size) ? p : mp_spread_internal(pool, p, size));
324 * Append a character to the growing buffer. Called with @p pointing after
325 * the last byte in the buffer, returns a pointer after the last byte
326 * of the new (possibly reallocated) buffer.
328 static inline char *mp_append_char(struct mempool *pool, char *p, uns c)
330 p = mp_spread(pool, p, 1);
336 * Append a memory block to the growing buffer. Called with @p pointing after
337 * the last byte in the buffer, returns a pointer after the last byte
338 * of the new (possibly reallocated) buffer.
340 static inline void *mp_append_block(struct mempool *pool, void *p, const void *block, uns size)
342 char *q = mp_spread(pool, p, size);
343 memcpy(q, block, size);
348 * Append a string to the growing buffer. Called with @p pointing after
349 * the last byte in the buffer, returns a pointer after the last byte
350 * of the new (possibly reallocated) buffer.
352 static inline void *mp_append_string(struct mempool *pool, void *p, const char *str)
354 return mp_append_block(pool, p, str, strlen(str));
358 * Close the growing buffer. The @end must point just behind the data, you want to keep
359 * allocated (so it can be in the interval `[@mp_ptr(@pool), @mp_ptr(@pool) + @mp_avail(@pool)]`).
360 * Returns a pointer to the beginning of the just closed block.
362 static inline void *mp_end(struct mempool *pool, void *end)
364 void *p = mp_ptr(pool);
365 pool->state.free[pool->idx] = (byte *)pool->state.last[pool->idx] - (byte *)end;
370 * Close the growing buffer as a string. That is, append a zero byte and call mp_end().
372 static inline char *mp_end_string(struct mempool *pool, void *end)
374 end = mp_append_char(pool, end, 0);
375 return mp_end(pool, end);
379 * Return size in bytes of the last allocated memory block (with @mp_alloc() or @mp_end()).
381 static inline uns mp_size(struct mempool *pool, void *ptr)
383 uns idx = mp_idx(pool, ptr);
384 return ((byte *)pool->state.last[idx] - (byte *)ptr) - pool->state.free[idx];
388 * Open the last memory block (allocated with @mp_alloc() or @mp_end())
389 * for growing and return its size in bytes. The contents and the start pointer
390 * remain unchanged. Do not forget to call @mp_end() to close it.
392 uns mp_open(struct mempool *pool, void *ptr);
395 * Inlined version of @mp_open().
397 static inline uns mp_open_fast(struct mempool *pool, void *ptr)
399 pool->idx = mp_idx(pool, ptr);
400 uns size = ((byte *)pool->state.last[pool->idx] - (byte *)ptr) - pool->state.free[pool->idx];
401 pool->state.free[pool->idx] += size;
406 * Reallocate the last memory block (allocated with @mp_alloc() or @mp_end())
407 * to the new @size. Behavior is similar to @mp_grow(), but the resulting
410 void *mp_realloc(struct mempool *pool, void *ptr, uns size);
413 * The same as @mp_realloc(), but fills the additional bytes (if any) with zeroes.
415 void *mp_realloc_zero(struct mempool *pool, void *ptr, uns size);
418 * Inlined version of @mp_realloc().
420 static inline void *mp_realloc_fast(struct mempool *pool, void *ptr, uns size)
422 mp_open_fast(pool, ptr);
423 ptr = mp_grow(pool, size);
424 mp_end(pool, (byte *)ptr + size);
430 * Storing and restoring state
431 * ---------------------------
433 * Mempools can remember history of what was allocated and return back
438 * Save the current state of a memory pool.
439 * Do not call this function with an opened growing buffer.
441 static inline void mp_save(struct mempool *pool, struct mempool_state *state)
443 *state = pool->state;
444 pool->state.next = state;
448 * Save the current state to a newly allocated mempool_state structure.
449 * Do not call this function with an opened growing buffer.
451 struct mempool_state *mp_push(struct mempool *pool);
454 * Restore the state saved by @mp_save() or @mp_push() and free all
455 * data allocated after that point (including the state structure itself).
456 * You can't reallocate the last memory block from the saved state.
458 void mp_restore(struct mempool *pool, struct mempool_state *state);
461 * Inlined version of @mp_restore().
463 static inline void mp_restore_fast(struct mempool *pool, struct mempool_state *state)
465 if (pool->state.last[0] != state->last[0] || pool->state.last[1] != state->last[1])
466 mp_restore(pool, state);
469 pool->state = *state;
470 pool->last_big = &pool->last_big;
475 * Restore the state saved by the last call to @mp_push().
476 * @mp_pop() and @mp_push() works as a stack so you can push more states safely.
478 void mp_pop(struct mempool *pool);
487 char *mp_strdup(struct mempool *, const char *) LIKE_MALLOC; /** Makes a copy of a string on a mempool. Returns NULL for NULL string. **/
488 void *mp_memdup(struct mempool *, const void *, uns) LIKE_MALLOC; /** Makes a copy of a memory block on a mempool. **/
490 * Concatenates all passed strings. The last parameter must be NULL.
491 * This will concatenate two strings:
493 * char *message = mp_multicat(pool, "hello ", "world", NULL);
495 char *mp_multicat(struct mempool *, ...) LIKE_MALLOC SENTINEL_CHECK;
497 * Concatenates two strings and stores result on @mp.
499 static inline char *LIKE_MALLOC mp_strcat(struct mempool *mp, const char *x, const char *y)
501 return mp_multicat(mp, x, y, NULL);
504 * Join strings and place @sep between each two neighboring.
505 * @p is the mempool to provide memory, @a is array of strings and @n
506 * tells how many there is of them.
508 char *mp_strjoin(struct mempool *p, char **a, uns n, uns sep) LIKE_MALLOC;
510 * Convert memory block to a string. Makes a copy of the given memory block
511 * in the mempool @p, adding an extra terminating zero byte at the end.
513 char *mp_str_from_mem(struct mempool *p, const void *mem, uns len) LIKE_MALLOC;
523 * printf() into a in-memory string, allocated on the memory pool.
525 char *mp_printf(struct mempool *mp, const char *fmt, ...) FORMAT_CHECK(printf,2,3) LIKE_MALLOC;
527 * Like @mp_printf(), but uses `va_list` for parameters.
529 char *mp_vprintf(struct mempool *mp, const char *fmt, va_list args) LIKE_MALLOC;
531 * Like @mp_printf(), but it appends the data at the end of string
532 * pointed to by @ptr. The string is @mp_open()ed, so you have to
533 * provide something that can be.
535 * Returns pointer to the beginning of the string (the pointer may have
536 * changed due to reallocation).
538 * Alternatively, this function may be called mp_printf_append() for compatibility with
539 * previous releases of LibUCW.
541 char *mp_append_printf(struct mempool *mp, char *ptr, const char *fmt, ...) FORMAT_CHECK(printf,3,4);
542 #define mp_printf_append mp_append_printf
544 * Like @mp_append_printf(), but uses `va_list` for parameters.
546 * Alternatively, this function may be called mp_vprintf_append() for compatibility with
547 * previous releases of LibUCW.
549 char *mp_append_vprintf(struct mempool *mp, char *ptr, const char *fmt, va_list args);
550 #define mp_vprintf_append mp_append_vprintf