2 * UCW Library -- Optimized Array Sorter
4 * (c) 2003--2007 Martin Mares <mj@ucw.cz>
6 * This software may be freely distributed and used according to the terms
7 * of the GNU Lesser General Public License.
13 #include "lib/sorter/common.h"
18 #define ASORT_MIN_SHIFT 2
20 #define ASORT_TRACE(x...) ASORT_XTRACE(1, x)
21 #define ASORT_XTRACE(level, x...) do { if (sorter_trace_array >= level) msg(L_DEBUG, x); } while(0)
24 asort_radix(struct asort_context *ctx, void *array, void *buffer, uns num_elts, uns hash_bits, uns swapped_output)
26 // swap_output == 0 if result should be returned in `array', otherwise in `buffer'
27 uns buckets = (1 << ctx->radix_bits);
28 uns shift = (hash_bits > ctx->radix_bits) ? (hash_bits - ctx->radix_bits) : 0;
32 static int reported[64];
33 if (!reported[hash_bits]++)
35 DBG(">>> n=%d h=%d s=%d sw=%d", num_elts, hash_bits, shift, swapped_output);
37 bzero(cnt, sizeof(cnt));
38 ctx->radix_count(array, num_elts, cnt, shift);
41 for (uns i=0; i<buckets; i++)
47 ASSERT(pos == num_elts);
49 ctx->radix_split(array, buffer, num_elts, cnt, shift);
51 for (uns i=0; i<buckets; i++)
54 if (n * ctx->elt_size < sorter_radix_threshold || shift < ASORT_MIN_SHIFT)
56 ctx->quicksort(buffer, n);
58 memcpy(array, buffer, n * ctx->elt_size);
61 asort_radix(ctx, buffer, array, n, shift, !swapped_output);
62 array += n * ctx->elt_size;
63 buffer += n * ctx->elt_size;
68 #ifdef CONFIG_UCW_THREADS
70 #include "lib/threads.h"
71 #include "lib/workqueue.h"
72 #include "lib/eltpool.h"
74 static uns asort_threads_use_count;
75 static uns asort_threads_ready;
76 static struct worker_pool asort_thread_pool;
79 rs_estimate_stack(void)
81 // Stack space needed by the recursive radix-sorter
82 uns ctrsize = sizeof(uns) * (1 << CONFIG_UCW_RADIX_SORTER_BITS);
83 uns maxdepth = (64 / CONFIG_UCW_RADIX_SORTER_BITS) + 1;
84 return ctrsize * maxdepth;
88 asort_start_threads(uns run)
91 asort_threads_use_count++;
92 if (run && !asort_threads_ready)
94 // XXX: If somebody overrides the radix-sorter parameters to insane values,
95 // he also should override the stack size to insane values.
96 asort_thread_pool.stack_size = default_thread_stack_size + rs_estimate_stack();
97 asort_thread_pool.num_threads = sorter_threads;
98 ASORT_TRACE("Initializing thread pool (%d threads, %dK stack)", sorter_threads, asort_thread_pool.stack_size >> 10);
99 worker_pool_init(&asort_thread_pool);
100 asort_threads_ready = 1;
106 asort_stop_threads(void)
109 if (!--asort_threads_use_count && asort_threads_ready)
111 ASORT_TRACE("Shutting down thread pool");
112 worker_pool_cleanup(&asort_thread_pool);
113 asort_threads_ready = 0;
120 struct asort_context *ctx;
124 #define LR_UNDEF -100
128 qs_handle_work(struct worker_thread *thr UNUSED, struct work *ww)
130 struct qs_work *w = (struct qs_work *) ww;
132 DBG("Thread %d: got %d elts", thr->id, w->num_elts);
133 if (w->num_elts * w->ctx->elt_size < sorter_thread_threshold)
135 w->ctx->quicksort(w->array, w->num_elts);
136 w->left = w->right = LR_UNDEF;
139 w->ctx->quicksplit(w->array, w->num_elts, &w->left, &w->right);
140 DBG("Thread %d: returning l=%d r=%d", thr->id, w->left, w->right);
143 static struct qs_work *
144 qs_alloc_work(struct asort_context *ctx)
146 struct qs_work *w = ep_alloc(ctx->eltpool);
148 w->w.go = qs_handle_work;
154 threaded_quicksort(struct asort_context *ctx)
157 struct qs_work *v, *w;
159 asort_start_threads(1);
160 work_queue_init(&asort_thread_pool, &q);
161 ctx->eltpool = ep_new(sizeof(struct qs_work), 1000);
163 w = qs_alloc_work(ctx);
164 w->array = ctx->array;
165 w->num_elts = ctx->num_elts;
166 work_submit(&q, &w->w);
168 while (v = (struct qs_work *) work_wait(&q))
170 if (v->left != LR_UNDEF)
174 w = qs_alloc_work(ctx);
176 w->num_elts = v->right + 1;
177 w->w.priority = v->w.priority + 1;
178 work_submit(&q, &w->w);
180 if (v->left < (int)v->num_elts - 1)
182 w = qs_alloc_work(ctx);
183 w->array = v->array + v->left * ctx->elt_size;
184 w->num_elts = v->num_elts - v->left;
185 w->w.priority = v->w.priority + 1;
186 work_submit(&q, &w->w);
189 ep_free(ctx->eltpool, v);
192 ep_delete(ctx->eltpool);
193 work_queue_cleanup(&q);
194 asort_stop_threads();
199 struct asort_context *ctx;
200 void *array, *buffer; // Like asort_radix().
208 rs_count(struct worker_thread *thr UNUSED, struct work *ww)
210 struct rs_work *w = (struct rs_work *) ww;
212 DBG("Thread %d: Counting %d items, shift=%d", thr->id, w->num_elts, w->shift);
213 w->ctx->radix_count(w->array, w->num_elts, w->cnt, w->shift);
214 DBG("Thread %d: Counting done", thr->id);
218 rs_split(struct worker_thread *thr UNUSED, struct work *ww)
220 struct rs_work *w = (struct rs_work *) ww;
222 DBG("Thread %d: Splitting %d items, shift=%d", thr->id, w->num_elts, w->shift);
223 w->ctx->radix_split(w->array, w->buffer, w->num_elts, w->cnt, w->shift);
224 DBG("Thread %d: Splitting done", thr->id);
228 rs_finish(struct worker_thread *thr UNUSED, struct work *ww)
230 struct rs_work *w = (struct rs_work *) ww;
233 DBG("Thread %d: Finishing %d items, shift=%d", thr->id, w->num_elts, w->shift);
234 if (w->shift < ASORT_MIN_SHIFT || w->num_elts * w->ctx->elt_size < sorter_radix_threshold)
236 w->ctx->quicksort(w->array, w->num_elts);
238 memcpy(w->buffer, w->array, w->num_elts * w->ctx->elt_size);
241 asort_radix(w->ctx, w->array, w->buffer, w->num_elts, w->shift, w->swap_output);
243 DBG("Thread %d: Finishing done", thr->id);
247 rs_wait_small(struct asort_context *ctx)
251 while (w = (struct rs_work *) work_wait(ctx->rs_work_queue))
253 DBG("Reaping small chunk of %d items", w->num_elts);
254 ep_free(ctx->eltpool, w);
259 rs_radix(struct asort_context *ctx, void *array, void *buffer, uns num_elts, uns hash_bits, uns swapped_output)
261 uns buckets = (1 << ctx->radix_bits);
262 uns shift = (hash_bits > ctx->radix_bits) ? (hash_bits - ctx->radix_bits) : 0;
264 uns blksize = num_elts / sorter_threads;
265 DBG(">>> n=%d h=%d s=%d blk=%d sw=%d", num_elts, hash_bits, shift, blksize, swapped_output);
267 // If there are any small chunks in progress, wait for them to finish
270 // Start parallel counting
272 for (uns i=0; i<sorter_threads; i++)
274 struct rs_work *w = ctx->rs_works[i];
280 w->num_elts = blksize;
281 if (i == sorter_threads-1)
282 w->num_elts += num_elts % sorter_threads;
284 iptr += w->num_elts * ctx->elt_size;
285 bzero(w->cnt, sizeof(uns) * buckets);
286 work_submit(ctx->rs_work_queue, &w->w);
289 // Get bucket sizes from the counts
290 bzero(cnt, sizeof(cnt));
291 for (uns i=0; i<sorter_threads; i++)
293 struct rs_work *w = (struct rs_work *) work_wait(ctx->rs_work_queue);
295 for (uns j=0; j<buckets; j++)
299 // Calculate bucket starts
301 for (uns i=0; i<buckets; i++)
307 ASSERT(pos == num_elts);
309 // Start parallel splitting
310 for (uns i=0; i<sorter_threads; i++)
312 struct rs_work *w = ctx->rs_works[i];
314 for (uns j=0; j<buckets; j++)
320 work_submit(ctx->rs_work_queue, &w->w);
322 ASSERT(cnt[buckets-1] == num_elts);
324 // Wait for splits to finish
325 while (work_wait(ctx->rs_work_queue))
328 // Recurse on buckets
330 for (uns i=0; i<buckets; i++)
332 uns n = cnt[i] - pos;
335 if (n * ctx->elt_size < sorter_thread_threshold)
337 struct rs_work *w = ep_alloc(ctx->eltpool);
345 w->swap_output = !swapped_output;
346 if (n * ctx->elt_size < sorter_thread_chunk)
348 DBG("Sorting block %d+%d inline", pos, n);
349 rs_finish(NULL, &w->w);
350 ep_free(ctx->eltpool, w);
354 DBG("Scheduling block %d+%d", pos, n);
355 work_submit(ctx->rs_work_queue, &w->w);
359 rs_radix(ctx, buffer, array, n, shift, !swapped_output);
361 array += n * ctx->elt_size;
362 buffer += n * ctx->elt_size;
367 threaded_radixsort(struct asort_context *ctx, uns swap)
371 asort_start_threads(1);
372 work_queue_init(&asort_thread_pool, &q);
374 // Prepare work structures for counting and splitting.
375 // We use big_alloc(), because we want to avoid cacheline aliasing between threads.
376 ctx->rs_work_queue = &q;
377 ctx->rs_works = alloca(sizeof(struct rs_work *) * sorter_threads);
378 for (uns i=0; i<sorter_threads; i++)
379 ctx->rs_works[i] = big_alloc(sizeof(struct rs_work) + sizeof(uns) * (1 << ctx->radix_bits));
381 // Prepare a pool for all remaining small bits which will be sorted on background.
382 ctx->eltpool = ep_new(sizeof(struct rs_work), 1000);
384 // Do the big splitting
385 rs_radix(ctx, ctx->array, ctx->buffer, ctx->num_elts, ctx->hash_bits, swap);
386 for (uns i=0; i<sorter_threads; i++)
387 big_free(ctx->rs_works[i], sizeof(struct rs_work) + sizeof(uns) * (1 << ctx->radix_bits));
389 // Finish the small blocks
392 ASSERT(!ctx->eltpool->num_allocated);
393 ep_delete(ctx->eltpool);
394 work_queue_cleanup(&q);
395 asort_stop_threads();
400 void asort_start_threads(uns run UNUSED) { }
401 void asort_stop_threads(void) { }
406 predict_swap(struct asort_context *ctx)
408 uns bits = ctx->radix_bits;
409 uns elts = ctx->num_elts;
412 while (elts * ctx->elt_size >= sorter_radix_threshold && bits >= ASORT_MIN_SHIFT)
414 DBG("Predicting pass: %d elts, %d bits", elts, bits);
416 elts >>= ctx->radix_bits;
417 bits = MAX(bits, ctx->radix_bits) - ctx->radix_bits;
423 asort_run(struct asort_context *ctx)
425 ASORT_TRACE("Array-sorting %d items per %d bytes, hash_bits=%d", ctx->num_elts, ctx->elt_size, ctx->hash_bits);
426 uns allow_threads UNUSED = (sorter_threads > 1 &&
427 ctx->num_elts * ctx->elt_size >= sorter_thread_threshold &&
428 !(sorter_debug & SORT_DEBUG_ASORT_NO_THREADS));
430 if (ctx->num_elts * ctx->elt_size < sorter_radix_threshold ||
431 ctx->hash_bits <= ASORT_MIN_SHIFT ||
433 (sorter_debug & SORT_DEBUG_ASORT_NO_RADIX))
435 #ifdef CONFIG_UCW_THREADS
438 ASORT_XTRACE(2, "Decided to use parallel quicksort");
439 threaded_quicksort(ctx);
443 ASORT_XTRACE(2, "Decided to use sequential quicksort");
444 ctx->quicksort(ctx->array, ctx->num_elts);
448 uns swap = predict_swap(ctx);
449 #ifdef CONFIG_UCW_THREADS
452 ASORT_XTRACE(2, "Decided to use parallel radix-sort (swap=%d)", swap);
453 threaded_radixsort(ctx, swap);
457 ASORT_XTRACE(2, "Decided to use sequential radix-sort (swap=%d)", swap);
458 asort_radix(ctx, ctx->array, ctx->buffer, ctx->num_elts, ctx->hash_bits, swap);
460 ctx->array = ctx->buffer;
463 ASORT_XTRACE(2, "Array-sort finished");