1 /* Two-level parallel SHA1 */
2 /* find -type f -print0 | qhash */
7 #include <ucw/fastbuf.h>
9 #include <ucw/string.h>
10 #include <ucw/workqueue.h>
16 #define BLOCK_SIZE 4096
19 #define PIECES (2*THREADS)
20 #define BLOCKS_PER_PIECE 2
24 static int threads_inited;
25 static struct worker_pool thread_pool = {
26 .num_threads = THREADS,
28 static struct work_queue work_queue;
33 byte out[BLOCKS_PER_PIECE][SHA1_SIZE];
38 static struct hash_work works[PIECES];
40 static void do_work(struct worker_thread *t UNUSED, struct work *w)
42 struct hash_work *hw = (struct hash_work *) w;
45 byte *in = hw->buffer;
46 uint remains = hw->len;
49 uint len = MIN(remains, BLOCK_SIZE);
50 sha1_hash_buffer(hw->out[i], in, len);
58 static void init_threads(void)
63 worker_pool_init(&thread_pool);
64 work_queue_init(&thread_pool, &work_queue);
66 for (int i=0; i<PIECES; i++)
68 works[i].w.go = do_work;
69 works[i].buffer = big_alloc(BLOCK_SIZE * BLOCKS_PER_PIECE);
74 static void hash_file(char *name, char *buf)
76 int fd = open(name, O_RDONLY);
79 fprintf(stderr, "Cannot open %s: %m\n", name);
84 sha1_context main_ctx;
88 uint current_piece = 0;
89 uint running_pieces = 0;
94 while (!eof || running_pieces)
96 if (!eof && running_pieces < PIECES)
98 struct hash_work *hw = &works[(current_piece + running_pieces) % PIECES];
99 int r = read(fd, hw->buffer, BLOCK_SIZE * BLOCKS_PER_PIECE);
102 fprintf(stderr, "Error reading %s: %m\n", name);
112 work_submit(&work_queue, &hw->w);
118 struct hash_work *hw = (struct hash_work *) work_wait(&work_queue);
120 while (running_pieces && !works[current_piece].running)
122 hw = &works[current_piece];
123 for (uint i=0; i < hw->out_len; i++)
126 char xx[SHA1_HEX_SIZE];
127 mem_to_hex(xx, hw->out[i], SHA1_SIZE, 0);
130 sha1_update(&main_ctx, hw->out[i], SHA1_SIZE);
132 current_piece = (current_piece + 1) % PIECES;
146 byte *h = sha1_final(&main_ctx);
147 buf += sprintf(buf, "%jd\t", total);
148 mem_to_hex(buf, h, SHA1_SIZE, 0);
153 static void hash_file(char *name, char *buf)
155 int fd = open(name, O_RDONLY);
158 fprintf(stderr, "Cannot open %s: %m\n", name);
163 static byte *hash_buffer;
165 hash_buffer = big_alloc(BLOCK_SIZE);
167 sha1_context block_ctx, main_ctx;
168 sha1_init(&main_ctx);
172 while ((len = read(fd, hash_buffer, BLOCK_SIZE)) > 0)
174 sha1_init(&block_ctx);
175 sha1_update(&block_ctx, hash_buffer, len);
176 byte *h = sha1_final(&block_ctx);
177 sha1_update(&main_ctx, h, SHA1_SIZE);
182 fprintf(stderr, "Error reading %s: %m\n", name);
190 byte *h = sha1_final(&main_ctx);
191 buf += sprintf(buf, "%jd\t", total);
192 mem_to_hex(buf, h, SHA1_SIZE, 0);
199 struct fastbuf *in = bopen_fd(0, NULL);
200 struct fastbuf *out = bopen_fd(1, NULL);
203 while (bgets0(in, line, sizeof(line)))
206 hash_file(line, hash);
207 bprintf(out, "%s\t%s\n", line, hash);