]> mj.ucw.cz Git - xsv.git/blob - xsv.c
Avoid unnecessary initialization of buffers
[xsv.git] / xsv.c
1 /*
2  *      A Swiss-Army Knife for CSV-like Files
3  *
4  *      (c) 2012 Martin Mares <mj@ucw.cz>
5  */
6
7 #define _GNU_SOURCE
8
9 #include <stdio.h>
10 #include <stdlib.h>
11 #include <string.h>
12 #include <stdarg.h>
13 #include <getopt.h>
14 #include <wchar.h>
15 #include <locale.h>
16
17 #include <pcre.h>
18
19 #ifdef __GNUC__
20 #define NONRET __attribute__((noreturn))
21 #else
22 #define NONRET
23 #endif
24
25 /*** General functions ***/
26
27 static void NONRET die(char *msg, ...)
28 {
29         va_list args;
30         va_start(args, msg);
31         fprintf(stderr, "xsv: ");
32         vfprintf(stderr, msg, args);
33         fputc('\n', stderr);
34         va_end(args);
35         exit(1);
36 }
37
38 /*** Memory allocation ***/
39
40 static void *xmalloc(size_t bytes)
41 {
42         void *p = malloc(bytes);
43         if (!p)
44                 die("Out of memory (cannot allocate %zu bytes)", bytes);
45         return p;
46 }
47
48 static void *xmalloc_zero(size_t bytes)
49 {
50         void *p = xmalloc(bytes);
51         memset(p, 0, bytes);
52         return p;
53 }
54
55 static void *xrealloc(void *old, size_t bytes)
56 {
57         void *p = realloc(old, bytes);
58         if (!p)
59                 die("Out of memory (cannot allocate %zu bytes)", bytes);
60         return p;
61 }
62
63 #define DECLARE_BUF(name, type) \
64         typedef struct { type *start; int count; int max; } name##_t;                           \
65         static inline void name##_init(name##_t *b) { b->start = NULL; b->count = b->max = 0; } \
66         static inline void name##_reset(name##_t *b) { b->count = 0; }                          \
67         static inline int name##_count(name##_t *b) { return b->count; }                        \
68         static void name##_extend(name##_t *b) {                                                \
69                 b->max = b->max ? 2*b->max : 16;                                                \
70                 b->start = xrealloc(b->start, b->max * sizeof(type));                           \
71         }                                                                                       \
72         static inline type *name##_push(name##_t *b) {                                          \
73                 if (b->count >= b->max) name##_extend(b);                                       \
74                 return &b->start[b->count++];                                                   \
75         }                                                                                       \
76         static inline type *name##_first(name##_t *b) { return b->start; }                      \
77         static inline type *name##_nth(name##_t *b, int n) { return &b->start[n]; }             \
78         // end
79
80 DECLARE_BUF(intarray, int);
81
82 /*** Formats and their parameters ***/
83
84 enum format_id {
85         FORM_UNSPEC,
86         FORM_TSV,
87         FORM_CSV,
88         FORM_WS,
89         FORM_REGEX,
90         FORM_TMP,
91         FORM_TABLE,
92 };
93
94 struct format {
95         enum format_id id;
96         int fs;
97         int quote;
98         int quiet;
99         int (*read_line)(void);
100         void (*write_line)(void);
101         int needs_two_passes;
102
103         // CSV backend:
104         int always_quote;
105
106         // WS backend:
107         int strict_ws;
108
109         // regex backend:
110         pcre *pcre;
111         pcre_extra *pcre_extra;
112
113         // Temporary file backend:
114         FILE *tmp_file;
115         intarray_t column_widths;
116
117         // Table backend:
118         int table_sep;
119 };
120
121 static struct format *in_format, *out_format;
122 static int want_trim;
123
124 struct field {
125         int start_pos;
126         int len;
127 };
128
129 DECLARE_BUF(fields, struct field);
130 DECLARE_BUF(line, unsigned char);
131
132 static fields_t in_fields, out_fields;
133 static struct field *in_field;
134 static line_t in_line;
135 static int line_number;
136
137 static void new_field(int pos)
138 {
139         in_field = fields_push(&in_fields);
140         in_field->start_pos = pos;
141         in_field->len = 0;
142 }
143
144 static void ensure_field(int pos)
145 {
146         if (!in_field)
147                 new_field(pos);
148 }
149
150 static void warn(struct format *fmt, char *msg, ...)
151 {
152         if (!fmt->quiet) {
153                 fprintf(stderr, "Warning at line %d: ", line_number);
154                 va_list args;
155                 va_start(args, msg);
156                 vfprintf(stderr, msg, args);
157                 va_end(args);
158                 fputc('\n', stderr);
159         }
160 }
161
162 static int next_line(void)
163 {
164         for (;;) {
165                 int c = getchar_unlocked();
166                 if (c == '\r')
167                         continue;
168                 if (c < 0)
169                         return !!line_count(&in_line);
170                 if (c == '\n')
171                         return 1;
172                 *line_push(&in_line) = c;
173         }
174 }
175
176 static int field_chars(struct field *f)
177 {
178         unsigned char *s = line_nth(&in_line, f->start_pos);
179         int i = 0;
180         mbstate_t mbs;
181         memset(&mbs, 0, sizeof(mbs));
182
183         int chars = 0;
184         while (i < f->len) {
185                 size_t k = mbrlen((char *) s + i, f->len - i, &mbs);
186                 if ((int) k <= 0)
187                         break;
188                 i += k;
189                 chars++;
190         }
191
192         return chars;
193 }
194
195 /*** CSV/TSV back-end */
196
197 static int csv_read(void)
198 {
199         int quoted = 0;
200         for (;;) {
201                 int c = getchar_unlocked();
202                 int i = line_count(&in_line);
203 restart:
204                 if (c == '\r')
205                         continue;
206                 if (c < 0 || c == '\n') {
207                         if (quoted)
208                                 warn(in_format, "Missing closing quote.");
209                         if (c < 0)
210                                 return !!fields_count(&in_fields);
211                         else
212                                 return 1;
213                 }
214                 if (quoted) {
215                         if (c == in_format->quote) {
216                                 c = getchar_unlocked();
217                                 if (c != in_format->quote) {
218                                         quoted = 0;
219                                         goto restart;
220                                 }
221                                 // Two quotes assimilate to one
222                         }
223                         // Fall through to pushing the character
224                 } else if (c == in_format->quote) {
225                         quoted = 1;
226                         continue;
227                 } else if (c == in_format->fs && !quoted) {
228                         ensure_field(i);
229                         new_field(i);
230                         continue;
231                 }
232                 ensure_field(i);
233                 *line_push(&in_line) = c;
234                 in_field->len++;
235         }
236 }
237
238 static int is_ws(int c)
239 {
240         return (c == ' ' || c == '\t' || c == '\f');
241 }
242
243 static void csv_write(void)
244 {
245         unsigned char *line = line_first(&in_line);
246         int n = fields_count(&out_fields);
247         for (int i=0; i<n; i++) {
248                 struct field *f = fields_nth(&out_fields, i);
249                 int need_quotes = 0;
250                 if (out_format->quote >= 0) {
251                         need_quotes = out_format->always_quote;
252                         for (int j=0; !need_quotes && j < f->len; j++) {
253                                 int c = line[f->start_pos + j];
254                                 if (c == out_format->fs || c == out_format->quote)
255                                         need_quotes = 1;
256                         }
257                 }
258                 if (i)
259                         putchar_unlocked(out_format->fs);
260                 if (need_quotes)
261                         putchar_unlocked(out_format->quote);
262                 for (int j=0; j < f->len; j++) {
263                         int c = line[f->start_pos + j];
264                         if (c == out_format->fs && !need_quotes)
265                                 warn(out_format, "Field separator found inside field and quoting is turned off.");
266                         if (c == out_format->quote)
267                                 putchar_unlocked(c);
268                         putchar_unlocked(c);
269                 }
270                 if (need_quotes)
271                         putchar_unlocked(out_format->quote);
272         }
273         putchar_unlocked('\n');
274 }
275
276 /*** White-space back-end ***/
277
278 static int ws_read(void)
279 {
280         if (!next_line())
281                 return 0;
282
283         unsigned char *line = line_first(&in_line);
284         int n = line_count(&in_line);
285         if (!n)
286                 return 1;
287
288         int ws = 0;
289         new_field(0);
290         for (int i=0; i<n; i++) {
291                 int c = line[i];
292                 if (is_ws(c)) {
293                         ws++;
294                 } else {
295                         if (ws) {
296                                 if (!in_field->start_pos &&
297                                     !in_field->len &&
298                                     !in_format->strict_ws)
299                                         in_field->start_pos = i;
300                                 else
301                                         new_field(i);
302                                 ws = 0;
303                         }
304                         in_field->len++;
305                 }
306         }
307
308         if (ws && in_format->strict_ws)
309                 new_field(n);
310         return 1;
311 }
312
313 /*** Regex back-end ***/
314
315 static const char *regex_set(struct format *f, char *rx)
316 {
317         const char *err;
318         int errpos;
319         f->pcre = pcre_compile(rx, PCRE_DOLLAR_ENDONLY, &err, &errpos, NULL);
320         if (!f->pcre)
321                 return err;
322
323         f->pcre_extra = pcre_study(f->pcre, 0, &err);
324         if (!f->pcre_extra)
325                 return err;
326
327         return NULL;
328 }
329
330 static int regex_read(void)
331 {
332         if (!next_line())
333                 return 0;
334
335         unsigned char *c = line_first(&in_line);
336         int n = line_count(&in_line);
337         if (!n)
338                 return 1;
339
340         int i = 0;
341         for (;;) {
342                 int ovec[3];
343                 int sep = pcre_exec(in_format->pcre, in_format->pcre_extra, (char *) c, n, i, 0, ovec, 3);
344                 if (sep < 0) {
345                         if (sep != PCRE_ERROR_NOMATCH)
346                                 warn(in_format, "PCRE matching error %d", sep);
347                         // No further occurrence of the separator: the rest is a single field
348                         new_field(i);
349                         in_field->len = n - i;
350                         return 1;
351                 }
352                 new_field(i);
353                 in_field->len = ovec[0] - i;
354                 i = ovec[1];
355         }
356 }
357
358 /*** Table back-end ***/
359
360 static void table_write(void)
361 {
362         for (int i = 0; i < fields_count(&in_fields); i++) {
363                 if (i)
364                         printf("%*s", out_format->table_sep, "");
365                 struct field *f = fields_nth(&in_fields, i);
366                 int fw = field_chars(f);
367                 int cw = *intarray_nth(&in_format->column_widths, i);
368                 if (fw > cw) {
369                         warn(out_format, "Internal error: Wrongly calculated column width (%d > %d)", fw, cw);
370                         cw = fw;
371                 }
372                 unsigned char *p = line_nth(&in_line, f->start_pos);
373                 for (int j = 0; j < f->len; j++)
374                         putchar_unlocked(p[j]);
375                 while (fw < cw) {
376                         putchar_unlocked(' ');
377                         fw++;
378                 }
379         }
380         putchar_unlocked('\n');
381 }
382
383 /*** Temporary file back-end ***/
384
385 static int tmp_read(void)
386 {
387         FILE *tf = in_format->tmp_file;
388
389         for (;;) {
390                 int c = getc_unlocked(tf);
391                 if (c < 0)
392                         return 0;
393                 if (c == 0xff)
394                         return 1;
395                 if (c == 0xfe) {
396                         c = getc_unlocked(tf);
397                         c = (c << 8) | getc_unlocked(tf);
398                         c = (c << 8) | getc_unlocked(tf);
399                         c = (c << 8) | getc_unlocked(tf);
400                 }
401                 new_field(line_count(&in_line));
402                 in_field->len = c;
403                 while (c--) {
404                         int x = getc_unlocked(tf);
405                         if (x < 0) {
406                                 warn(in_format, "Truncated temporary file");
407                                 return 0;
408                         }
409                         *line_push(&in_line) = x;
410                 }
411         }
412
413         if (ferror_unlocked(tf))
414                 die("I/O error when reading temporary file");
415 }
416
417 static void tmp_write(void)
418 {
419         FILE *tf = out_format->tmp_file;
420
421         for (int i = 0; i < fields_count(&in_fields); i++) {
422                 struct field *f = fields_nth(&in_fields, i);
423                 if (f->len < 0xfe)
424                         putc_unlocked(f->len, tf);
425                 else {
426                         putc_unlocked(0xfe, tf);
427                         putc_unlocked((f->len >> 24) & 0xff, tf);
428                         putc_unlocked((f->len >> 16) & 0xff, tf);
429                         putc_unlocked((f->len >> 8) & 0xff, tf);
430                         putc_unlocked(f->len & 0xff, tf);
431                 }
432
433                 unsigned char *p = line_nth(&in_line, f->start_pos);
434                 for (int j = 0; j < f->len; j++)
435                         putc_unlocked(*p++, tf);
436
437                 intarray_t *w = &out_format->column_widths;
438                 while (i >= intarray_count(w))
439                         *intarray_push(w) = 0;
440                 int fw = field_chars(f);
441                 if (*intarray_nth(w, i) < fw)
442                         *intarray_nth(w, i) = fw;
443         }
444         putc_unlocked(0xff, tf);
445
446         if (ferror_unlocked(tf))
447                 die("I/O error when writing temporary file");
448 }
449
450 /*** Transforms ***/
451
452 static void trim_fields(void)
453 {
454         unsigned char *line = line_first(&in_line);
455         for (int i = 0; i < fields_count(&in_fields); i++) {
456                 struct field *f = fields_nth(&in_fields, i);
457                 while (f->len && is_ws(line[f->start_pos]))
458                         f->start_pos++, f->len--;
459                 while (f->len && is_ws(line[f->start_pos + f->len - 1]))
460                         f->len--;
461         }
462 }
463
464 /*** Field selection ***/
465
466 struct selector {
467         int first_field, last_field;
468 };
469
470 DECLARE_BUF(selectors, struct selector);
471 static selectors_t selectors;
472
473 static char *parse_selector(char *str)
474 {
475         char buf[strlen(str) + 1];
476         strcpy(buf, str);
477
478         struct selector *s = selectors_push(&selectors);
479         char *sep = strchr(buf, '-');
480         if (sep) {
481                 *sep++ = 0;
482                 s->first_field = atoi(buf);
483                 s->last_field = atoi(sep);
484         } else
485                 s->first_field = s->last_field = atoi(buf);
486
487         return NULL;
488 }
489
490 static void finish_parse_selectors(void)
491 {
492         if (!selectors_count(&selectors))
493                 parse_selector("-");
494 }
495
496 static void select_fields(void)
497 {
498         for (int i = 0; i < selectors_count(&selectors); i++) {
499                 struct selector *s = selectors_nth(&selectors, i);
500                 int first = s->first_field;
501                 if (first <= 0)
502                         first = 1;
503                 int last = s->last_field;
504                 if (last <= 0)
505                         last = fields_count(&in_fields);
506                 for (int j = first; j <= last; j++) {
507                         struct field *f = fields_push(&out_fields);
508                         if (j >= 1 && j <= fields_count(&in_fields))
509                                 *f = *fields_nth(&in_fields, j-1);
510                         else
511                                 f->start_pos = f->len = 0;
512                 }
513         }
514 }
515
516 /*** Processing of files ***/
517
518 static void one_pass(void)
519 {
520         line_number = 0;
521         for (;;) {
522                 line_number++;
523                 fields_reset(&in_fields);
524                 line_reset(&in_line);
525                 in_field = NULL;
526                 if (!in_format->read_line())
527                         break;
528                 if (ferror_unlocked(stdin))
529                         die("I/O error when reading standard input");
530
531                 if (want_trim)
532                         trim_fields();
533
534                 fields_reset(&out_fields);
535                 select_fields();
536
537                 out_format->write_line();
538                 if (ferror_unlocked(stdout))
539                         die("I/O error when writing standard input");
540         }
541 }
542
543 static void two_pass(void)
544 {
545         struct format *final_format = out_format;
546
547         // We need to use character set info from the current locale
548         setlocale(LC_CTYPE, "");
549
550         // Pass 1: Set up writer of intermediate format
551         out_format = xmalloc_zero(sizeof(*out_format));
552         out_format->id = FORM_TMP;
553         out_format->read_line = tmp_read;
554         out_format->write_line = tmp_write;
555         out_format->tmp_file = tmpfile();
556         one_pass();
557
558         // Pass 2: Set up reader of intermediate format
559         in_format = out_format;
560         rewind(in_format->tmp_file);
561         out_format = final_format;
562         one_pass();
563         fclose(in_format->tmp_file);
564 }
565
566 /*** Parsing of arguments ***/
567
568 static void NONRET usage(void)
569 {
570         printf("\
571 Usage: xsv <in-format> [<out-format>] <options> [<fields>]\n\
572 \n\
573 Formats:\n\
574 -t, --tsv               TAB-separated values (default)\n\
575 -c, --csv               Comma-separated values\n\
576 -w, --ws                Values separated by arbitrary whitespace\n\
577 -W, --strict-ws         Like --ws, but recognize empty columns at start/end\n\
578 -r, --regex=<rx>        Separator given by Perl regular expression (input only)\n\
579     --table             Format a table (output only)\n\
580 \n\
581 Format parameters:\n\
582 -d, --fs=<char>         Delimiter of fields\n\
583 -q, --quiet             Do not show warnings\n\
584     --always-quote      Put quotes around all fields (CSV output only)\n\
585     --table-sep=<n>     Separate table columns by <n> spaces (default: 2)\n\
586 \n\
587 Other options:\n\
588     --trim              Trim leading and trailing whitespaces in fields\n\
589 ");
590         exit(0);
591 }
592
593 static void NONRET bad_args(const char *msg, ...)
594 {
595         if (msg) {
596                 va_list args;
597                 va_start(args, msg);
598                 fprintf(stderr, "xsv: ");
599                 vfprintf(stderr, msg, args);
600                 fputc('\n', stderr);
601                 va_end(args);
602         }
603         fprintf(stderr, "Try `xsv --help' for more information.\n");
604         exit(1);
605 }
606
607 static const char short_options[] = "cd:qr:twW";
608
609 enum long_options {
610         OPT_HELP = 256,
611         OPT_TRIM,
612         OPT_ALWAYS_QUOTE,
613         OPT_TABLE,
614         OPT_TABLE_SEP,
615 };
616
617 static const struct option long_options[] = {
618         { "always-quote",       0,      NULL,   OPT_ALWAYS_QUOTE },
619         { "csv",                0,      NULL,   'c' },
620         { "fs",                 1,      NULL,   'd' },
621         { "quiet",              0,      NULL,   'q' },
622         { "regex",              1,      NULL,   'r' },
623         { "strict-ws",          0,      NULL,   'W' },
624         { "table",              0,      NULL,   OPT_TABLE },
625         { "table-sep",          1,      NULL,   OPT_TABLE_SEP },
626         { "trim",               0,      NULL,   OPT_TRIM },
627         { "tsv",                0,      NULL,   't' },
628         { "ws",                 0,      NULL,   'w' },
629         { "help",               0,      NULL,   OPT_HELP },
630         { NULL,                 0,      NULL,   0 },
631 };
632
633 static void set_format(int format_id)
634 {
635         struct format *f = xmalloc_zero(sizeof(*f));
636         f->id = format_id;
637
638         switch (format_id) {
639                 case FORM_TSV:
640                         f->fs = '\t';
641                         f->quote = -1;
642                         f->read_line = csv_read;
643                         f->write_line = csv_write;
644                         break;
645                 case FORM_CSV:
646                         f->fs = ',';
647                         f->quote = '"';
648                         f->read_line = csv_read;
649                         f->write_line = csv_write;
650                         break;
651                 case FORM_WS:
652                         f->fs = ' ';
653                         f->quote = -1;
654                         f->read_line = ws_read;
655                         f->write_line = csv_write;
656                         break;
657                 case FORM_REGEX:
658                         f->read_line = regex_read;
659                         break;
660                 case FORM_TABLE:
661                         f->write_line = table_write;
662                         f->needs_two_passes = 1;
663                         f->table_sep = 2;
664                         break;
665         }
666
667         if (!in_format)
668                 in_format = f;
669         else if (!out_format)
670                 out_format = f;
671         else
672                 bad_args("At most two formats may be given.");
673 }
674
675 static struct format *current_format(void)
676 {
677         if (out_format)
678                 return out_format;
679         if (in_format)
680                 return in_format;
681         set_format(FORM_TSV);
682         return in_format;
683 }
684
685 int main(int argc, char **argv)
686 {
687         int opt;
688         const char *err;
689
690         while ((opt = getopt_long(argc, argv, short_options, long_options, NULL)) >= 0)
691                 switch (opt) {
692                         case 'c':
693                                 set_format(FORM_CSV);
694                                 break;
695                         case 'd':
696                                 if (optarg[0])
697                                         current_format()->fs = optarg[0];
698                                 else
699                                         bad_args("No field delimiter given.");
700                                 break;
701                         case 'q':
702                                 current_format()->quiet = 1;
703                                 break;
704                         case 'r':
705                                 set_format(FORM_REGEX);
706                                 err = regex_set(current_format(), optarg);
707                                 if (err)
708                                         bad_args("Error compiling regex: %s", err);
709                                 break;
710                         case 't':
711                                 set_format(FORM_TSV);
712                                 break;
713                         case 'w':
714                                 set_format(FORM_WS);
715                                 break;
716                         case 'W':
717                                 set_format(FORM_WS);
718                                 current_format()->strict_ws = 1;
719                                 break;
720                         case OPT_ALWAYS_QUOTE:
721                                 if (current_format()->id != FORM_CSV)
722                                         bad_args("--always-quote makes sense only for CSV.");
723                                 current_format()->always_quote = 1;
724                                 break;
725                         case OPT_HELP:
726                                 usage();
727                         case OPT_TRIM:
728                                 want_trim = 1;
729                                 break;
730                         case OPT_TABLE:
731                                 set_format(FORM_TABLE);
732                                 break;
733                         case OPT_TABLE_SEP:
734                                 current_format()->table_sep = atoi(optarg);
735                                 break;
736                         default:
737                                 bad_args(NULL);
738                 }
739
740         current_format();
741         if (!out_format)
742                 out_format = in_format;
743         if (!in_format->read_line)
744                 bad_args("Write-only format selected for input.");
745         if (!out_format->write_line)
746                 bad_args("Read-only format selected for output.");
747
748         for (int i = optind; i < argc; i++) {
749                 err = parse_selector(argv[i]);
750                 if (err)
751                         bad_args(err);
752         }
753         finish_parse_selectors();
754
755         if (out_format->needs_two_passes)
756                 two_pass();
757         else
758                 one_pass();
759         return 0;
760 }