2 * A Swiss-Army Knife for CSV-like Files
4 * (c) 2012 Martin Mares <mj@ucw.cz>
20 #define NONRET __attribute__((noreturn))
25 /*** General functions ***/
27 static void NONRET die(char *msg, ...)
31 fprintf(stderr, "xsv: ");
32 vfprintf(stderr, msg, args);
38 /*** Memory allocation ***/
40 static void *xmalloc(size_t bytes)
42 void *p = malloc(bytes);
44 die("Out of memory (cannot allocate %zu bytes)", bytes);
48 static void *xmalloc_zero(size_t bytes)
50 void *p = xmalloc(bytes);
55 static void *xrealloc(void *old, size_t bytes)
57 void *p = realloc(old, bytes);
59 die("Out of memory (cannot allocate %zu bytes)", bytes);
63 #define DECLARE_BUF(name, type) \
64 typedef struct { type *start; int count; int max; } name##_t; \
65 static inline void name##_init(name##_t *b) { b->start = NULL; b->count = b->max = 0; } \
66 static inline void name##_reset(name##_t *b) { b->count = 0; } \
67 static inline int name##_count(name##_t *b) { return b->count; } \
68 static void name##_extend(name##_t *b) { \
69 b->max = b->max ? 2*b->max : 16; \
70 b->start = xrealloc(b->start, b->max * sizeof(type)); \
72 static inline type *name##_push(name##_t *b) { \
73 if (b->count >= b->max) name##_extend(b); \
74 return &b->start[b->count++]; \
76 static inline type *name##_first(name##_t *b) { return b->start; } \
77 static inline type *name##_nth(name##_t *b, int n) { return &b->start[n]; } \
80 DECLARE_BUF(intarray, int);
82 /*** Formats and their parameters ***/
99 int (*read_line)(void);
100 void (*write_line)(void);
101 int needs_two_passes;
111 pcre_extra *pcre_extra;
113 // Temporary file backend:
115 intarray_t column_widths;
121 static struct format *in_format, *out_format;
122 static int want_trim;
129 DECLARE_BUF(fields, struct field);
130 DECLARE_BUF(line, unsigned char);
132 static fields_t in_fields, out_fields;
133 static struct field *in_field;
134 static line_t in_line;
135 static int line_number;
137 static void new_field(int pos)
139 in_field = fields_push(&in_fields);
140 in_field->start_pos = pos;
144 static void ensure_field(int pos)
150 static void warn(struct format *fmt, char *msg, ...)
153 fprintf(stderr, "Warning at line %d: ", line_number);
156 vfprintf(stderr, msg, args);
162 static int next_line(void)
165 int c = getchar_unlocked();
169 return !!line_count(&in_line);
172 *line_push(&in_line) = c;
176 static int field_chars(struct field *f)
178 unsigned char *s = line_nth(&in_line, f->start_pos);
181 memset(&mbs, 0, sizeof(mbs));
185 size_t k = mbrlen((char *) s + i, f->len - i, &mbs);
195 /*** CSV/TSV back-end */
197 static int csv_read(void)
201 int c = getchar_unlocked();
202 int i = line_count(&in_line);
206 if (c < 0 || c == '\n') {
208 warn(in_format, "Missing closing quote.");
210 return !!fields_count(&in_fields);
215 if (c == in_format->quote) {
216 c = getchar_unlocked();
217 if (c != in_format->quote) {
221 // Two quotes assimilate to one
223 // Fall through to pushing the character
224 } else if (c == in_format->quote) {
227 } else if (c == in_format->fs && !quoted) {
233 *line_push(&in_line) = c;
238 static int is_ws(int c)
240 return (c == ' ' || c == '\t' || c == '\f');
243 static void csv_write(void)
245 unsigned char *line = line_first(&in_line);
246 int n = fields_count(&out_fields);
247 for (int i=0; i<n; i++) {
248 struct field *f = fields_nth(&out_fields, i);
250 if (out_format->quote >= 0) {
251 need_quotes = out_format->always_quote;
252 for (int j=0; !need_quotes && j < f->len; j++) {
253 int c = line[f->start_pos + j];
254 if (c == out_format->fs || c == out_format->quote)
259 putchar_unlocked(out_format->fs);
261 putchar_unlocked(out_format->quote);
262 for (int j=0; j < f->len; j++) {
263 int c = line[f->start_pos + j];
264 if (c == out_format->fs && !need_quotes)
265 warn(out_format, "Field separator found inside field and quoting is turned off.");
266 if (c == out_format->quote)
271 putchar_unlocked(out_format->quote);
273 putchar_unlocked('\n');
276 /*** White-space back-end ***/
278 static int ws_read(void)
283 unsigned char *line = line_first(&in_line);
284 int n = line_count(&in_line);
290 for (int i=0; i<n; i++) {
296 if (!in_field->start_pos &&
298 !in_format->strict_ws)
299 in_field->start_pos = i;
308 if (ws && in_format->strict_ws)
313 /*** Regex back-end ***/
315 static const char *regex_set(struct format *f, char *rx)
319 f->pcre = pcre_compile(rx, PCRE_DOLLAR_ENDONLY, &err, &errpos, NULL);
323 f->pcre_extra = pcre_study(f->pcre, 0, &err);
330 static int regex_read(void)
335 unsigned char *c = line_first(&in_line);
336 int n = line_count(&in_line);
343 int sep = pcre_exec(in_format->pcre, in_format->pcre_extra, (char *) c, n, i, 0, ovec, 3);
345 if (sep != PCRE_ERROR_NOMATCH)
346 warn(in_format, "PCRE matching error %d", sep);
347 // No further occurrence of the separator: the rest is a single field
349 in_field->len = n - i;
353 in_field->len = ovec[0] - i;
358 /*** Table back-end ***/
360 static void table_write(void)
362 for (int i = 0; i < fields_count(&in_fields); i++) {
364 printf("%*s", out_format->table_sep, "");
365 struct field *f = fields_nth(&in_fields, i);
366 int fw = field_chars(f);
367 int cw = *intarray_nth(&in_format->column_widths, i);
369 warn(out_format, "Internal error: Wrongly calculated column width (%d > %d)", fw, cw);
372 unsigned char *p = line_nth(&in_line, f->start_pos);
373 for (int j = 0; j < f->len; j++)
374 putchar_unlocked(p[j]);
376 putchar_unlocked(' ');
380 putchar_unlocked('\n');
383 /*** Temporary file back-end ***/
385 static int tmp_read(void)
387 FILE *tf = in_format->tmp_file;
390 int c = getc_unlocked(tf);
396 c = getc_unlocked(tf);
397 c = (c << 8) | getc_unlocked(tf);
398 c = (c << 8) | getc_unlocked(tf);
399 c = (c << 8) | getc_unlocked(tf);
401 new_field(line_count(&in_line));
404 int x = getc_unlocked(tf);
406 warn(in_format, "Truncated temporary file");
409 *line_push(&in_line) = x;
414 static void tmp_write(void)
416 FILE *tf = out_format->tmp_file;
418 for (int i = 0; i < fields_count(&in_fields); i++) {
419 struct field *f = fields_nth(&in_fields, i);
421 putc_unlocked(f->len, tf);
423 putc_unlocked(0xfe, tf);
424 putc_unlocked((f->len >> 24) & 0xff, tf);
425 putc_unlocked((f->len >> 16) & 0xff, tf);
426 putc_unlocked((f->len >> 8) & 0xff, tf);
427 putc_unlocked(f->len & 0xff, tf);
430 unsigned char *p = line_nth(&in_line, f->start_pos);
431 for (int j = 0; j < f->len; j++)
432 putc_unlocked(*p++, tf);
434 intarray_t *w = &out_format->column_widths;
435 while (i >= intarray_count(w))
436 *intarray_push(w) = 0;
437 int fw = field_chars(f);
438 if (*intarray_nth(w, i) < fw)
439 *intarray_nth(w, i) = fw;
441 putc_unlocked(0xff, tf);
446 static void trim_fields(void)
448 unsigned char *line = line_first(&in_line);
449 for (int i = 0; i < fields_count(&in_fields); i++) {
450 struct field *f = fields_nth(&in_fields, i);
451 while (f->len && is_ws(line[f->start_pos]))
452 f->start_pos++, f->len--;
453 while (f->len && is_ws(line[f->start_pos + f->len - 1]))
458 /*** Field selection ***/
461 int first_field, last_field;
464 DECLARE_BUF(selectors, struct selector);
465 static selectors_t selectors;
467 static char *parse_selector(char *str)
469 char buf[strlen(str) + 1];
472 struct selector *s = selectors_push(&selectors);
473 char *sep = strchr(buf, '-');
476 s->first_field = atoi(buf);
477 s->last_field = atoi(sep);
479 s->first_field = s->last_field = atoi(buf);
484 static void finish_parse_selectors(void)
486 if (!selectors_count(&selectors))
490 static void select_fields(void)
492 for (int i = 0; i < selectors_count(&selectors); i++) {
493 struct selector *s = selectors_nth(&selectors, i);
494 int first = s->first_field;
497 int last = s->last_field;
499 last = fields_count(&in_fields);
500 for (int j = first; j <= last; j++) {
501 struct field *f = fields_push(&out_fields);
502 if (j >= 1 && j <= fields_count(&in_fields))
503 *f = *fields_nth(&in_fields, j-1);
505 f->start_pos = f->len = 0;
510 /*** Processing of files ***/
512 static void one_pass(void)
517 fields_reset(&in_fields);
518 line_reset(&in_line);
520 if (!in_format->read_line())
526 fields_reset(&out_fields);
529 out_format->write_line();
533 static void two_pass(void)
535 struct format *final_format = out_format;
537 // We need to use character set info from the current locale
538 setlocale(LC_CTYPE, "");
540 // Pass 1: Set up writer of intermediate format
541 out_format = xmalloc_zero(sizeof(*out_format));
542 out_format->id = FORM_TMP;
543 out_format->read_line = tmp_read;
544 out_format->write_line = tmp_write;
545 out_format->tmp_file = tmpfile();
546 intarray_init(&out_format->column_widths);
549 // Pass 2: Set up reader of intermediate format
550 in_format = out_format;
551 rewind(in_format->tmp_file);
552 out_format = final_format;
554 fclose(in_format->tmp_file);
557 /*** Parsing of arguments ***/
559 static void NONRET usage(void)
562 Usage: xsv <in-format> [<out-format>] <options> [<fields>]\n\
565 -t, --tsv TAB-separated values (default)\n\
566 -c, --csv Comma-separated values\n\
567 -w, --ws Values separated by arbitrary whitespace\n\
568 -W, --strict-ws Like --ws, but recognize empty columns at start/end\n\
569 -r, --regex=<rx> Separator given by Perl regular expression (input only)\n\
570 --table Format a table (output only)\n\
572 Format parameters:\n\
573 -d, --fs=<char> Delimiter of fields\n\
574 -q, --quiet Do not show warnings\n\
575 --always-quote Put quotes around all fields (CSV output only)\n\
576 --table-sep=<n> Separate table columns by <n> spaces (default: 2)\n\
579 --trim Trim leading and trailing whitespaces in fields\n\
584 static void NONRET bad_args(const char *msg, ...)
589 fprintf(stderr, "xsv: ");
590 vfprintf(stderr, msg, args);
594 fprintf(stderr, "Try `xsv --help' for more information.\n");
598 static const char short_options[] = "cd:qr:twW";
608 static const struct option long_options[] = {
609 { "always-quote", 0, NULL, OPT_ALWAYS_QUOTE },
610 { "csv", 0, NULL, 'c' },
611 { "fs", 1, NULL, 'd' },
612 { "quiet", 0, NULL, 'q' },
613 { "regex", 1, NULL, 'r' },
614 { "strict-ws", 0, NULL, 'W' },
615 { "table", 0, NULL, OPT_TABLE },
616 { "table-sep", 1, NULL, OPT_TABLE_SEP },
617 { "trim", 0, NULL, OPT_TRIM },
618 { "tsv", 0, NULL, 't' },
619 { "ws", 0, NULL, 'w' },
620 { "help", 0, NULL, OPT_HELP },
621 { NULL, 0, NULL, 0 },
624 static void set_format(int format_id)
626 struct format *f = xmalloc_zero(sizeof(*f));
633 f->read_line = csv_read;
634 f->write_line = csv_write;
639 f->read_line = csv_read;
640 f->write_line = csv_write;
645 f->read_line = ws_read;
646 f->write_line = csv_write;
649 f->read_line = regex_read;
652 f->write_line = table_write;
653 f->needs_two_passes = 1;
660 else if (!out_format)
663 bad_args("At most two formats may be given.");
666 static struct format *current_format(void)
672 set_format(FORM_TSV);
676 int main(int argc, char **argv)
681 while ((opt = getopt_long(argc, argv, short_options, long_options, NULL)) >= 0)
684 set_format(FORM_CSV);
688 current_format()->fs = optarg[0];
690 bad_args("No field delimiter given.");
693 current_format()->quiet = 1;
696 set_format(FORM_REGEX);
697 err = regex_set(current_format(), optarg);
699 bad_args("Error compiling regex: %s", err);
702 set_format(FORM_TSV);
709 current_format()->strict_ws = 1;
711 case OPT_ALWAYS_QUOTE:
712 if (current_format()->id != FORM_CSV)
713 bad_args("--always-quote makes sense only for CSV.");
714 current_format()->always_quote = 1;
722 set_format(FORM_TABLE);
725 current_format()->table_sep = atoi(optarg);
733 out_format = in_format;
734 if (!in_format->read_line)
735 bad_args("Write-only format selected for input.");
736 if (!out_format->write_line)
737 bad_args("Read-only format selected for output.");
739 for (int i = optind; i < argc; i++) {
740 err = parse_selector(argv[i]);
744 finish_parse_selectors();
746 fields_init(&in_fields);
747 fields_init(&out_fields);
750 if (out_format->needs_two_passes)