2 * A Swiss-Army Knife for CSV-like Files
4 * (c) 2012 Martin Mares <mj@ucw.cz>
20 #define NONRET __attribute__((noreturn))
21 #define UNUSED __attribute__((unused))
27 /*** General functions ***/
29 static void NONRET die(char *msg, ...)
33 fprintf(stderr, "xsv: ");
34 vfprintf(stderr, msg, args);
40 /*** Memory allocation ***/
42 static void *xmalloc(size_t bytes)
44 void *p = malloc(bytes);
46 die("Out of memory (cannot allocate %zu bytes)", bytes);
50 static void *xmalloc_zero(size_t bytes)
52 void *p = xmalloc(bytes);
57 static void *xrealloc(void *old, size_t bytes)
59 void *p = realloc(old, bytes);
61 die("Out of memory (cannot allocate %zu bytes)", bytes);
65 #define DECLARE_BUF(name, type) \
66 typedef struct { type *start; int count; int max; } name##_t; \
67 static inline void name##_init(name##_t *b) { b->start = NULL; b->count = b->max = 0; } \
68 static inline void name##_reset(name##_t *b) { b->count = 0; } \
69 static inline int name##_count(name##_t *b) { return b->count; } \
70 static void name##_extend(name##_t *b) { \
71 b->max = b->max ? 2*b->max : 16; \
72 b->start = xrealloc(b->start, b->max * sizeof(type)); \
74 static inline type *name##_push(name##_t *b) { \
75 if (b->count >= b->max) name##_extend(b); \
76 return &b->start[b->count++]; \
78 static inline type *name##_first(name##_t *b) { return b->start; } \
79 static inline type *name##_nth(name##_t *b, int n) { return &b->start[n]; } \
82 DECLARE_BUF(intarray, int);
83 DECLARE_BUF(stringarray, char *);
85 /*** Formats and their parameters ***/
103 int (*read_line)(struct format *fmt);
104 void (*write_line)(struct format *fmt);
105 void (*write_grid)(struct format *fmt, int pos); // -1=above, 1=below, 0=after header
110 char *set_field_names;
111 struct field_names *field_names;
118 pcre_extra *pcre_extra;
120 // Temporary file backend:
128 static struct format *in_format, *out_format;
129 static int want_trim, want_equalize, want_stats;
136 DECLARE_BUF(fields, struct field);
137 DECLARE_BUF(line, unsigned char);
139 static fields_t in_fields, out_fields;
140 static struct field *in_field;
141 static line_t in_line;
142 static int line_number;
144 static int read_line(void)
146 fields_reset(&in_fields);
147 line_reset(&in_line);
149 if (!in_format->read_line(in_format))
151 if (ferror_unlocked(stdin))
152 die("I/O error when reading standard input");
156 static void write_line(void)
158 out_format->write_line(out_format);
159 if (ferror_unlocked(stdout))
160 die("I/O error when writing standard input");
163 static void write_grid(int pos)
165 if (out_format->write_grid) {
166 out_format->write_grid(out_format, pos);
167 if (ferror_unlocked(stdout))
168 die("I/O error when writing standard input");
172 static void new_field(int pos)
174 in_field = fields_push(&in_fields);
175 in_field->start_pos = pos;
179 static void ensure_field(int pos)
185 static unsigned char *get_field(fields_t *fields, int i, int *len)
187 struct field *f = fields_nth(fields, i);
189 return line_nth(&in_line, f->start_pos);
192 static void warn(struct format *fmt, char *msg, ...)
195 fprintf(stderr, "Warning at line %d: ", line_number);
198 vfprintf(stderr, msg, args);
204 static int next_line(void)
207 int c = getchar_unlocked();
211 return !!line_count(&in_line);
214 *line_push(&in_line) = c;
218 static int field_chars(struct field *f)
220 unsigned char *s = line_nth(&in_line, f->start_pos);
223 memset(&mbs, 0, sizeof(mbs));
227 size_t k = mbrlen((char *) s + i, f->len - i, &mbs);
237 /*** Field statistics ***/
239 static intarray_t column_widths;
241 static void update_stats(void)
246 for (int i = 0; i < fields_count(&out_fields); i++) {
247 struct field *f = fields_nth(&out_fields, i);
248 intarray_t *w = &column_widths;
250 while (i >= intarray_count(w))
251 *intarray_push(w) = 0;
252 int fw = field_chars(f);
253 if (*intarray_nth(w, i) < fw)
254 *intarray_nth(w, i) = fw;
258 /*** CSV/TSV back-end */
260 static int csv_read(struct format *fmt)
264 int c = getchar_unlocked();
265 int i = line_count(&in_line);
269 if (c < 0 || c == '\n') {
271 warn(fmt, "Missing closing quote.");
273 return !!fields_count(&in_fields);
278 if (c == fmt->quote) {
279 c = getchar_unlocked();
280 if (c != fmt->quote) {
284 // Two quotes assimilate to one
286 // Fall through to pushing the character
287 } else if (c == fmt->quote) {
290 } else if (c == fmt->fs && !quoted) {
296 *line_push(&in_line) = c;
301 static int is_ws(int c)
303 return (c == ' ' || c == '\t' || c == '\f');
306 static void csv_write(struct format *fmt)
308 for (int i=0; i < fields_count(&out_fields); i++) {
310 unsigned char *p = get_field(&out_fields, i, &len);
313 if (fmt->quote >= 0) {
314 need_quotes = fmt->always_quote;
315 for (int j=0; !need_quotes && j < len; j++) {
316 if (p[j] == fmt->fs || p[j] == fmt->quote)
321 putchar_unlocked(fmt->fs);
323 putchar_unlocked(fmt->quote);
324 for (int j=0; j < len; j++) {
326 if (c == fmt->fs && !need_quotes)
327 warn(fmt, "Field separator found inside field and quoting is turned off.");
333 putchar_unlocked(fmt->quote);
335 putchar_unlocked('\n');
338 /*** White-space back-end ***/
340 static int ws_read(struct format *fmt)
345 unsigned char *line = line_first(&in_line);
346 int n = line_count(&in_line);
352 for (int i=0; i<n; i++) {
358 if (!in_field->start_pos &&
361 in_field->start_pos = i;
370 if (ws && !fmt->sloppy)
375 /*** Regex back-end ***/
377 static const char *regex_set(struct format *f, char *rx)
381 f->pcre = pcre_compile(rx, PCRE_DOLLAR_ENDONLY, &err, &errpos, NULL);
385 f->pcre_extra = pcre_study(f->pcre, 0, &err);
392 static int regex_read(struct format *fmt)
397 unsigned char *c = line_first(&in_line);
398 int n = line_count(&in_line);
405 int err = pcre_exec(fmt->pcre, fmt->pcre_extra, (char *) c, n, i, 0, ovec, 3);
407 if (err != PCRE_ERROR_NOMATCH)
408 warn(fmt, "PCRE matching error %d", err);
409 // No further occurrence of the separator: the rest is a single field
410 if (!fmt->sloppy || i < n) {
412 in_field->len = n - i;
416 if (!fmt->sloppy || ovec[0]) {
418 in_field->len = ovec[0] - i;
424 /*** Table back-end ***/
426 static void table_write(struct format *fmt)
428 for (int i = 0; i < intarray_count(&column_widths); i++) {
429 if (fmt->table_grid) {
430 putchar_unlocked('|');
431 printf("%*s", fmt->table_sep / 2, "");
433 printf("%*s", fmt->table_sep, "");
435 int cw = *intarray_nth(&column_widths, i);
437 if (i < fields_count(&out_fields)) {
439 unsigned char *p = get_field(&out_fields, i, &len);
440 fw = field_chars(fields_nth(&out_fields, i));
442 warn(fmt, "Internal error: Wrongly calculated width of column %d (%d > %d)", i, fw, cw);
449 putchar_unlocked(' ');
454 printf("%*s", fmt->table_sep - fmt->table_sep / 2, "");
458 putchar_unlocked('|');
459 putchar_unlocked('\n');
462 static void table_write_grid(struct format *fmt, int pos UNUSED)
464 if (!fmt->table_grid)
467 for (int i = 0; i < intarray_count(&column_widths); i++) {
468 putchar_unlocked('+');
469 int w = fmt->table_sep + *intarray_nth(&column_widths, i);
473 putchar_unlocked('+');
474 putchar_unlocked('\n');
477 /*** Temporary file back-end ***/
479 static int tmp_read(struct format *fmt)
481 FILE *tf = fmt->tmp_file;
484 int c = getc_unlocked(tf);
490 c = getc_unlocked(tf);
491 c = (c << 8) | getc_unlocked(tf);
492 c = (c << 8) | getc_unlocked(tf);
493 c = (c << 8) | getc_unlocked(tf);
495 new_field(line_count(&in_line));
498 int x = getc_unlocked(tf);
500 die("Truncated temporary file");
501 *line_push(&in_line) = x;
505 if (ferror_unlocked(tf))
506 die("I/O error when reading temporary file");
509 static void tmp_write(struct format *fmt)
511 FILE *tf = fmt->tmp_file;
513 for (int i = 0; i < fields_count(&out_fields); i++) {
515 unsigned char *p = get_field(&out_fields, i, &len);
518 putc_unlocked(len, tf);
520 putc_unlocked(0xfe, tf);
521 putc_unlocked((len >> 24) & 0xff, tf);
522 putc_unlocked((len >> 16) & 0xff, tf);
523 putc_unlocked((len >> 8) & 0xff, tf);
524 putc_unlocked(len & 0xff, tf);
528 putc_unlocked(*p++, tf);
530 putc_unlocked(0xff, tf);
532 if (ferror_unlocked(tf))
533 die("I/O error when writing temporary file");
538 static void trim_fields(void)
540 unsigned char *line = line_first(&in_line);
541 for (int i = 0; i < fields_count(&in_fields); i++) {
542 struct field *f = fields_nth(&in_fields, i);
543 while (f->len && is_ws(line[f->start_pos]))
544 f->start_pos++, f->len--;
545 while (f->len && is_ws(line[f->start_pos + f->len - 1]))
550 static void equalize_fields(void)
552 while (fields_count(&out_fields) < intarray_count(&column_widths)) {
553 struct field *f = fields_push(&out_fields);
554 f->start_pos = f->len = 0;
558 /*** Field names and headers ***/
564 static void add_field(struct field_names *fn, char *name, int namelen)
566 char *n = xmalloc(namelen + 1);
567 memcpy(n, name, namelen);
569 *stringarray_push(&fn->names) = n;
572 static void add_field_names(struct field_names *fn, char *names)
576 char *q = strchr(p, ',');
577 int len = q ? q-p : (int) strlen(p);
578 add_field(fn, p, len);
583 static void read_header(void)
585 if (!(in_format->has_header || in_format->set_field_names))
588 struct field_names *fn = xmalloc_zero(sizeof(*fn));
589 in_format->field_names = fn;
591 if (in_format->has_header) {
593 die("Missing input header");
596 if (in_format->set_field_names) {
597 add_field_names(fn, in_format->set_field_names);
599 for (int i = 0; i < fields_count(&in_fields); i++) {
601 char *s = (char *) get_field(&in_fields, i, &len);
602 add_field(fn, s, len);
607 static void write_header(void)
609 if (!out_format->has_header) {
614 if (out_format->set_field_names) {
615 struct field_names *fn = xmalloc_zero(sizeof(*fn));
616 out_format->field_names = fn;
617 add_field_names(fn, out_format->set_field_names);
618 } else if (in_format->field_names)
619 out_format->field_names = in_format->field_names;
621 die("Output header requested, but no field names specified");
623 line_reset(&in_line);
624 fields_reset(&out_fields);
625 struct field_names *fn = out_format->field_names;
626 for (int i = 0; i < stringarray_count(&fn->names); i++) {
627 struct field *f = fields_push(&out_fields);
628 f->start_pos = line_count(&in_line);
630 char *s = *stringarray_nth(&fn->names, i);
632 *line_push(&in_line) = *s++;
637 // This is tricky: when we are formatting a table, field names are normally
638 // calculated in pass 1, but the header is written in pass 2, so we have to
639 // update column statistics, because field name can be too wide to fit.
650 static void write_footer(void)
655 static int find_field_by_name(struct field_names *fn, char *name)
657 for (int i = 0; i < stringarray_count(&fn->names); i++)
658 if (!strcmp(*stringarray_nth(&fn->names, i), name))
663 /*** Field selection ***/
666 int first_field, last_field; // 0 means "boundary"
669 DECLARE_BUF(selectors, struct selector);
670 static selectors_t selectors;
672 static int parse_field_num(char *str)
677 if (*str < '0' || *str > '9')
681 f = 10*f + *str - '0';
687 static int parse_field(char *str)
692 int f = parse_field_num(str);
696 if (in_format->field_names && (f = find_field_by_name(in_format->field_names, str)) > 0)
699 die("Unknown field %s", str);
702 static char *parse_selector(char *str)
704 char buf[strlen(str) + 1];
707 struct selector *s = selectors_push(&selectors);
708 char *sep = strchr(buf, '-');
711 s->first_field = parse_field(buf);
712 s->last_field = parse_field(sep);
714 s->first_field = s->last_field = parse_field(buf);
719 static void finish_parse_selectors(void)
721 if (!selectors_count(&selectors))
725 static void select_fields(void)
727 for (int i = 0; i < selectors_count(&selectors); i++) {
728 struct selector *s = selectors_nth(&selectors, i);
729 int first = s->first_field;
732 int last = s->last_field;
734 last = fields_count(&in_fields);
735 for (int j = first; j <= last; j++) {
736 struct field *f = fields_push(&out_fields);
737 if (j >= 1 && j <= fields_count(&in_fields))
738 *f = *fields_nth(&in_fields, j-1);
740 f->start_pos = f->len = 0;
745 static void select_all_fields(void)
747 for (int i = 0; i < fields_count(&in_fields); i++)
748 *fields_push(&out_fields) = *fields_nth(&in_fields, i);
751 /*** Processing of files ***/
753 static void one_pass(int pass)
763 if (want_trim && (pass & 1))
766 fields_reset(&out_fields);
772 if (want_equalize && (pass & 2))
782 static void two_pass(void)
784 struct format *final_format = out_format;
786 // We need to use character set info from the current locale
787 setlocale(LC_CTYPE, "");
789 // Pass 1: Set up writer of intermediate format
790 out_format = xmalloc_zero(sizeof(*out_format));
791 out_format->id = FORM_TMP;
792 out_format->read_line = tmp_read;
793 out_format->write_line = tmp_write;
794 out_format->tmp_file = tmpfile();
795 out_format->field_names = in_format->field_names;
798 // Pass 2: Set up reader of intermediate format
799 in_format = out_format;
800 rewind(in_format->tmp_file);
802 out_format = final_format;
805 fclose(in_format->tmp_file);
808 /*** Parsing of arguments ***/
810 static void NONRET usage(void)
813 Usage: xsv <in-format> [<out-format>] <options> [<fields>]\n\
816 -t, --tsv TAB-separated values (default)\n\
817 -c, --csv Comma-separated values\n\
818 -w, --ws Values separated by arbitrary whitespace\n\
819 -r, --regex=<rx> Separator given by Perl regular expression (input only)\n\
820 --table Format a table (output only)\n\
822 Format parameters:\n\
823 -d, --fs=<char> Delimiter of fields\n\
824 -f, --fields=<f>,... Set field names\n\
825 -h, --header The first line contains field names\n\
826 -q, --quiet Do not show warnings\n\
827 --always-quote Put quotes around all fields (CSV output only)\n\
828 --table-sep=<n> Separate table columns by <n> spaces (default: 2)\n\
829 --grid Separate table columns by grid lines\n\
830 -s, --sloppy Ignore separators at the start/end of line (ws/regex only)\n\
833 --trim Trim leading and trailing whitespaces in fields\n\
834 --equalize Pad all lines to the maximum number of fields\n\
839 static void NONRET bad_args(const char *msg, ...)
844 fprintf(stderr, "xsv: ");
845 vfprintf(stderr, msg, args);
849 fprintf(stderr, "Try `xsv --help' for more information.\n");
853 static const char short_options[] = "cd:f:hqr:twW";
865 static const struct option long_options[] = {
866 { "always-quote", 0, NULL, OPT_ALWAYS_QUOTE },
867 { "csv", 0, NULL, 'c' },
868 { "equalize", 0, NULL, OPT_EQUALIZE },
869 { "fields", 1, NULL, 'f' },
870 { "fs", 1, NULL, 'd' },
871 { "grid", 0, NULL, OPT_GRID },
872 { "header", 0, NULL, 'h' },
873 { "quiet", 0, NULL, 'q' },
874 { "regex", 1, NULL, 'r' },
875 { "sloppy", 0, NULL, 's' },
876 { "table", 0, NULL, OPT_TABLE },
877 { "table-sep", 1, NULL, OPT_TABLE_SEP },
878 { "trim", 0, NULL, OPT_TRIM },
879 { "tsv", 0, NULL, 't' },
880 { "ws", 0, NULL, 'w' },
881 { "help", 0, NULL, OPT_HELP },
882 { NULL, 0, NULL, 0 },
885 static void set_format(int format_id)
887 struct format *f = xmalloc_zero(sizeof(*f));
894 f->read_line = csv_read;
895 f->write_line = csv_write;
900 f->read_line = csv_read;
901 f->write_line = csv_write;
906 f->read_line = ws_read;
907 f->write_line = csv_write;
910 f->read_line = regex_read;
913 f->write_line = table_write;
914 f->write_grid = table_write_grid;
922 else if (!out_format)
925 bad_args("At most two formats may be given.");
928 static struct format *current_format(void)
934 set_format(FORM_TSV);
938 int main(int argc, char **argv)
943 while ((opt = getopt_long(argc, argv, short_options, long_options, NULL)) >= 0)
946 set_format(FORM_CSV);
950 current_format()->fs = optarg[0];
952 bad_args("No field delimiter given.");
955 current_format()->set_field_names = optarg;
958 current_format()->has_header = 1;
961 current_format()->quiet = 1;
964 set_format(FORM_REGEX);
965 err = regex_set(current_format(), optarg);
967 bad_args("Error compiling regex: %s", err);
970 if (current_format()->id != FORM_WS && current_format()->id != FORM_REGEX)
971 bad_args("--sloppy makes sense only for --ws or --regex.");
972 current_format()->sloppy = 1;
975 set_format(FORM_TSV);
980 case OPT_ALWAYS_QUOTE:
981 if (current_format()->id != FORM_CSV)
982 bad_args("--always-quote makes sense only for --csv.");
983 current_format()->always_quote = 1;
991 set_format(FORM_TABLE);
994 current_format()->table_sep = atoi(optarg);
997 current_format()->table_grid = 1;
1008 out_format = in_format;
1009 if (!in_format->read_line)
1010 bad_args("Write-only format selected for input.");
1011 if (!out_format->write_line)
1012 bad_args("Read-only format selected for output.");
1015 for (int i = optind; i < argc; i++) {
1016 err = parse_selector(argv[i]);
1020 finish_parse_selectors();
1022 want_stats = out_format->needs_stats | want_equalize;