xsv.c

   1 /*
   2  *      A Swiss-Army Knife for CSV-like Files
   3  *
   4  *      (c) 2012 Martin Mares <mj@ucw.cz>
   5  */
   6
   7 #define _GNU_SOURCE
   8
   9 #include <stdio.h>
  10 #include <stdlib.h>
  11 #include <string.h>
  12 #include <stdarg.h>
  13 #include <getopt.h>
  14 #include <wchar.h>
  15 #include <locale.h>
  16
  17 #include <pcre.h>
  18
  19 #ifdef __GNUC__
  20 #define NONRET __attribute__((noreturn))
  21 #else
  22 #define NONRET
  23 #endif
  24
  25 /*** General functions ***/
  26
  27 static void NONRET die(char *msg, ...)
  28 {
  29         va_list args;
  30         va_start(args, msg);
  31         fprintf(stderr, "xsv: ");
  32         vfprintf(stderr, msg, args);
  33         fputc('\n', stderr);
  34         va_end(args);
  35         exit(1);
  36 }
  37
  38 /*** Memory allocation ***/
  39
  40 static void *xmalloc(size_t bytes)
  41 {
  42         void *p = malloc(bytes);
  43         if (!p)
  44                 die("Out of memory (cannot allocate %zu bytes)", bytes);
  45         return p;
  46 }
  47
  48 static void *xmalloc_zero(size_t bytes)
  49 {
  50         void *p = xmalloc(bytes);
  51         memset(p, 0, bytes);
  52         return p;
  53 }
  54
  55 static void *xrealloc(void *old, size_t bytes)
  56 {
  57         void *p = realloc(old, bytes);
  58         if (!p)
  59                 die("Out of memory (cannot allocate %zu bytes)", bytes);
  60         return p;
  61 }
  62
  63 #define DECLARE_BUF(name, type) \
  64         typedef struct { type *start; int count; int max; } name##_t;                           \
  65         static inline void name##_init(name##_t *b) { b->start = NULL; b->count = b->max = 0; } \
  66         static inline void name##_reset(name##_t *b) { b->count = 0; }                          \
  67         static inline int name##_count(name##_t *b) { return b->count; }                        \
  68         static void name##_extend(name##_t *b) {                                                \
  69                 b->max = b->max ? 2*b->max : 16;                                                \
  70                 b->start = xrealloc(b->start, b->max * sizeof(type));                           \
  71         }                                                                                       \
  72         static inline type *name##_push(name##_t *b) {                                          \
  73                 if (b->count >= b->max) name##_extend(b);                                       \
  74                 return &b->start[b->count++];                                                   \
  75         }                                                                                       \
  76         static inline type *name##_first(name##_t *b) { return b->start; }                      \
  77         static inline type *name##_nth(name##_t *b, int n) { return &b->start[n]; }             \
  78         // end
  79
  80 DECLARE_BUF(intarray, int);
  81
  82 /*** Formats and their parameters ***/
  83
  84 enum format_id {
  85         FORM_UNSPEC,
  86         FORM_TSV,
  87         FORM_CSV,
  88         FORM_WS,
  89         FORM_REGEX,
  90         FORM_TMP,
  91         FORM_TABLE,
  92 };
  93
  94 struct format {
  95         enum format_id id;
  96         int fs;
  97         int quote;
  98         int quiet;
  99         int (*read_line)(void);
 100         void (*write_line)(void);
 101         int needs_two_passes;
 102
 103         // CSV backend:
 104         int always_quote;
 105
 106         // WS backend:
 107         int strict_ws;
 108
 109         // regex backend:
 110         pcre *pcre;
 111         pcre_extra *pcre_extra;
 112
 113         // Temporary file backend:
 114         FILE *tmp_file;
 115         intarray_t column_widths;
 116
 117         // Table backend:
 118         int table_sep;
 119 };
 120
 121 static struct format *in_format, *out_format;
 122 static int want_trim;
 123
 124 struct field {
 125         int start_pos;
 126         int len;
 127 };
 128
 129 DECLARE_BUF(fields, struct field);
 130 DECLARE_BUF(line, unsigned char);
 131
 132 static fields_t in_fields, out_fields;
 133 static struct field *in_field;
 134 static line_t in_line;
 135 static int line_number;
 136
 137 static void new_field(int pos)
 138 {
 139         in_field = fields_push(&in_fields);
 140         in_field->start_pos = pos;
 141         in_field->len = 0;
 142 }
 143
 144 static void ensure_field(int pos)
 145 {
 146         if (!in_field)
 147                 new_field(pos);
 148 }
 149
 150 static void warn(struct format *fmt, char *msg, ...)
 151 {
 152         if (!fmt->quiet) {
 153                 fprintf(stderr, "Warning at line %d: ", line_number);
 154                 va_list args;
 155                 va_start(args, msg);
 156                 vfprintf(stderr, msg, args);
 157                 va_end(args);
 158                 fputc('\n', stderr);
 159         }
 160 }
 161
 162 static int next_line(void)
 163 {
 164         for (;;) {
 165                 int c = getchar_unlocked();
 166                 if (c == '\r')
 167                         continue;
 168                 if (c < 0)
 169                         return !!line_count(&in_line);
 170                 if (c == '\n')
 171                         return 1;
 172                 *line_push(&in_line) = c;
 173         }
 174 }
 175
 176 static int field_chars(struct field *f)
 177 {
 178         unsigned char *s = line_nth(&in_line, f->start_pos);
 179         int i = 0;
 180         mbstate_t mbs;
 181         memset(&mbs, 0, sizeof(mbs));
 182
 183         int chars = 0;
 184         while (i < f->len) {
 185                 size_t k = mbrlen((char *) s + i, f->len - i, &mbs);
 186                 if ((int) k <= 0)
 187                         break;
 188                 i += k;
 189                 chars++;
 190         }
 191
 192         return chars;
 193 }
 194
 195 /*** CSV/TSV back-end */
 196
 197 static int csv_read(void)
 198 {
 199         int quoted = 0;
 200         for (;;) {
 201                 int c = getchar_unlocked();
 202                 int i = line_count(&in_line);
 203 restart:
 204                 if (c == '\r')
 205                         continue;
 206                 if (c < 0 || c == '\n') {
 207                         if (quoted)
 208                                 warn(in_format, "Missing closing quote.");
 209                         if (c < 0)
 210                                 return !!fields_count(&in_fields);
 211                         else
 212                                 return 1;
 213                 }
 214                 if (quoted) {
 215                         if (c == in_format->quote) {
 216                                 c = getchar_unlocked();
 217                                 if (c != in_format->quote) {
 218                                         quoted = 0;
 219                                         goto restart;
 220                                 }
 221                                 // Two quotes assimilate to one
 222                         }
 223                         // Fall through to pushing the character
 224                 } else if (c == in_format->quote) {
 225                         quoted = 1;
 226                         continue;
 227                 } else if (c == in_format->fs && !quoted) {
 228                         ensure_field(i);
 229                         new_field(i);
 230                         continue;
 231                 }
 232                 ensure_field(i);
 233                 *line_push(&in_line) = c;
 234                 in_field->len++;
 235         }
 236 }
 237
 238 static int is_ws(int c)
 239 {
 240         return (c == ' ' || c == '\t' || c == '\f');
 241 }
 242
 243 static void csv_write(void)
 244 {
 245         unsigned char *line = line_first(&in_line);
 246         int n = fields_count(&out_fields);
 247         for (int i=0; i<n; i++) {
 248                 struct field *f = fields_nth(&out_fields, i);
 249                 int need_quotes = 0;
 250                 if (out_format->quote >= 0) {
 251                         need_quotes = out_format->always_quote;
 252                         for (int j=0; !need_quotes && j < f->len; j++) {
 253                                 int c = line[f->start_pos + j];
 254                                 if (c == out_format->fs || c == out_format->quote)
 255                                         need_quotes = 1;
 256                         }
 257                 }
 258                 if (i)
 259                         putchar_unlocked(out_format->fs);
 260                 if (need_quotes)
 261                         putchar_unlocked(out_format->quote);
 262                 for (int j=0; j < f->len; j++) {
 263                         int c = line[f->start_pos + j];
 264                         if (c == out_format->fs && !need_quotes)
 265                                 warn(out_format, "Field separator found inside field and quoting is turned off.");
 266                         if (c == out_format->quote)
 267                                 putchar_unlocked(c);
 268                         putchar_unlocked(c);
 269                 }
 270                 if (need_quotes)
 271                         putchar_unlocked(out_format->quote);
 272         }
 273         putchar_unlocked('\n');
 274 }
 275
 276 /*** White-space back-end ***/
 277
 278 static int ws_read(void)
 279 {
 280         if (!next_line())
 281                 return 0;
 282
 283         unsigned char *line = line_first(&in_line);
 284         int n = line_count(&in_line);
 285         if (!n)
 286                 return 1;
 287
 288         int ws = 0;
 289         new_field(0);
 290         for (int i=0; i<n; i++) {
 291                 int c = line[i];
 292                 if (is_ws(c)) {
 293                         ws++;
 294                 } else {
 295                         if (ws) {
 296                                 if (!in_field->start_pos &&
 297                                     !in_field->len &&
 298                                     !in_format->strict_ws)
 299                                         in_field->start_pos = i;
 300                                 else
 301                                         new_field(i);
 302                                 ws = 0;
 303                         }
 304                         in_field->len++;
 305                 }
 306         }
 307
 308         if (ws && in_format->strict_ws)
 309                 new_field(n);
 310         return 1;
 311 }
 312
 313 /*** Regex back-end ***/
 314
 315 static const char *regex_set(struct format *f, char *rx)
 316 {
 317         const char *err;
 318         int errpos;
 319         f->pcre = pcre_compile(rx, PCRE_DOLLAR_ENDONLY, &err, &errpos, NULL);
 320         if (!f->pcre)
 321                 return err;
 322
 323         f->pcre_extra = pcre_study(f->pcre, 0, &err);
 324         if (!f->pcre_extra)
 325                 return err;
 326
 327         return NULL;
 328 }
 329
 330 static int regex_read(void)
 331 {
 332         if (!next_line())
 333                 return 0;
 334
 335         unsigned char *c = line_first(&in_line);
 336         int n = line_count(&in_line);
 337         if (!n)
 338                 return 1;
 339
 340         int i = 0;
 341         for (;;) {
 342                 int ovec[3];
 343                 int sep = pcre_exec(in_format->pcre, in_format->pcre_extra, (char *) c, n, i, 0, ovec, 3);
 344                 if (sep < 0) {
 345                         if (sep != PCRE_ERROR_NOMATCH)
 346                                 warn(in_format, "PCRE matching error %d", sep);
 347                         // No further occurrence of the separator: the rest is a single field
 348                         new_field(i);
 349                         in_field->len = n - i;
 350                         return 1;
 351                 }
 352                 new_field(i);
 353                 in_field->len = ovec[0] - i;
 354                 i = ovec[1];
 355         }
 356 }
 357
 358 /*** Table back-end ***/
 359
 360 static void table_write(void)
 361 {
 362         for (int i = 0; i < fields_count(&in_fields); i++) {
 363                 if (i)
 364                         printf("%*s", out_format->table_sep, "");
 365                 struct field *f = fields_nth(&in_fields, i);
 366                 int fw = field_chars(f);
 367                 int cw = *intarray_nth(&in_format->column_widths, i);
 368                 if (fw > cw) {
 369                         warn(out_format, "Internal error: Wrongly calculated column width (%d > %d)", fw, cw);
 370                         cw = fw;
 371                 }
 372                 unsigned char *p = line_nth(&in_line, f->start_pos);
 373                 for (int j = 0; j < f->len; j++)
 374                         putchar_unlocked(p[j]);
 375                 while (fw < cw) {
 376                         putchar_unlocked(' ');
 377                         fw++;
 378                 }
 379         }
 380         putchar_unlocked('\n');
 381 }
 382
 383 /*** Temporary file back-end ***/
 384
 385 static int tmp_read(void)
 386 {
 387         FILE *tf = in_format->tmp_file;
 388
 389         for (;;) {
 390                 int c = getc_unlocked(tf);
 391                 if (c < 0)
 392                         return 0;
 393                 if (c == 0xff)
 394                         return 1;
 395                 if (c == 0xfe) {
 396                         c = getc_unlocked(tf);
 397                         c = (c << 8) | getc_unlocked(tf);
 398                         c = (c << 8) | getc_unlocked(tf);
 399                         c = (c << 8) | getc_unlocked(tf);
 400                 }
 401                 new_field(line_count(&in_line));
 402                 in_field->len = c;
 403                 while (c--) {
 404                         int x = getc_unlocked(tf);
 405                         if (x < 0) {
 406                                 warn(in_format, "Truncated temporary file");
 407                                 return 0;
 408                         }
 409                         *line_push(&in_line) = x;
 410                 }
 411         }
 412
 413         if (ferror_unlocked(tf))
 414                 die("I/O error when reading temporary file");
 415 }
 416
 417 static void tmp_write(void)
 418 {
 419         FILE *tf = out_format->tmp_file;
 420
 421         for (int i = 0; i < fields_count(&in_fields); i++) {
 422                 struct field *f = fields_nth(&in_fields, i);
 423                 if (f->len < 0xfe)
 424                         putc_unlocked(f->len, tf);
 425                 else {
 426                         putc_unlocked(0xfe, tf);
 427                         putc_unlocked((f->len >> 24) & 0xff, tf);
 428                         putc_unlocked((f->len >> 16) & 0xff, tf);
 429                         putc_unlocked((f->len >> 8) & 0xff, tf);
 430                         putc_unlocked(f->len & 0xff, tf);
 431                 }
 432
 433                 unsigned char *p = line_nth(&in_line, f->start_pos);
 434                 for (int j = 0; j < f->len; j++)
 435                         putc_unlocked(*p++, tf);
 436
 437                 intarray_t *w = &out_format->column_widths;
 438                 while (i >= intarray_count(w))
 439                         *intarray_push(w) = 0;
 440                 int fw = field_chars(f);
 441                 if (*intarray_nth(w, i) < fw)
 442                         *intarray_nth(w, i) = fw;
 443         }
 444         putc_unlocked(0xff, tf);
 445
 446         if (ferror_unlocked(tf))
 447                 die("I/O error when writing temporary file");
 448 }
 449
 450 /*** Transforms ***/
 451
 452 static void trim_fields(void)
 453 {
 454         unsigned char *line = line_first(&in_line);
 455         for (int i = 0; i < fields_count(&in_fields); i++) {
 456                 struct field *f = fields_nth(&in_fields, i);
 457                 while (f->len && is_ws(line[f->start_pos]))
 458                         f->start_pos++, f->len--;
 459                 while (f->len && is_ws(line[f->start_pos + f->len - 1]))
 460                         f->len--;
 461         }
 462 }
 463
 464 /*** Field selection ***/
 465
 466 struct selector {
 467         int first_field, last_field;
 468 };
 469
 470 DECLARE_BUF(selectors, struct selector);
 471 static selectors_t selectors;
 472
 473 static char *parse_selector(char *str)
 474 {
 475         char buf[strlen(str) + 1];
 476         strcpy(buf, str);
 477
 478         struct selector *s = selectors_push(&selectors);
 479         char *sep = strchr(buf, '-');
 480         if (sep) {
 481                 *sep++ = 0;
 482                 s->first_field = atoi(buf);
 483                 s->last_field = atoi(sep);
 484         } else
 485                 s->first_field = s->last_field = atoi(buf);
 486
 487         return NULL;
 488 }
 489
 490 static void finish_parse_selectors(void)
 491 {
 492         if (!selectors_count(&selectors))
 493                 parse_selector("-");
 494 }
 495
 496 static void select_fields(void)
 497 {
 498         for (int i = 0; i < selectors_count(&selectors); i++) {
 499                 struct selector *s = selectors_nth(&selectors, i);
 500                 int first = s->first_field;
 501                 if (first <= 0)
 502                         first = 1;
 503                 int last = s->last_field;
 504                 if (last <= 0)
 505                         last = fields_count(&in_fields);
 506                 for (int j = first; j <= last; j++) {
 507                         struct field *f = fields_push(&out_fields);
 508                         if (j >= 1 && j <= fields_count(&in_fields))
 509                                 *f = *fields_nth(&in_fields, j-1);
 510                         else
 511                                 f->start_pos = f->len = 0;
 512                 }
 513         }
 514 }
 515
 516 /*** Processing of files ***/
 517
 518 static void one_pass(void)
 519 {
 520         line_number = 0;
 521         for (;;) {
 522                 line_number++;
 523                 fields_reset(&in_fields);
 524                 line_reset(&in_line);
 525                 in_field = NULL;
 526                 if (!in_format->read_line())
 527                         break;
 528                 if (ferror_unlocked(stdin))
 529                         die("I/O error when reading standard input");
 530
 531                 if (want_trim)
 532                         trim_fields();
 533
 534                 fields_reset(&out_fields);
 535                 select_fields();
 536
 537                 out_format->write_line();
 538                 if (ferror_unlocked(stdout))
 539                         die("I/O error when writing standard input");
 540         }
 541 }
 542
 543 static void two_pass(void)
 544 {
 545         struct format *final_format = out_format;
 546
 547         // We need to use character set info from the current locale
 548         setlocale(LC_CTYPE, "");
 549
 550         // Pass 1: Set up writer of intermediate format
 551         out_format = xmalloc_zero(sizeof(*out_format));
 552         out_format->id = FORM_TMP;
 553         out_format->read_line = tmp_read;
 554         out_format->write_line = tmp_write;
 555         out_format->tmp_file = tmpfile();
 556         one_pass();
 557
 558         // Pass 2: Set up reader of intermediate format
 559         in_format = out_format;
 560         rewind(in_format->tmp_file);
 561         out_format = final_format;
 562         one_pass();
 563         fclose(in_format->tmp_file);
 564 }
 565
 566 /*** Parsing of arguments ***/
 567
 568 static void NONRET usage(void)
 569 {
 570         printf("\
 571 Usage: xsv <in-format> [<out-format>] <options> [<fields>]\n\
 572 \n\
 573 Formats:\n\
 574 -t, --tsv               TAB-separated values (default)\n\
 575 -c, --csv               Comma-separated values\n\
 576 -w, --ws                Values separated by arbitrary whitespace\n\
 577 -W, --strict-ws         Like --ws, but recognize empty columns at start/end\n\
 578 -r, --regex=<rx>        Separator given by Perl regular expression (input only)\n\
 579     --table             Format a table (output only)\n\
 580 \n\
 581 Format parameters:\n\
 582 -d, --fs=<char>         Delimiter of fields\n\
 583 -q, --quiet             Do not show warnings\n\
 584     --always-quote      Put quotes around all fields (CSV output only)\n\
 585     --table-sep=<n>     Separate table columns by <n> spaces (default: 2)\n\
 586 \n\
 587 Other options:\n\
 588     --trim              Trim leading and trailing whitespaces in fields\n\
 589 ");
 590         exit(0);
 591 }
 592
 593 static void NONRET bad_args(const char *msg, ...)
 594 {
 595         if (msg) {
 596                 va_list args;
 597                 va_start(args, msg);
 598                 fprintf(stderr, "xsv: ");
 599                 vfprintf(stderr, msg, args);
 600                 fputc('\n', stderr);
 601                 va_end(args);
 602         }
 603         fprintf(stderr, "Try `xsv --help' for more information.\n");
 604         exit(1);
 605 }
 606
 607 static const char short_options[] = "cd:qr:twW";
 608
 609 enum long_options {
 610         OPT_HELP = 256,
 611         OPT_TRIM,
 612         OPT_ALWAYS_QUOTE,
 613         OPT_TABLE,
 614         OPT_TABLE_SEP,
 615 };
 616
 617 static const struct option long_options[] = {
 618         { "always-quote",       0,      NULL,   OPT_ALWAYS_QUOTE },
 619         { "csv",                0,      NULL,   'c' },
 620         { "fs",                 1,      NULL,   'd' },
 621         { "quiet",              0,      NULL,   'q' },
 622         { "regex",              1,      NULL,   'r' },
 623         { "strict-ws",          0,      NULL,   'W' },
 624         { "table",              0,      NULL,   OPT_TABLE },
 625         { "table-sep",          1,      NULL,   OPT_TABLE_SEP },
 626         { "trim",               0,      NULL,   OPT_TRIM },
 627         { "tsv",                0,      NULL,   't' },
 628         { "ws",                 0,      NULL,   'w' },
 629         { "help",               0,      NULL,   OPT_HELP },
 630         { NULL,                 0,      NULL,   0 },
 631 };
 632
 633 static void set_format(int format_id)
 634 {
 635         struct format *f = xmalloc_zero(sizeof(*f));
 636         f->id = format_id;
 637
 638         switch (format_id) {
 639                 case FORM_TSV:
 640                         f->fs = '\t';
 641                         f->quote = -1;
 642                         f->read_line = csv_read;
 643                         f->write_line = csv_write;
 644                         break;
 645                 case FORM_CSV:
 646                         f->fs = ',';
 647                         f->quote = '"';
 648                         f->read_line = csv_read;
 649                         f->write_line = csv_write;
 650                         break;
 651                 case FORM_WS:
 652                         f->fs = ' ';
 653                         f->quote = -1;
 654                         f->read_line = ws_read;
 655                         f->write_line = csv_write;
 656                         break;
 657                 case FORM_REGEX:
 658                         f->read_line = regex_read;
 659                         break;
 660                 case FORM_TABLE:
 661                         f->write_line = table_write;
 662                         f->needs_two_passes = 1;
 663                         f->table_sep = 2;
 664                         break;
 665         }
 666
 667         if (!in_format)
 668                 in_format = f;
 669         else if (!out_format)
 670                 out_format = f;
 671         else
 672                 bad_args("At most two formats may be given.");
 673 }
 674
 675 static struct format *current_format(void)
 676 {
 677         if (out_format)
 678                 return out_format;
 679         if (in_format)
 680                 return in_format;
 681         set_format(FORM_TSV);
 682         return in_format;
 683 }
 684
 685 int main(int argc, char **argv)
 686 {
 687         int opt;
 688         const char *err;
 689
 690         while ((opt = getopt_long(argc, argv, short_options, long_options, NULL)) >= 0)
 691                 switch (opt) {
 692                         case 'c':
 693                                 set_format(FORM_CSV);
 694                                 break;
 695                         case 'd':
 696                                 if (optarg[0])
 697                                         current_format()->fs = optarg[0];
 698                                 else
 699                                         bad_args("No field delimiter given.");
 700                                 break;
 701                         case 'q':
 702                                 current_format()->quiet = 1;
 703                                 break;
 704                         case 'r':
 705                                 set_format(FORM_REGEX);
 706                                 err = regex_set(current_format(), optarg);
 707                                 if (err)
 708                                         bad_args("Error compiling regex: %s", err);
 709                                 break;
 710                         case 't':
 711                                 set_format(FORM_TSV);
 712                                 break;
 713                         case 'w':
 714                                 set_format(FORM_WS);
 715                                 break;
 716                         case 'W':
 717                                 set_format(FORM_WS);
 718                                 current_format()->strict_ws = 1;
 719                                 break;
 720                         case OPT_ALWAYS_QUOTE:
 721                                 if (current_format()->id != FORM_CSV)
 722                                         bad_args("--always-quote makes sense only for CSV.");
 723                                 current_format()->always_quote = 1;
 724                                 break;
 725                         case OPT_HELP:
 726                                 usage();
 727                         case OPT_TRIM:
 728                                 want_trim = 1;
 729                                 break;
 730                         case OPT_TABLE:
 731                                 set_format(FORM_TABLE);
 732                                 break;
 733                         case OPT_TABLE_SEP:
 734                                 current_format()->table_sep = atoi(optarg);
 735                                 break;
 736                         default:
 737                                 bad_args(NULL);
 738                 }
 739
 740         current_format();
 741         if (!out_format)
 742                 out_format = in_format;
 743         if (!in_format->read_line)
 744                 bad_args("Write-only format selected for input.");
 745         if (!out_format->write_line)
 746                 bad_args("Read-only format selected for output.");
 747
 748         for (int i = optind; i < argc; i++) {
 749                 err = parse_selector(argv[i]);
 750                 if (err)
 751                         bad_args(err);
 752         }
 753         finish_parse_selectors();
 754
 755         if (out_format->needs_two_passes)
 756                 two_pass();
 757         else
 758                 one_pass();
 759         return 0;
 760 }