xsv.c

   1 /*
   2  *      A Swiss-Army Knife for CSV-like Files
   3  *
   4  *      (c) 2012 Martin Mares <mj@ucw.cz>
   5  */
   6
   7 #include <stdio.h>
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <stdarg.h>
  11 #include <getopt.h>
  12
  13 #include <pcre.h>
  14
  15 /*** Memory allocation ***/
  16
  17 static void *xmalloc(size_t bytes)
  18 {
  19         void *p = malloc(bytes);
  20         if (!p) {
  21                 fprintf(stderr, "xsv: Out of memory (cannot allocate %zu bytes)\n", bytes);
  22                 exit(1);
  23         }
  24         return p;
  25 }
  26
  27 static void *xmalloc_zero(size_t bytes)
  28 {
  29         void *p = xmalloc(bytes);
  30         memset(p, 0, bytes);
  31         return p;
  32 }
  33
  34 static void *xrealloc(void *old, size_t bytes)
  35 {
  36         void *p = realloc(old, bytes);
  37         if (!p) {
  38                 fprintf(stderr, "xsv: Out of memory (cannot allocate %zu bytes)\n", bytes);
  39                 exit(1);
  40         }
  41         return p;
  42 }
  43
  44 #define DECLARE_BUF(name, type) \
  45         typedef struct { type *start; int count; int max; } name##_t;                           \
  46         static inline void name##_init(name##_t *b) { b->start = NULL; b->count = b->max = 0; } \
  47         static inline void name##_reset(name##_t *b) { b->count = 0; }                          \
  48         static inline int name##_count(name##_t *b) { return b->count; }                        \
  49         static void name##_extend(name##_t *b) {                                                \
  50                 b->max = b->max ? 2*b->max : 16;                                                \
  51                 b->start = xrealloc(b->start, b->max * sizeof(type));                           \
  52         }                                                                                       \
  53         static inline type *name##_push(name##_t *b) {                                          \
  54                 if (b->count >= b->max) name##_extend(b);                                       \
  55                 return &b->start[b->count++];                                                   \
  56         }                                                                                       \
  57         static inline type *name##_first(name##_t *b) { return b->start; }                      \
  58         static inline type *name##_nth(name##_t *b, int n) { return &b->start[n]; }             \
  59         // end
  60
  61 DECLARE_BUF(intarray, int);
  62
  63 /*** Formats and their parameters ***/
  64
  65 enum format_id {
  66         FORM_UNSPEC,
  67         FORM_TSV,
  68         FORM_CSV,
  69         FORM_WS,
  70         FORM_REGEX,
  71         FORM_TMP,
  72         FORM_TABLE,
  73 };
  74
  75 struct format {
  76         enum format_id id;
  77         int fs;
  78         int quote;
  79         int quiet;
  80         int (*read_line)(void);
  81         void (*write_line)(void);
  82         int needs_two_passes;
  83
  84         // CSV backend:
  85         int always_quote;
  86
  87         // WS backend:
  88         int strict_ws;
  89
  90         // regex backend:
  91         pcre *pcre;
  92         pcre_extra *pcre_extra;
  93
  94         // Temporary file backend:
  95         FILE *tmp_file;
  96         intarray_t column_widths;
  97
  98         // Table backend:
  99         int table_sep;
 100 };
 101
 102 static struct format *in_format, *out_format;
 103 static int want_trim;
 104
 105 struct field {
 106         int start_pos;
 107         int len;
 108 };
 109
 110 DECLARE_BUF(fields, struct field);
 111 DECLARE_BUF(line, unsigned char);
 112
 113 static fields_t in_fields, out_fields;
 114 static struct field *in_field;
 115 static line_t in_line;
 116 static int line_number;
 117
 118 static void new_field(int pos)
 119 {
 120         in_field = fields_push(&in_fields);
 121         in_field->start_pos = pos;
 122         in_field->len = 0;
 123 }
 124
 125 static void ensure_field(int pos)
 126 {
 127         if (!in_field)
 128                 new_field(pos);
 129 }
 130
 131 static void warn(struct format *fmt, char *msg, ...)
 132 {
 133         if (!fmt->quiet) {
 134                 fprintf(stderr, "Warning at line %d: ", line_number);
 135                 va_list args;
 136                 va_start(args, msg);
 137                 vfprintf(stderr, args, msg);
 138                 va_end(args);
 139                 fputc('\n', stderr);
 140         }
 141 }
 142
 143 static int next_line(void)
 144 {
 145         for (;;) {
 146                 int c = getchar();
 147                 if (c == '\r')
 148                         continue;
 149                 if (c < 0)
 150                         return !!line_count(&in_line);
 151                 if (c == '\n')
 152                         return 1;
 153                 *line_push(&in_line) = c;
 154         }
 155 }
 156
 157 /*** CSV/TSV back-end */
 158
 159 static int csv_read(void)
 160 {
 161         int quoted = 0;
 162         for (;;) {
 163                 int c = getchar();
 164                 int i = line_count(&in_line);
 165 restart:
 166                 if (c == '\r')
 167                         continue;
 168                 if (c < 0 || c == '\n') {
 169                         if (quoted)
 170                                 warn(in_format, "Missing closing quote.");
 171                         if (c < 0)
 172                                 return !!fields_count(&in_fields);
 173                         else
 174                                 return 1;
 175                 }
 176                 if (quoted) {
 177                         if (c == in_format->quote) {
 178                                 c = getchar();
 179                                 if (c != in_format->quote) {
 180                                         quoted = 0;
 181                                         goto restart;
 182                                 }
 183                                 // Two quotes assimilate to one
 184                         }
 185                         // Fall through to pushing the character
 186                 } else if (c == in_format->quote) {
 187                         quoted = 1;
 188                         continue;
 189                 } else if (c == in_format->fs && !quoted) {
 190                         ensure_field(i);
 191                         new_field(i);
 192                         continue;
 193                 }
 194                 ensure_field(i);
 195                 *line_push(&in_line) = c;
 196                 in_field->len++;
 197         }
 198 }
 199
 200 static int is_ws(int c)
 201 {
 202         return (c == ' ' || c == '\t' || c == '\f');
 203 }
 204
 205 static void csv_write(void)
 206 {
 207         unsigned char *line = line_first(&in_line);
 208         int n = fields_count(&out_fields);
 209         for (int i=0; i<n; i++) {
 210                 struct field *f = fields_nth(&out_fields, i);
 211                 int need_quotes = 0;
 212                 if (out_format->quote >= 0) {
 213                         need_quotes = out_format->always_quote;
 214                         for (int j=0; !need_quotes && j < f->len; j++) {
 215                                 int c = line[f->start_pos + j];
 216                                 if (c == out_format->fs || c == out_format->quote)
 217                                         need_quotes = 1;
 218                         }
 219                 }
 220                 if (i)
 221                         putchar(out_format->fs);
 222                 if (need_quotes)
 223                         putchar(out_format->quote);
 224                 for (int j=0; j < f->len; j++) {
 225                         int c = line[f->start_pos + j];
 226                         if (c == out_format->fs && !need_quotes)
 227                                 warn(out_format, "Field separator found inside field and quoting is turned off.");
 228                         if (c == out_format->quote)
 229                                 putchar(c);
 230                         putchar(c);
 231                 }
 232                 if (need_quotes)
 233                         putchar(out_format->quote);
 234         }
 235         putchar('\n');
 236 }
 237
 238 /*** White-space back-end ***/
 239
 240 static int ws_read(void)
 241 {
 242         if (!next_line())
 243                 return 0;
 244
 245         unsigned char *line = line_first(&in_line);
 246         int n = line_count(&in_line);
 247         if (!n)
 248                 return 1;
 249
 250         int ws = 0;
 251         new_field(0);
 252         for (int i=0; i<n; i++) {
 253                 int c = line[i];
 254                 if (is_ws(c)) {
 255                         ws++;
 256                 } else {
 257                         if (ws) {
 258                                 if (!in_field->start_pos &&
 259                                     !in_field->len &&
 260                                     !in_format->strict_ws)
 261                                         in_field->start_pos = i;
 262                                 else
 263                                         new_field(i);
 264                                 ws = 0;
 265                         }
 266                         in_field->len++;
 267                 }
 268         }
 269
 270         if (ws && in_format->strict_ws)
 271                 new_field(n);
 272         return 1;
 273 }
 274
 275 /*** Regex back-end ***/
 276
 277 static const char *regex_set(struct format *f, char *rx)
 278 {
 279         const char *err;
 280         int errpos;
 281         f->pcre = pcre_compile(rx, PCRE_DOLLAR_ENDONLY, &err, &errpos, NULL);
 282         if (!f->pcre)
 283                 return err;
 284
 285         f->pcre_extra = pcre_study(f->pcre, 0, &err);
 286         if (!f->pcre_extra)
 287                 return err;
 288
 289         return NULL;
 290 }
 291
 292 static int regex_read(void)
 293 {
 294         if (!next_line())
 295                 return 0;
 296
 297         unsigned char *c = line_first(&in_line);
 298         int n = line_count(&in_line);
 299         if (!n)
 300                 return 1;
 301
 302         int i = 0;
 303         for (;;) {
 304                 int ovec[3];
 305                 int sep = pcre_exec(in_format->pcre, in_format->pcre_extra, (char *) c, n, i, 0, ovec, 3);
 306                 if (sep < 0) {
 307                         if (sep != PCRE_ERROR_NOMATCH)
 308                                 warn(in_format, "PCRE matching error %d", sep);
 309                         // No further occurrence of the separator: the rest is a single field
 310                         new_field(i);
 311                         in_field->len = n - i;
 312                         return 1;
 313                 }
 314                 new_field(i);
 315                 in_field->len = ovec[0] - i;
 316                 i = ovec[1];
 317         }
 318 }
 319
 320 /*** Table back-end ***/
 321
 322 static void table_write(void)
 323 {
 324         for (int i = 0; i < fields_count(&in_fields); i++) {
 325                 if (i)
 326                         printf("%*s", out_format->table_sep, "");
 327                 struct field *f = fields_nth(&in_fields, i);
 328                 int w = *intarray_nth(&in_format->column_widths, i);
 329                 if (f->len > w) {
 330                         warn(out_format, "Internal error: Wrongly calculated column width (%d > %d)", f->len, w);
 331                         w = f->len;
 332                 }
 333                 int j = 0;
 334                 unsigned char *p = line_nth(&in_line, f->start_pos);
 335                 while (j < f->len) {
 336                         putchar(*p++);
 337                         j++;
 338                 }
 339                 while (j < w) {
 340                         putchar(' ');
 341                         j++;
 342                 }
 343         }
 344         putchar('\n');
 345 }
 346
 347 /*** Temporary file back-end ***/
 348
 349 static int tmp_read(void)
 350 {
 351         FILE *tf = in_format->tmp_file;
 352
 353         for (;;) {
 354                 int c = fgetc(tf);
 355                 if (c < 0)
 356                         return 0;
 357                 if (c == 0xff)
 358                         return 1;
 359                 if (c == 0xfe) {
 360                         c = fgetc(tf);
 361                         c = (c << 8) | fgetc(tf);
 362                         c = (c << 8) | fgetc(tf);
 363                         c = (c << 8) | fgetc(tf);
 364                 }
 365                 new_field(line_count(&in_line));
 366                 in_field->len = c;
 367                 while (c--) {
 368                         int x = fgetc(tf);
 369                         if (x < 0) {
 370                                 warn(in_format, "Truncated temporary file");
 371                                 return 0;
 372                         }
 373                         *line_push(&in_line) = x;
 374                 }
 375         }
 376 }
 377
 378 static void tmp_write(void)
 379 {
 380         FILE *tf = out_format->tmp_file;
 381
 382         for (int i = 0; i < fields_count(&in_fields); i++) {
 383                 struct field *f = fields_nth(&in_fields, i);
 384                 if (f->len < 0xfe)
 385                         fputc(f->len, tf);
 386                 else {
 387                         fputc(0xfe, tf);
 388                         fputc((f->len >> 24) & 0xff, tf);
 389                         fputc((f->len >> 16) & 0xff, tf);
 390                         fputc((f->len >> 8) & 0xff, tf);
 391                         fputc(f->len & 0xff, tf);
 392                 }
 393
 394                 unsigned char *p = line_nth(&in_line, f->start_pos);
 395                 for (int j = 0; j < f->len; j++)
 396                         fputc(*p++, tf);
 397
 398                 intarray_t *w = &out_format->column_widths;
 399                 while (i >= intarray_count(w))
 400                         *intarray_push(w) = 0;
 401                 if (*intarray_nth(w, i) < f->len)
 402                         *intarray_nth(w, i) = f->len;
 403         }
 404         fputc(0xff, tf);
 405 }
 406
 407 /*** Transforms ***/
 408
 409 static void trim_fields(void)
 410 {
 411         unsigned char *line = line_first(&in_line);
 412         for (int i = 0; i < fields_count(&in_fields); i++) {
 413                 struct field *f = fields_nth(&in_fields, i);
 414                 while (f->len && is_ws(line[f->start_pos]))
 415                         f->start_pos++, f->len--;
 416                 while (f->len && is_ws(line[f->start_pos + f->len - 1]))
 417                         f->len--;
 418         }
 419 }
 420
 421 /*** Field selection ***/
 422
 423 struct selector {
 424         int first_field, last_field;
 425 };
 426
 427 DECLARE_BUF(selectors, struct selector);
 428 static selectors_t selectors;
 429
 430 static char *parse_selector(char *str)
 431 {
 432         char buf[strlen(str) + 1];
 433         strcpy(buf, str);
 434
 435         struct selector *s = selectors_push(&selectors);
 436         char *sep = strchr(buf, '-');
 437         if (sep) {
 438                 *sep++ = 0;
 439                 s->first_field = atoi(buf);
 440                 s->last_field = atoi(sep);
 441         } else
 442                 s->first_field = s->last_field = atoi(buf);
 443
 444         return NULL;
 445 }
 446
 447 static void finish_parse_selectors(void)
 448 {
 449         if (!selectors_count(&selectors))
 450                 parse_selector("-");
 451 }
 452
 453 static void select_fields(void)
 454 {
 455         for (int i = 0; i < selectors_count(&selectors); i++) {
 456                 struct selector *s = selectors_nth(&selectors, i);
 457                 int first = s->first_field;
 458                 if (first <= 0)
 459                         first = 1;
 460                 int last = s->last_field;
 461                 if (last <= 0)
 462                         last = fields_count(&in_fields);
 463                 for (int j = first; j <= last; j++) {
 464                         struct field *f = fields_push(&out_fields);
 465                         if (j >= 1 && j <= fields_count(&in_fields))
 466                                 *f = *fields_nth(&in_fields, j-1);
 467                         else
 468                                 f->start_pos = f->len = 0;
 469                 }
 470         }
 471 }
 472
 473 /*** Processing of files ***/
 474
 475 static void one_pass(void)
 476 {
 477         line_number = 0;
 478         for (;;) {
 479                 line_number++;
 480                 fields_reset(&in_fields);
 481                 line_reset(&in_line);
 482                 in_field = NULL;
 483                 if (!in_format->read_line())
 484                         break;
 485
 486                 if (want_trim)
 487                         trim_fields();
 488
 489                 fields_reset(&out_fields);
 490                 select_fields();
 491
 492                 out_format->write_line();
 493         }
 494 }
 495
 496 static void two_pass(void)
 497 {
 498         struct format *final_format = out_format;
 499
 500         // Pass 1: Set up writer of intermediate format
 501         out_format = xmalloc_zero(sizeof(*out_format));
 502         out_format->id = FORM_TMP;
 503         out_format->read_line = tmp_read;
 504         out_format->write_line = tmp_write;
 505         out_format->tmp_file = tmpfile();
 506         intarray_init(&out_format->column_widths);
 507         one_pass();
 508
 509         // Pass 2: Set up reader of intermediate format
 510         in_format = out_format;
 511         rewind(in_format->tmp_file);
 512         out_format = final_format;
 513         one_pass();
 514         fclose(in_format->tmp_file);
 515 }
 516
 517 /*** Parsing of arguments ***/
 518
 519 static void usage(void)
 520 {
 521         printf("\
 522 Usage: xsv <in-format> [<out-format>] <options> [<fields>]\n\
 523 \n\
 524 Formats:\n\
 525 -t, --tsv               TAB-separated values (default)\n\
 526 -c, --csv               Comma-separated values\n\
 527 -w, --ws                Values separated by arbitrary whitespace\n\
 528 -W, --strict-ws         Like --ws, but recognize empty columns at start/end\n\
 529 -r, --regex=<rx>        Separator given by Perl regular expression (input only)\n\
 530     --table             Format a table (output only)\n\
 531 \n\
 532 Format parameters:\n\
 533 -d, --fs=<char>         Delimiter of fields\n\
 534 -q, --quiet             Do not show warnings\n\
 535     --always-quote      Put quotes around all fields (CSV output only)\n\
 536     --table-sep=<n>     Separate table columns by <n> spaces (default: 2)\n\
 537 \n\
 538 Other options:\n\
 539     --trim              Trim leading and trailing whitespaces in fields\n\
 540 ");
 541         exit(0);
 542 }
 543
 544 static void bad_args(const char *msg, ...)
 545 {
 546         if (msg) {
 547                 va_list args;
 548                 va_start(args, msg);
 549                 fprintf(stderr, "xsv: ");
 550                 vfprintf(stderr, msg, args);
 551                 fputc('\n', stderr);
 552                 va_end(args);
 553         }
 554         fprintf(stderr, "Try `xsv --help' for more information.\n");
 555         exit(1);
 556 }
 557
 558 static const char short_options[] = "cd:qr:twW";
 559
 560 enum long_options {
 561         OPT_HELP = 256,
 562         OPT_TRIM,
 563         OPT_ALWAYS_QUOTE,
 564         OPT_TABLE,
 565         OPT_TABLE_SEP,
 566 };
 567
 568 static const struct option long_options[] = {
 569         { "always-quote",       0,      NULL,   OPT_ALWAYS_QUOTE },
 570         { "csv",                0,      NULL,   'c' },
 571         { "fs",                 1,      NULL,   'd' },
 572         { "quiet",              0,      NULL,   'q' },
 573         { "regex",              1,      NULL,   'r' },
 574         { "strict-ws",          0,      NULL,   'W' },
 575         { "table",              0,      NULL,   OPT_TABLE },
 576         { "table-sep",          1,      NULL,   OPT_TABLE_SEP },
 577         { "trim",               0,      NULL,   OPT_TRIM },
 578         { "tsv",                0,      NULL,   't' },
 579         { "ws",                 0,      NULL,   'w' },
 580         { "help",               0,      NULL,   OPT_HELP },
 581         { NULL,                 0,      NULL,   0 },
 582 };
 583
 584 static void set_format(int format_id)
 585 {
 586         struct format *f = xmalloc_zero(sizeof(*f));
 587         f->id = format_id;
 588
 589         switch (format_id) {
 590                 case FORM_TSV:
 591                         f->fs = '\t';
 592                         f->quote = -1;
 593                         f->read_line = csv_read;
 594                         f->write_line = csv_write;
 595                         break;
 596                 case FORM_CSV:
 597                         f->fs = ',';
 598                         f->quote = '"';
 599                         f->read_line = csv_read;
 600                         f->write_line = csv_write;
 601                         break;
 602                 case FORM_WS:
 603                         f->fs = ' ';
 604                         f->quote = -1;
 605                         f->read_line = ws_read;
 606                         f->write_line = csv_write;
 607                         break;
 608                 case FORM_REGEX:
 609                         f->read_line = regex_read;
 610                         break;
 611                 case FORM_TABLE:
 612                         f->write_line = table_write;
 613                         f->needs_two_passes = 1;
 614                         f->table_sep = 2;
 615                         break;
 616         }
 617
 618         if (!in_format)
 619                 in_format = f;
 620         else if (!out_format)
 621                 out_format = f;
 622         else
 623                 bad_args("At most two formats may be given.");
 624 }
 625
 626 static struct format *current_format(void)
 627 {
 628         if (out_format)
 629                 return out_format;
 630         if (in_format)
 631                 return in_format;
 632         set_format(FORM_TSV);
 633         return in_format;
 634 }
 635
 636 int main(int argc, char **argv)
 637 {
 638         int opt;
 639         const char *err;
 640
 641         while ((opt = getopt_long(argc, argv, short_options, long_options, NULL)) >= 0)
 642                 switch (opt) {
 643                         case 'c':
 644                                 set_format(FORM_CSV);
 645                                 break;
 646                         case 'd':
 647                                 if (optarg[0])
 648                                         current_format()->fs = optarg[0];
 649                                 else
 650                                         bad_args("No field delimiter given.");
 651                                 break;
 652                         case 'q':
 653                                 current_format()->quiet = 1;
 654                                 break;
 655                         case 'r':
 656                                 set_format(FORM_REGEX);
 657                                 err = regex_set(current_format(), optarg);
 658                                 if (err)
 659                                         bad_args("Error compiling regex: %s", err);
 660                                 break;
 661                         case 't':
 662                                 set_format(FORM_TSV);
 663                                 break;
 664                         case 'w':
 665                                 set_format(FORM_WS);
 666                                 break;
 667                         case 'W':
 668                                 set_format(FORM_WS);
 669                                 current_format()->strict_ws = 1;
 670                                 break;
 671                         case OPT_ALWAYS_QUOTE:
 672                                 if (current_format()->id != FORM_CSV)
 673                                         bad_args("--always-quote makes sense only for CSV.");
 674                                 current_format()->always_quote = 1;
 675                                 break;
 676                         case OPT_HELP:
 677                                 usage();
 678                         case OPT_TRIM:
 679                                 want_trim = 1;
 680                                 break;
 681                         case OPT_TABLE:
 682                                 set_format(FORM_TABLE);
 683                                 break;
 684                         case OPT_TABLE_SEP:
 685                                 current_format()->table_sep = atoi(optarg);
 686                                 break;
 687                         default:
 688                                 bad_args(NULL);
 689                 }
 690
 691         current_format();
 692         if (!out_format)
 693                 out_format = in_format;
 694         if (!in_format->read_line)
 695                 bad_args("Write-only format selected for input.");
 696         if (!out_format->write_line)
 697                 bad_args("Read-only format selected for output.");
 698
 699         for (int i = optind; i < argc; i++) {
 700                 err = parse_selector(argv[i]);
 701                 if (err)
 702                         bad_args(err);
 703         }
 704         finish_parse_selectors();
 705
 706         fields_init(&in_fields);
 707         fields_init(&out_fields);
 708         line_init(&in_line);
 709
 710         if (out_format->needs_two_passes)
 711                 two_pass();
 712         else
 713                 one_pass();
 714         return 0;
 715 }