2 * UCW Library -- URL Functions
4 * (c) 1997--2004 Martin Mares <mj@ucw.cz>
5 * (c) 2001--2005 Robert Spalek <robert@ucw.cz>
7 * This software may be freely distributed and used according to the terms
8 * of the GNU Lesser General Public License.
10 * XXX: The buffer handling in this module is really horrible, but it works.
15 #include <ucw/chartype.h>
17 #include <ucw/prime.h>
26 static uint url_ignore_spaces;
27 static uint url_ignore_underflow;
28 static char *url_component_separators = "";
29 static uint url_min_repeat_count = 0x7fffffff;
30 static uint url_max_repeat_length = 0;
31 static uint url_max_occurences = ~0U;
34 static struct cf_section url_config = {
36 CF_UINT("IgnoreSpaces", &url_ignore_spaces),
37 CF_UINT("IgnoreUnderflow", &url_ignore_underflow),
38 CF_STRING("ComponentSeparators", &url_component_separators),
39 CF_UINT("MinRepeatCount", &url_min_repeat_count),
40 CF_UINT("MaxRepeatLength", &url_max_repeat_length),
41 CF_UINT("MaxOccurences", &url_max_occurences),
46 static void CONSTRUCTOR url_init_config(void)
48 cf_declare_section("URL", &url_config, 0);
52 /* Escaping and de-escaping */
57 return (x<10) ? (x + '0') : (x - 10 + 'A');
61 url_deescape(const char *s, char *d)
64 char *end = d + MAX_URL_SIZE - 10;
68 return URL_ERR_TOO_LONG;
72 if (!Cxdigit(s[1]) || !Cxdigit(s[2]))
73 return URL_ERR_INVALID_ESCAPE;
74 val = Cxvalue(s[1])*16 + Cxvalue(s[2]);
76 return URL_ERR_INVALID_ESCAPED_CHAR;
80 val = NCC_SEMICOLON; break;
82 val = NCC_SLASH; break;
84 val = NCC_QUEST; break;
86 val = NCC_COLON; break;
90 val = NCC_EQUAL; break;
94 val = NCC_HASH; break;
96 val = NCC_DOLLAR; break;
98 val = NCC_PLUS; break;
100 val = NCC_COMMA; break;
105 else if ((byte) *s > 0x20)
112 if (!url_ignore_spaces || !(!*s || d == dstart))
117 return URL_ERR_TOO_LONG;
123 return URL_ERR_INVALID_CHAR;
130 url_enescape(const char *s, char *d)
132 char *end = d + MAX_URL_SIZE - 10;
138 return URL_ERR_TOO_LONG;
139 if (Calnum(c) || /* RFC 2396 (2.1-2.3): Only alphanumerics ... */
140 c == '!' || c == '*' || c == '\'' || c == '(' || c == ')' || /* ... and some exceptions and reserved chars */
141 c == '$' || c == '-' || c == '_' || c == '.' || c == '+' ||
142 c == ',' || c == '=' || c == '&' || c == '#' || c == ';' ||
143 c == '/' || c == '?' || c == ':' || c == '@' || c == '~'
148 uint val = (byte)(((byte)*s < NCC_MAX) ? NCC_CHARS[(byte)*s] : *s);
150 *d++ = enhex(val >> 4);
151 *d++ = enhex(val & 0x0f);
160 url_enescape_friendly(const char *src, char *dest)
162 char *end = dest + MAX_URL_SIZE - 10;
163 const byte *srcb = src;
167 return URL_ERR_TOO_LONG;
168 if ((byte)*srcb < NCC_MAX)
169 *dest++ = NCC_CHARS[*srcb++];
170 else if (*srcb >= 0x20 && *srcb < 0x7f)
175 *dest++ = enhex((byte)*srcb >> 4);
176 *dest++ = enhex(*srcb++ & 0x0f);
183 /* Split an URL (several parts may be copied to the destination buffer) */
185 char *url_proto_names[URL_PROTO_MAX] = URL_PNAMES;
186 static int url_proto_path_flags[URL_PROTO_MAX] = URL_PATH_FLAGS;
189 url_identify_protocol(const char *p)
193 for(i=1; i<URL_PROTO_MAX; i++)
194 if (!strcasecmp(p, url_proto_names[i]))
196 return URL_PROTO_UNKNOWN;
200 url_split(char *s, struct url *u, char *d)
202 bzero(u, sizeof(struct url));
204 u->bufend = d + MAX_URL_SIZE - 10;
206 if (s[0] != '/') /* Seek for "protocol:" */
209 while (*p && Calnum(*p))
211 if (p != s && *p == ':')
217 u->protoid = url_identify_protocol(u->protocol);
219 if (url_proto_path_flags[u->protoid] && (s[0] != '/' || s[1] != '/'))
221 /* The protocol requires complete host spec, but it's missing -> treat as a relative path instead */
222 int len = d - u->protocol;
231 if (s[0] == '/') /* Host spec or absolute path */
233 if (s[1] == '/') /* Host spec */
241 while (*s && *s != '/' && *s != '?') /* Copy user:passwd@host:port */
250 else /* This shouldn't happen with sane URL's, but we need to be sure */
255 if (at) /* user:passwd present */
258 if (e = strchr(q, ':'))
267 if (e) /* host:port present */
271 p = strtoul(e, &ep, 10);
272 if (ep && *ep || p > 65535)
273 return URL_ERR_INVALID_PORT;
274 else if (p) /* Port 0 (e.g. in :/) is treated as default port */
286 /* Normalization according to given base URL */
288 static uint std_ports[] = URL_DEFPORTS; /* Default port numbers */
291 relpath_merge(struct url *u, struct url *b)
299 if (a[0] == '/') /* Absolute path => OK */
301 if (o[0] != '/' && o[0] != '?')
302 return URL_PATH_UNDERFLOW;
304 if (!a[0]) /* Empty URL -> inherit everything */
310 u->rest = d; /* We know we'll need to copy the path somewhere else */
312 if (a[0] == '#') /* Another fragment */
314 for(p=o; *p && *p != '#'; p++)
318 if (a[0] == '?') /* New query */
320 for(p=o; *p && *p != '#' && *p != '?'; p++)
325 p = NULL; /* Copy original path and find the last slash */
326 while (*o && *o != '?' && *o != '#')
329 return URL_ERR_TOO_LONG;
330 if ((*d++ = *o++) == '/')
334 return URL_ERR_REL_NOTHING;
341 if (a[1] == '/' || !a[1]) /* Skip "./" and ".$" */
348 else if (a[1] == '.' && (a[2] == '/' || !a[2])) /* "../" */
356 * RFC 1808 says we should leave ".." as a path segment, but
357 * we intentionally break the rule and refuse the URL.
359 if (!url_ignore_underflow)
360 return URL_PATH_UNDERFLOW;
364 d--; /* Discard trailing slash */
371 while (a[0] && a[0] != '/')
374 return URL_ERR_TOO_LONG;
386 copy: /* Combine part of old URL with the new one */
391 return URL_ERR_TOO_LONG;
396 return URL_ERR_TOO_LONG;
401 url_normalize(struct url *u, struct url *b)
406 if (url_proto_path_flags[u->protoid] && (!u->host || !*u->host) ||
407 !u->host && u->user ||
408 !u->user && u->pass ||
410 return URL_SYNTAX_ERROR;
414 /* Now we know it's a relative URL. Do we have any base? */
415 if (!b || !url_proto_path_flags[b->protoid])
416 return URL_ERR_REL_NOTHING;
417 u->protocol = b->protocol;
418 u->protoid = b->protoid;
420 /* Reference to the same host */
427 if (err = relpath_merge(u, b))
432 /* Change path "?" to "/?" because it's the true meaning */
433 if (u->rest[0] == '?')
435 int l = strlen(u->rest);
436 if (u->bufend - u->buf < l+1)
437 return URL_ERR_TOO_LONG;
439 memcpy(u->buf+1, u->rest, l+1);
444 /* Fill in missing info */
446 u->port = std_ports[u->protoid];
451 /* Name canonicalization */
459 if (*b >= 'A' && *b <= 'Z')
466 kill_end_dot(char *b)
472 k = b + strlen(b) - 1;
473 while (k > b && *k == '.')
479 url_canonicalize(struct url *u)
483 lowercase(u->protocol);
485 kill_end_dot(u->host);
486 if ((!u->rest || !*u->rest) && url_proto_path_flags[u->protoid])
488 if (u->rest && (c = strchr(u->rest, '#'))) /* Kill fragment reference */
493 /* Pack a broken-down URL */
496 append(char *d, const char *s, char *e)
509 url_pack(struct url *u, char *d)
511 char *e = d + MAX_URL_SIZE - 10;
515 d = append(d, u->protocol, e);
516 d = append(d, ":", e);
517 u->protoid = url_identify_protocol(u->protocol);
521 d = append(d, "//", e);
524 d = append(d, u->user, e);
527 d = append(d, ":", e);
528 d = append(d, u->pass, e);
530 d = append(d, "@", e);
532 d = append(d, u->host, e);
533 if (u->port != std_ports[u->protoid] && u->port != ~0U)
536 sprintf(z, "%d", u->port);
537 d = append(d, ":", e);
542 d = append(d, u->rest, e);
544 return URL_ERR_TOO_LONG;
551 static char *errmsg[] = {
552 "Something is wrong",
556 "Invalid escaped character",
557 "Invalid port number",
558 "Relative URL not allowed",
567 if (err >= sizeof(errmsg) / sizeof(char *))
572 /* Standard cookbook recipes */
575 url_canon_split_rel(const char *u, char *buf1, char *buf2, struct url *url, struct url *base)
579 if (err = url_deescape(u, buf1))
581 if (err = url_split(buf1, url, buf2))
583 if (err = url_normalize(url, base))
585 return url_canonicalize(url);
589 url_auto_canonicalize_rel(const char *src, char *dst, struct url *base)
591 char buf1[MAX_URL_SIZE], buf2[MAX_URL_SIZE], buf3[MAX_URL_SIZE];
595 (void)((err = url_canon_split_rel(src, buf1, buf2, &ur, base)) ||
596 (err = url_pack(&ur, buf3)) ||
597 (err = url_enescape(buf3, dst)));
605 int main(int argc, char **argv)
607 char buf1[MAX_URL_SIZE], buf2[MAX_URL_SIZE], buf3[MAX_URL_SIZE], buf4[MAX_URL_SIZE];
609 struct url url, url0;
610 char *base = "http://mj@www.hell.org/123/sub_dir;param/index.html;param?query&zzz/sub;query+#fragment?";
612 if (argc != 2 && argc != 3)
616 if (err = url_deescape(argv[1], buf1))
618 printf("deesc: error %d\n", err);
621 printf("deesc: %s\n", buf1);
622 if (err = url_split(buf1, &url, buf2))
624 printf("split: error %d\n", err);
627 printf("split: @%s@%s@%s@%s@%d@%s\n", url.protocol, url.user, url.pass, url.host, url.port, url.rest);
628 if (err = url_split(base, &url0, buf3))
630 printf("split base: error %d\n", err);
633 if (err = url_normalize(&url0, NULL))
635 printf("normalize base: error %d\n", err);
638 printf("base: @%s@%s@%s@%s@%d@%s\n", url0.protocol, url0.user, url0.pass, url0.host, url0.port, url0.rest);
639 if (err = url_normalize(&url, &url0))
641 printf("normalize: error %d\n", err);
644 printf("normalize: @%s@%s@%s@%s@%d@%s\n", url.protocol, url.user, url.pass, url.host, url.port, url.rest);
645 if (err = url_canonicalize(&url))
647 printf("canonicalize: error %d\n", err);
650 printf("canonicalize: @%s@%s@%s@%s@%d@%s\n", url.protocol, url.user, url.pass, url.host, url.port, url.rest);
651 if (err = url_pack(&url, buf4))
653 printf("pack: error %d\n", err);
656 printf("pack: %s\n", buf4);
657 if (err = url_enescape(buf4, buf2))
659 printf("enesc: error %d\n", err);
662 printf("enesc: %s\n", buf2);
676 hashf(const char *start, int length)
680 hf = (hf << 8 | hf >> 24) ^ *start++;
685 repeat_count(struct component *comp, uint count, uint len)
687 struct component *orig_comp = comp;
697 for (i=0; i<len; i++)
698 if (comp[i].hash != orig_comp[i].hash
699 || comp[i].length != orig_comp[i].length
700 || memcmp(comp[i].start, orig_comp[i].start, comp[i].length))
706 url_has_repeated_component(const char *url)
708 struct component *comp;
709 uint comps, comp_len, rep_prefix, hash_size, *hash, *next;
713 for (comps=0, c=url; c; comps++)
715 c = strpbrk(c, url_component_separators);
719 if (comps < url_min_repeat_count && comps <= url_max_occurences)
721 comp = alloca(comps * sizeof(*comp));
722 for (i=0, c=url; c; i++)
725 c = strpbrk(c, url_component_separators);
728 comp[i].length = c - comp[i].start;
732 comp[i].length = strlen(comp[i].start);
735 for (i=0; i<comps; i++)
736 comp[i].hash = hashf(comp[i].start, comp[i].length);
737 if (comps > url_max_occurences)
739 hash_size = next_table_prime(comps);
740 hash = alloca(hash_size * sizeof(*hash));
741 next = alloca(comps * sizeof(*next));
742 memset(hash, 255, hash_size * sizeof(*hash));
743 for (i=0; i<comps; i++)
745 j = comp[i].hash % hash_size;
746 for (k = hash[j]; ~k && (comp[i].hash != comp[k].hash || comp[i].length != comp[k].length ||
747 memcmp(comp[k].start, comp[i].start, comp[i].length)); k = next[k]);
756 if (comp[k].count++ >= url_max_occurences)
761 for (comp_len = 1; comp_len <= url_max_repeat_length && comp_len <= comps; comp_len++)
762 for (rep_prefix = 0; rep_prefix <= comps - comp_len; rep_prefix++)
763 if (repeat_count(comp + rep_prefix, comps - rep_prefix, comp_len) >= url_min_repeat_count)