Renamed charset import scripts.

[libucw.git] / lib / url.c

diff --git a/lib/url.c b/lib/url.c
index 480ba1807ed5b69f756209511196da123818643f..d98c3a79832edda5e780ab03445a2b8fb9ffce35 100644 (file)
--- a/lib/url.c
+++ b/lib/url.c
@@ -1,16 +1,45 @@
 /*
  *     Sherlock Library -- URL Functions (according to RFC 1738 and 1808)
  *
 /*
  *     Sherlock Library -- URL Functions (according to RFC 1738 and 1808)
  *

- *     (c) 1997--1999 Martin Mares, <mj@atrey.karlin.mff.cuni.cz>
+ *     (c) 1997--2002 Martin Mares <mj@ucw.cz>
+ *     (c) 2001 Robert Spalek <robert@ucw.cz>
+ *
+ *     This software may be freely distributed and used according to the terms
+ *     of the GNU Lesser General Public License.

  */
 
  */
 

+#include "lib/lib.h"
+#include "lib/url.h"
+#include "lib/chartype.h"
+#include "lib/conf.h"
+

 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>

+#include <alloca.h>
+
+/* Configuration */
+
+static uns url_ignore_spaces;
+static uns url_ignore_underflow;
+static byte *url_component_separators = "";
+static uns url_min_repeat_count = 0x7fffffff;
+static uns url_max_repeat_length = 0;
+
+static struct cfitem url_config[] = {
+  { "URL",             CT_SECTION,     NULL },
+  { "IgnoreSpaces",    CT_INT,         &url_ignore_spaces },
+  { "IgnoreUnderflow", CT_INT,         &url_ignore_underflow },
+  { "ComponentSeparators",     CT_STRING,      &url_component_separators },
+  { "MinRepeatCount",          CT_INT,         &url_min_repeat_count },
+  { "MaxRepeatLength",         CT_INT,         &url_max_repeat_length },
+  { NULL,              CT_STOP,        NULL }
+};

 
 

-#include "lib.h"
-#include "url.h"
-#include "string.h"
+static void CONSTRUCTOR url_init_config(void)
+{
+  cf_register(url_config);
+}

 
 /* Escaping and de-escaping */
 
 
 /* Escaping and de-escaping */
 


@@ -23,6 +52,7 @@ enhex(uns x)
 int
 url_deescape(byte *s, byte *d)
 {
 int
 url_deescape(byte *s, byte *d)
 {

+  byte *dstart = d;

   byte *end = d + MAX_URL_SIZE - 10;
   while (*s)
     {
   byte *end = d + MAX_URL_SIZE - 10;
   while (*s)
     {


@@ -58,8 +88,23 @@ url_deescape(byte *s, byte *d)
          *d++ = val;
          s += 3;
        }
          *d++ = val;
          s += 3;
        }

-      else if (*s >= 0x20)
+      else if (*s > 0x20)

        *d++ = *s++;
        *d++ = *s++;

+      else if (Cspace(*s))
+       {
+         byte *s0 = s;
+         while (Cspace(*s))
+           s++;
+         if (!url_ignore_spaces || !(!*s || d == dstart))
+           {
+             while (Cspace(*s0))
+               {
+                 if (d >= end)
+                   return URL_ERR_TOO_LONG;
+                 *d++ = *s0++;
+               }
+           }
+       }

       else
        return URL_ERR_INVALID_CHAR;
     }
       else
        return URL_ERR_INVALID_CHAR;
     }


@@ -162,6 +207,11 @@ url_split(byte *s, struct url *u, byte *d)
            {
              *w++ = 0;
              u->user = q;
            {
              *w++ = 0;
              u->user = q;

+             if (e = strchr(q, ':'))
+               {
+                 *e++ = 0;
+                 u->pass = e;
+               }

            }
          else
            w = q;
            }
          else
            w = q;


@@ -256,17 +306,23 @@ relpath_merge(struct url *u, struct url *b)
          else if (a[1] == '.' && (a[2] == '/' || !a[2])) /* "../" */
            {
              a += 2;
          else if (a[1] == '.' && (a[2] == '/' || !a[2])) /* "../" */
            {
              a += 2;

-             if (d <= u->buf + 1)
-               /*
-                * RFC 1808 says we should leave ".." as a path segment, but
-                * we intentionally break the rule and refuse the URL.
-                */
-               return URL_PATH_UNDERFLOW;
-             d--;                      /* Discard trailing slash */
-             while (d[-1] != '/')
-               d--;

              if (a[0])
                a++;
              if (a[0])
                a++;

+             if (d <= u->buf + 1)
+               {
+                 /*
+                  * RFC 1808 says we should leave ".." as a path segment, but
+                  * we intentionally break the rule and refuse the URL.
+                  */
+                 if (!url_ignore_underflow)
+                   return URL_PATH_UNDERFLOW;
+               }
+             else
+               {
+                 d--;                  /* Discard trailing slash */
+                 while (d[-1] != '/')
+                   d--;
+               }

              continue;
            }
        }
              continue;
            }
        }


@@ -308,6 +364,7 @@ url_normalize(struct url *u, struct url *b)
   if (url_proto_path_flags[u->protoid] && !u->host ||
       u->host && !*u->host ||
       !u->host && u->user ||
   if (url_proto_path_flags[u->protoid] && !u->host ||
       u->host && !*u->host ||
       !u->host && u->user ||

+      !u->user && u->pass ||

       !u->rest)
     return URL_SYNTAX_ERROR;
 
       !u->rest)
     return URL_SYNTAX_ERROR;
 


@@ -324,6 +381,7 @@ url_normalize(struct url *u, struct url *b)
        {
          u->host = b->host;
          u->user = b->user;
        {
          u->host = b->host;
          u->user = b->user;

+         u->pass = b->pass;

          u->port = b->port;
          if (err = relpath_merge(u, b))
            return err;
          u->port = b->port;
          if (err = relpath_merge(u, b))
            return err;


@@ -359,8 +417,8 @@ kill_end_dot(byte *b)
   if (b)
     {
       k = b + strlen(b) - 1;
   if (b)
     {
       k = b + strlen(b) - 1;

-      if (k > b && *k == '.')
-       *k = 0;
+      while (k > b && *k == '.')
+       *k-- = 0;

     }
 }
 
     }
 }
 


@@ -411,6 +469,11 @@ url_pack(struct url *u, byte *d)
       if (u->user)
        {
          d = append(d, u->user, e);
       if (u->user)
        {
          d = append(d, u->user, e);

+         if (u->pass)
+           {
+             d = append(d, ":", e);
+             d = append(d, u->pass, e);
+           }

          d = append(d, "@", e);
        }
       d = append(d, u->host, e);
          d = append(d, "@", e);
        }
       d = append(d, u->host, e);


@@ -453,7 +516,7 @@ url_error(uns err)
   return errmsg[err];
 }
 
   return errmsg[err];
 }
 

-/* A "macro" for canonical split */
+/* Standard cookbook recipes */

 
 int
 url_canon_split(byte *u, byte *buf1, byte *buf2, struct url *url)
 
 int
 url_canon_split(byte *u, byte *buf1, byte *buf2, struct url *url)


@@ -469,6 +532,19 @@ url_canon_split(byte *u, byte *buf1, byte *buf2, struct url *url)
   return url_canonicalize(url);
 }
 
   return url_canonicalize(url);
 }
 

+int
+url_auto_canonicalize(byte *src, byte *dst)
+{
+  byte buf1[MAX_URL_SIZE], buf2[MAX_URL_SIZE], buf3[MAX_URL_SIZE];
+  int err;
+  struct url ur;
+
+  (void)((err = url_canon_split(src, buf1, buf2, &ur)) ||
+   (err = url_pack(&ur, buf3)) ||
+   (err = url_enescape(buf3, dst)));
+  return err;
+}
+

 /* Testing */
 
 #ifdef TEST
 /* Testing */
 
 #ifdef TEST


@@ -492,7 +568,7 @@ int main(int argc, char **argv)
       printf("split: error %d\n", err);
       return 1;
     }
       printf("split: error %d\n", err);
       return 1;
     }

-  printf("split: @%s@%s@%s@%d@%s\n", url.protocol, url.user, url.host, url.port, url.rest);
+  printf("split: @%s@%s@%s@%s@%d@%s\n", url.protocol, url.user, url.pass, url.host, url.port, url.rest);

   if (err = url_split("http://mj@www.hell.org/123/sub_dir/index.html;param?query&zzz/subquery#fragment", &url0, buf3))
     {
       printf("split base: error %d\n", err);
   if (err = url_split("http://mj@www.hell.org/123/sub_dir/index.html;param?query&zzz/subquery#fragment", &url0, buf3))
     {
       printf("split base: error %d\n", err);


@@ -503,19 +579,19 @@ int main(int argc, char **argv)
       printf("normalize base: error %d\n", err);
       return 1;
     }
       printf("normalize base: error %d\n", err);
       return 1;
     }

-  printf("base: @%s@%s@%s@%d@%s\n", url0.protocol, url0.user, url0.host, url0.port, url0.rest);
+  printf("base: @%s@%s@%s@%s@%d@%s\n", url0.protocol, url0.user, url0.pass, url0.host, url0.port, url0.rest);

   if (err = url_normalize(&url, &url0))
     {
       printf("normalize: error %d\n", err);
       return 1;
     }
   if (err = url_normalize(&url, &url0))
     {
       printf("normalize: error %d\n", err);
       return 1;
     }

-  printf("normalize: @%s@%s@%s@%d@%s\n", url.protocol, url.user, url.host, url.port, url.rest);
+  printf("normalize: @%s@%s@%s@%s@%d@%s\n", url.protocol, url.user, url.pass, url.host, url.port, url.rest);

   if (err = url_canonicalize(&url))
     {
       printf("canonicalize: error %d\n", err);
       return 1;
     }
   if (err = url_canonicalize(&url))
     {
       printf("canonicalize: error %d\n", err);
       return 1;
     }

-  printf("canonicalize: @%s@%s@%s@%d@%s\n", url.protocol, url.user, url.host, url.port, url.rest);
+  printf("canonicalize: @%s@%s@%s@%s@%d@%s\n", url.protocol, url.user, url.pass, url.host, url.port, url.rest);

   if (err = url_pack(&url, buf4))
     {
       printf("pack: error %d\n", err);
   if (err = url_pack(&url, buf4))
     {
       printf("pack: error %d\n", err);


@@ -532,3 +608,78 @@ int main(int argc, char **argv)
 }
 
 #endif
 }
 
 #endif

+
+struct component {
+       byte *start;
+       int length;
+       u32 hash;
+};
+
+static inline u32
+hashf(byte *start, int length)
+{
+       u32 hf = length;
+       while (length-- > 0)
+               hf = (hf << 8 | hf >> 24) ^ *start++;
+       return hf;
+}
+
+static inline uns
+repeat_count(struct component *comp, uns count, uns len)
+{
+       struct component *orig_comp = comp;
+       uns found = 0;
+       while (1)
+       {
+               uns i;
+               comp += len;
+               count -= len;
+               found++;
+               if (count < len)
+                       return found;
+               for (i=0; i<len; i++)
+                       if (comp[i].hash != orig_comp[i].hash
+                       || comp[i].length != orig_comp[i].length
+                       || memcmp(comp[i].start, orig_comp[i].start, comp[i].length))
+                               return found;
+       }
+}
+
+int
+url_has_repeated_component(byte *url)
+{
+       struct component *comp;
+       uns comps, comp_len, rep_prefix;
+       byte *c;
+       uns i;
+
+       for (comps=0, c=url; c; comps++)
+       {
+               c = strpbrk(c, url_component_separators);
+               if (c)
+                       c++;
+       }
+       if (comps < url_min_repeat_count)
+               return 0;
+       comp = alloca(comps * sizeof(struct component));
+       for (i=0, c=url; c; i++)
+       {
+               comp[i].start = c;
+               c = strpbrk(c, url_component_separators);
+               if (c)
+               {
+                       comp[i].length = c - comp[i].start;
+                       c++;
+               }
+               else
+                       comp[i].length = strlen(comp[i].start);
+       }
+       ASSERT(i == comps);
+       for (i=0; i<comps; i++)
+               comp[i].hash = hashf(comp[i].start, comp[i].length);
+       for (comp_len = 1; comp_len <= url_max_repeat_length && comp_len <= comps; comp_len++)
+               for (rep_prefix = 0; rep_prefix <= comps - comp_len; rep_prefix++)
+                       if (repeat_count(comp + rep_prefix, comps - rep_prefix, comp_len) >= url_min_repeat_count)
+                               return comp_len;
+       return 0;
+}