X-Git-Url: http://mj.ucw.cz/gitweb/?a=blobdiff_plain;f=lib%2Funicode.h;h=9a3fe07a27ea2bd7785b2f55c440145fd7d7b266;hb=c4bf633211b0424492b5a3937d6a6d2e0d79a4cf;hp=685e9f81a09646ac0e42d75c5abcc26da72981f4;hpb=69808c861b8df0745bd0bc93793f558e0055da7e;p=libucw.git

diff --git a/lib/unicode.h b/lib/unicode.h
index 685e9f81..9a3fe07a 100644
--- a/lib/unicode.h
+++ b/lib/unicode.h
@@ -3,6 +3,7 @@
  *
  *	(c) 1997--2004 Martin Mares <mj@ucw.cz>
  *	(c) 2004 Robert Spalek <robert@ucw.cz>
+ *	(c) 2007 Pavel Charvat <pchar@ucw.cz>
  *
  *	This software may be freely distributed and used according to the terms
  *	of the GNU Lesser General Public License.
@@ -11,10 +12,14 @@
 #ifndef _UCW_UNICODE_H
 #define _UCW_UNICODE_H
 
+#include "lib/unaligned.h"
+
 /* Macros for handling UTF-8 */
 
 #define UNI_REPLACEMENT 0xfffc
 
+/* Encode a character from the basic multilingual plane [0, 0xFFFF]
+ * (subset of Unicode 4.0); up to 3 bytes needed (RFC2279) */
 static inline byte *
 utf8_put(byte *p, uns u)
 {
@@ -35,6 +40,8 @@ utf8_put(byte *p, uns u)
   return p;
 }
 
+/* Encode a value from the range [0, 0x7FFFFFFF];
+ * (superset of Unicode 4.0) up to 6 bytes needed (RFC2279) */
 static inline byte *
 utf8_32_put(byte *p, uns u)
 {
@@ -76,8 +83,10 @@ put1: *p++ = 0x80 | (u & 0x3f);
 
 #define UTF8_GET_NEXT if (unlikely((*p & 0xc0) != 0x80)) goto bad; u = (u << 6) | (*p++ & 0x3f)
 
+/* Decode a character from the basic multilingual plane [0, 0xFFFF]
+ * or return 'repl' if the encoding has been corrupted */
 static inline byte *
-utf8_get(const byte *p, uns *uu)
+utf8_get_repl(const byte *p, uns *uu, uns repl)
 {
   uns u = *p++;
   if (u < 0x80)
@@ -86,7 +95,7 @@ utf8_get(const byte *p, uns *uu)
     {
       /* Incorrect byte sequence */
     bad:
-      u = UNI_REPLACEMENT;
+      u = repl;
     }
   else if (u < 0xe0)
     {
@@ -105,8 +114,10 @@ utf8_get(const byte *p, uns *uu)
   return (byte *)p;
 }
 
+/* Decode a value from the range [0, 0x7FFFFFFF] 
+ * or return 'repl' if the encoding has been corrupted */
 static inline byte *
-utf8_32_get(const byte *p, uns *uu)
+utf8_32_get_repl(const byte *p, uns *uu, uns repl)
 {
   uns u = *p++;
   if (u < 0x80)
@@ -115,7 +126,7 @@ utf8_32_get(const byte *p, uns *uu)
     {
       /* Incorrect byte sequence */
     bad:
-      u = UNI_REPLACEMENT;
+      u = repl;
     }
   else if (u < 0xe0)
     {
@@ -152,6 +163,22 @@ get1: UTF8_GET_NEXT;
   return (byte *)p;
 }
 
+/* Decode a character from the basic multilingual plane [0, 0xFFFF]
+ * or return UNI_REPLACEMENT if the encoding has been corrupted */
+static inline byte *
+utf8_get(const byte *p, uns *uu)
+{
+  return utf8_get_repl(p, uu, UNI_REPLACEMENT);
+}
+
+/* Decode a value from the range [0, 0x7FFFFFFF] 
+ * or return UNI_REPLACEMENT if the encoding has been corrupted */
+static inline byte *
+utf8_32_get(const byte *p, uns *uu)
+{
+  return utf8_32_get_repl(p, uu, UNI_REPLACEMENT);
+}
+
 #define PUT_UTF8(p,u) p = utf8_put(p, u)
 #define GET_UTF8(p,u) p = (byte*)utf8_get(p, &(u))
 
@@ -200,10 +227,108 @@ utf8_encoding_len(uns c)
   return 6;
 }
 
+/* Encode a character from the range [0, 0xD7FF] or [0xE000,0x11FFFF];
+ * up to 4 bytes needed */
+static inline void *
+utf16_le_put(void *p, uns u)
+{
+  if (u < 0xd800 || (u < 0x10000 && u >= 0xe000))
+    {
+      put_u16_le(p, u);
+      return p + 2;
+    }
+  else if ((u -= 0x10000) < 0x100000)
+    {
+      put_u16_le(p, 0xd800 | (u >> 10));
+      put_u16_le(p + 2, 0xdc00 | (u & 0x3ff));
+      return p + 4;
+    }
+  else
+    ASSERT(0);
+}
+
+static inline void *
+utf16_be_put(void *p, uns u)
+{
+  if (u < 0xd800 || (u < 0x10000 && u >= 0xe000))
+    {
+      put_u16_be(p, u);
+      return p + 2;
+    }
+  else if ((u -= 0x10000) < 0x100000)
+    {
+      put_u16_be(p, 0xd800 | (u >> 10));
+      put_u16_be(p + 2, 0xdc00 | (u & 0x3ff));
+      return p + 4;
+    }
+  else
+    ASSERT(0);
+}
+
+/* Decode a character from the range [0, 0xD7FF] or [0xE000,11FFFF]
+ * or return `repl' if the encoding has been corrupted */
+static inline void *
+utf16_le_get_repl(const void *p, uns *uu, uns repl)
+{
+  uns u = get_u16_le(p), x, y;
+  x = u - 0xd800;
+  if (x < 0x800)
+    if (x < 0x400 && (y = get_u16_le(p + 2) - 0xdc00) < 0x400)
+      {
+	u = 0x10000 + (x << 10) + y;
+	p += 2;
+      }
+    else
+      u = repl;
+  *uu = u;
+  return (void *)(p + 2);
+}
+
+static inline void *
+utf16_be_get_repl(const void *p, uns *uu, uns repl)
+{
+  uns u = get_u16_be(p), x, y;
+  x = u - 0xd800;
+  if (x < 0x800)
+    if (x < 0x400 && (y = get_u16_be(p + 2) - 0xdc00) < 0x400)
+      {
+	u = 0x10000 + (x << 10) + y;
+	p += 2;
+      }
+    else
+      u = repl;
+  *uu = u;
+  return (void *)(p + 2);
+}
+
+/* Decode a character from the range [0, 0xD7FF] or [0xE000,11FFFF]
+ * or return UNI_REPLACEMENT if the encoding has been corrupted */
+static inline void *
+utf16_le_get(const void *p, uns *uu)
+{
+  return utf16_le_get_repl(p, uu, UNI_REPLACEMENT);
+}
+
+static inline void *
+utf16_be_get(const void *p, uns *uu)
+{
+  return utf16_be_get_repl(p, uu, UNI_REPLACEMENT);
+}
+
+static inline uns
+unicode_sanitize_char(uns u)
+{
+  if (u >= 0x10000 ||			// We don't accept anything outside the basic plane
+      u >= 0xd800 && u < 0xf900 ||	// neither we do surrogates
+      u >= 0x80 && u < 0xa0 ||		// nor latin-1 control chars
+      u < 0x20 && u != '\t')
+    return UNI_REPLACEMENT;
+  return u;
+}
+
 /* unicode-utf8.c */
 
 uns utf8_strlen(const byte *str);
 uns utf8_strnlen(const byte *str, uns n);
-uns utf8_check(const byte *str);
 
 #endif