2 * UCW Library -- Unicode Characters
4 * (c) 1997--2004 Martin Mares <mj@ucw.cz>
5 * (c) 2004 Robert Spalek <robert@ucw.cz>
6 * (c) 2007 Pavel Charvat <pchar@ucw.cz>
8 * This software may be freely distributed and used according to the terms
9 * of the GNU Lesser General Public License.
12 #ifndef _UCW_UNICODE_H
13 #define _UCW_UNICODE_H
15 #include "lib/unaligned.h"
17 /* Macros for handling UTF-8 */
19 #define UNI_REPLACEMENT 0xfffc
21 /* Encode a character from the basic multilingual plane [0, 0xFFFF]
22 * (subset of Unicode 4.0); up to 3 bytes needed (RFC2279) */
24 utf8_put(byte *p, uns u)
30 *p++ = 0xc0 | (u >> 6);
31 *p++ = 0x80 | (u & 0x3f);
36 *p++ = 0xe0 | (u >> 12);
37 *p++ = 0x80 | ((u >> 6) & 0x3f);
38 *p++ = 0x80 | (u & 0x3f);
43 /* Encode a value from the range [0, 0x7FFFFFFF];
44 * (superset of Unicode 4.0) up to 6 bytes needed (RFC2279) */
46 utf8_32_put(byte *p, uns u)
52 *p++ = 0xc0 | (u >> 6);
57 *p++ = 0xe0 | (u >> 12);
62 *p++ = 0xf0 | (u >> 18);
67 *p++ = 0xf8 | (u >> 24);
70 else if (u < (1U<<31))
72 *p++ = 0xfc | (u >> 30);
73 *p++ = 0x80 | ((u >> 24) & 0x3f);
74 put4: *p++ = 0x80 | ((u >> 18) & 0x3f);
75 put3: *p++ = 0x80 | ((u >> 12) & 0x3f);
76 put2: *p++ = 0x80 | ((u >> 6) & 0x3f);
77 put1: *p++ = 0x80 | (u & 0x3f);
84 #define UTF8_GET_NEXT if (unlikely((*p & 0xc0) != 0x80)) goto bad; u = (u << 6) | (*p++ & 0x3f)
86 /* Decode a character from the basic multilingual plane [0, 0xFFFF]
87 * or return UNI_REPLACEMENT if the encoding has been corrupted */
89 utf8_get(const byte *p, uns *uu)
94 else if (unlikely(u < 0xc0))
96 /* Incorrect byte sequence */
105 else if (likely(u < 0xf0))
117 /* Decode a value from the range [0, 0x7FFFFFFF]
118 * or return UNI_REPLACEMENT if the encoding has been corrupted */
120 utf8_32_get(const byte *p, uns *uu)
125 else if (unlikely(u < 0xc0))
127 /* Incorrect byte sequence */
166 #define PUT_UTF8(p,u) p = utf8_put(p, u)
167 #define GET_UTF8(p,u) p = (byte*)utf8_get(p, &(u))
169 #define PUT_UTF8_32(p,u) p = utf8_32_put(p, u)
170 #define GET_UTF8_32(p,u) p = (byte*)utf8_32_get(p, &(u))
172 #define UTF8_SKIP(p) do { \
175 while (c & 0x40 && *p >= 0x80 && *p < 0xc0) \
179 #define UTF8_SKIP_BWD(p) while ((*--(p) & 0xc0) == 0x80)
198 utf8_encoding_len(uns c)
202 ASSERT(c >= 0xc0 && c < 0xfe);
214 /* Encode a character from the range [0, 0xD7FF] or [0xE000,0x11FFFF];
215 * up to 4 bytes needed */
217 utf16_le_put(void *p, uns u)
219 if (u < 0xd800 || (u < 0x10000 && u >= 0xe000))
224 else if ((u -= 0x10000) < 0x100000)
226 put_u16_le(p, u >> 10);
227 put_u16_le(p + 2, u & 0x7ff);
235 utf16_be_put(void *p, uns u)
237 if (u < 0xd800 || (u < 0x10000 && u >= 0xe000))
242 else if ((u -= 0x10000) < 0x100000)
244 put_u16_be(p, u >> 10);
245 put_u16_be(p + 2, u & 0x7ff);
252 /* Decode a character from the range [0, 0xD7FF] or [0xE000,11FFFF]
253 * or return `repl' if the encoding has been corrupted */
255 utf16_le_get_repl(const void *p, uns *uu, uns repl)
257 uns u = get_u16_le(p), x, y;
260 if (x < 0x400 && (y = get_u16_le(p + 2) - 0xdc00) < 0x400)
262 u = 0x10000 + (x << 10) + y;
268 return (void *)(p + 2);
272 utf16_be_get_repl(const void *p, uns *uu, uns repl)
274 uns u = get_u16_be(p), x, y;
277 if (x < 0x400 && (y = get_u16_be(p + 2) - 0xdc00) < 0x400)
279 u = 0x10000 + (x << 10) + y;
285 return (void *)(p + 2);
288 /* Decode a character from the range [0, 0xD7FF] or [0xE000,11FFFF]
289 * or return UNI_REPLACEMENT if the encoding has been corrupted */
291 utf16_le_get(const void *p, uns *uu)
293 return utf16_le_get_repl(p, uu, UNI_REPLACEMENT);
297 utf16_be_get(const void *p, uns *uu)
299 return utf16_be_get_repl(p, uu, UNI_REPLACEMENT);
303 unicode_sanitize_char(uns u)
305 if (u >= 0x10000 || // We don't accept anything outside the basic plane
306 u >= 0xd800 && u < 0xf900 || // neither we do surrogates
307 u >= 0x80 && u < 0xa0 || // nor latin-1 control chars
308 u < 0x20 && u != '\t')
309 return UNI_REPLACEMENT;
315 uns utf8_strlen(const byte *str);
316 uns utf8_strnlen(const byte *str, uns n);