Several bug fixes in the logger:

[libucw.git] / lib / index.h

diff --git a/lib/index.h b/lib/index.h
index 6579fc4d09aec556531726687a2ef1a3715272f0..b17e9a62626440c342d1632086ffa94aa8bc7e1b 100644 (file)
--- a/lib/index.h
+++ b/lib/index.h
@@ -11,9 +11,21 @@
 #include SHERLOCK_CUSTOM
 #include "charset/unistream.h"
 
-/* Words */
+/*
+ *  Words and word complexes
+ *
+ *  MAX_WORD_LEN is the maximum length (measured in UTF-8 characters, excluding
+ *  the terminating zero byte if there's any) of any word which may appear in the
+ *  indices or in the bucket file. Naturally, the same constant also bounds
+ *  the number of UCS-2 characters in a word.
+ *
+ *  Caveat: If you are upcasing/downcasing the word, the UTF-8 encoding can
+ *  expand, although at most twice, so you need to reserve 2*MAX_WORD_LEN bytes.
+ *
+ *  MAX_COMPLEX_LEN is the upper bound on number of words in any word complex.
+ */
 
-#define MAX_WORD_LEN           64
+#define MAX_WORD_LEN           64      /* a multiple of 4 */
 #define MAX_COMPLEX_LEN                10
 
 /* Word and string types are defined in lib/custom.h */
@@ -28,7 +40,9 @@ struct index_params {
 
 struct card_attr {
   u32 card;                            /* Reference to card description (either oid or filepos) */
+#ifdef CONFIG_SITES
   u32 site_id;
+#endif
   CUSTOM_CARD_ATTRS                    /* Include all custom attributes */
   byte weight;
   byte flags;
@@ -86,6 +100,17 @@ fp_hash(struct fingerprint *fp)
     p++;                                                       \
 } while (0)
 
+#define SKIP_TAGGED_CHAR(p) do {                               \
+  if (*p >= 0x80 && *p < 0xc0)                                 \
+    {                                                          \
+      uns u = *p++;                                            \
+      if (u >= 0xa0 && u < 0xb0 && *p >= 0x80 && *p < 0xc0)    \
+       p++;                                                    \
+    }                                                          \
+  else                                                         \
+    UTF8_SKIP(p);                                              \
+} while (0)
+
 static inline uns
 bget_tagged_char(struct fastbuf *f)
 {