lib/index.h

   1 /*
   2  *      Sherlock: Data structures used in indices
   3  *
   4  *      (c) 2001--2002 Martin Mares <mj@ucw.cz>
   5  */
   6
   7 #ifndef _SHERLOCK_INDEX_H
   8 #define _SHERLOCK_INDEX_H
   9
  10 #include "lib/fastbuf.h"
  11 #include SHERLOCK_CUSTOM
  12 #include "charset/unistream.h"
  13
  14 /* Words */
  15
  16 #define MAX_WORD_LEN            64
  17 #define MAX_COMPLEX_LEN         10
  18
  19 /* Word and string types are defined in lib/custom.h */
  20
  21 /* Global index parameters */
  22
  23 struct index_params {
  24   sh_time_t ref_time;                   /* Reference time (for document ages etc.) */
  25 };
  26
  27 /* Index card attributes */
  28
  29 struct card_attr {
  30   u32 card;                             /* Reference to card description (either oid or filepos) */
  31 #ifdef CONFIG_SITES
  32   u32 site_id;
  33 #endif
  34   CUSTOM_CARD_ATTRS                     /* Include all custom attributes */
  35   byte weight;
  36   byte flags;
  37   byte age;                             /* Document age in pseudo-logarithmic units wrt. reference time */
  38   // byte rfu[1];                       /* If no custom attributes are defined */
  39 };
  40
  41 enum card_flag {
  42   CARD_FLAG_EMPTY = 1,                  /* Empty document (redirect, robot file etc.) [scanner] */
  43   CARD_FLAG_ACCENTED = 2,               /* Document contains accented characters [scanner] */
  44   CARD_FLAG_DUP = 4,                    /* Removed as a duplicate [merger] */
  45   CARD_FLAG_MERGED = 8,                 /* Destination of a merge [merger] */
  46   CARD_FLAG_IMAGE = 16,                 /* Is an image object [scanner] */
  47 };
  48
  49 #define CARD_POS_SHIFT 5                /* Card positions are shifted this # of bytes to the right */
  50
  51 /* String fingerprints */
  52
  53 struct fingerprint {
  54   byte hash[12];
  55 };
  56
  57 void fingerprint(byte *string, struct fingerprint *fp);
  58
  59 static inline u32
  60 fp_hash(struct fingerprint *fp)
  61 {
  62   return (fp->hash[0] << 24) | (fp->hash[1] << 16) | (fp->hash[2] << 8) | fp->hash[3];
  63 }
  64
  65 /* Reading of tagged text (Unicode values, tags mapped to 0x80000000 and higher) */
  66
  67 #define GET_TAGGED_CHAR(p,u) do {                               \
  68   u = *p;                                                       \
  69   if (u >= 0xc0)                                                \
  70     GET_UTF8_CHAR(p,u);                                         \
  71   else if (u >= 0x80)                                           \
  72     {                                                           \
  73       p++;                                                      \
  74       if (u >= 0xb0)                                            \
  75         {                                                       \
  76           ASSERT(u == 0xb0);                                    \
  77           u += 0x80020000;                                      \
  78         }                                                       \
  79       else if (u >= 0xa0)                                       \
  80         {                                                       \
  81           ASSERT(*p >= 0x80 && *p <= 0xbf);                     \
  82           u = 0x80010000 + ((u & 0x0f) << 6) + (*p++ & 0x3f);   \
  83         }                                                       \
  84       else                                                      \
  85         u += 0x80000000;                                        \
  86     }                                                           \
  87   else                                                          \
  88     p++;                                                        \
  89 } while (0)
  90
  91 #define SKIP_TAGGED_CHAR(p) do {                                \
  92   if (*p >= 0x80 && *p < 0xc0)                                  \
  93     {                                                           \
  94       uns u = *p++;                                             \
  95       if (u >= 0xa0 && u < 0xb0 && *p >= 0x80 && *p < 0xc0)     \
  96         p++;                                                    \
  97     }                                                           \
  98   else                                                          \
  99     UTF8_SKIP(p);                                               \
 100 } while (0)
 101
 102 static inline uns
 103 bget_tagged_char(struct fastbuf *f)
 104 {
 105   uns u = bgetc(f);
 106   if ((int)u < 0x80)
 107     ;
 108   else if (u < 0xc0)
 109     {
 110       if (u >= 0xb0)
 111         {
 112           ASSERT(u == 0xb0);
 113           u += 0x80020000;
 114         }
 115       else if (u >= 0xa0)
 116         {
 117           uns v = bgetc(f);
 118           ASSERT(v >= 0x80 && v <= 0xbf);
 119           u = 0x80010000 + ((u & 0x0f) << 6) + (v & 0x3f);
 120         }
 121       else
 122         u += 0x80000000;
 123     }
 124   else
 125     {
 126       bungetc(f);
 127       u = bget_utf8(f);
 128     }
 129   return u;
 130 }
 131
 132 /* Conversion of document age from seconds to our internal units */
 133
 134 static inline int
 135 convert_age(sh_time_t lastmod, sh_time_t reftime)
 136 {
 137   sh_time_t age;
 138   if (reftime < lastmod)                /* past times */
 139     return -1;
 140   age = (reftime - lastmod) / 3600;
 141   if (age < 48)                         /* last 2 days: 1 hour resolution */
 142     return age;
 143   age = (age-48) / 24;
 144   if (age < 64)                         /* next 64 days: 1 day resolution */
 145     return 48 + age;
 146   age = (age-64) / 7;
 147   if (age < 135)                        /* next 135 weeks: 1 week resolution */
 148     return 112 + age;
 149   age = (age-135) / 52;
 150   if (age < 8)                          /* next 8 years: 1 year resolution */
 151     return 247 + age;
 152   return 255;                           /* then just "infinite future" */
 153 }
 154
 155 #endif