lib/index.h

   1 /*
   2  *      Sherlock: Data structures used in indices
   3  *
   4  *      (c) 2001--2002 Martin Mares <mj@ucw.cz>
   5  */
   6
   7 #ifndef _SHERLOCK_INDEX_H
   8 #define _SHERLOCK_INDEX_H
   9
  10 #include "lib/fastbuf.h"
  11 #include SHERLOCK_CUSTOM
  12 #include "charset/unistream.h"
  13
  14 /* Words */
  15
  16 #define MAX_WORD_LEN            64
  17 #define MAX_COMPLEX_LEN         10
  18
  19 /* Word and string types are defined in lib/custom.h */
  20
  21 /* Global index parameters */
  22
  23 struct index_params {
  24   sh_time_t ref_time;                   /* Reference time (for document ages etc.) */
  25 };
  26
  27 /* Index card attributes */
  28
  29 struct card_attr {
  30   u32 card;                             /* Reference to card description (either oid or filepos) */
  31   u32 site_id;
  32   CUSTOM_CARD_ATTRS                     /* Include all custom attributes */
  33   byte weight;
  34   byte flags;
  35   byte age;                             /* Document age in pseudo-logarithmic units wrt. reference time */
  36   // byte rfu[1];                       /* If no custom attributes are defined */
  37 };
  38
  39 enum card_flag {
  40   CARD_FLAG_EMPTY = 1,                  /* Empty document (redirect, robot file etc.) [scanner] */
  41   CARD_FLAG_ACCENTED = 2,               /* Document contains accented characters [scanner] */
  42   CARD_FLAG_DUP = 4,                    /* Removed as a duplicate [merger] */
  43   CARD_FLAG_MERGED = 8,                 /* Destination of a merge [merger] */
  44   CARD_FLAG_IMAGE = 16,                 /* Is an image object [scanner] */
  45 };
  46
  47 #define CARD_POS_SHIFT 5                /* Card positions are shifted this # of bytes to the right */
  48
  49 /* String fingerprints */
  50
  51 struct fingerprint {
  52   byte hash[12];
  53 };
  54
  55 void fingerprint(byte *string, struct fingerprint *fp);
  56
  57 static inline u32
  58 fp_hash(struct fingerprint *fp)
  59 {
  60   return (fp->hash[0] << 24) | (fp->hash[1] << 16) | (fp->hash[2] << 8) | fp->hash[3];
  61 }
  62
  63 /* Reading of tagged text (Unicode values, tags mapped to 0x80000000 and higher) */
  64
  65 #define GET_TAGGED_CHAR(p,u) do {                               \
  66   u = *p;                                                       \
  67   if (u >= 0xc0)                                                \
  68     GET_UTF8_CHAR(p,u);                                         \
  69   else if (u >= 0x80)                                           \
  70     {                                                           \
  71       p++;                                                      \
  72       if (u >= 0xb0)                                            \
  73         {                                                       \
  74           ASSERT(u == 0xb0);                                    \
  75           u += 0x80020000;                                      \
  76         }                                                       \
  77       else if (u >= 0xa0)                                       \
  78         {                                                       \
  79           ASSERT(*p >= 0x80 && *p <= 0xbf);                     \
  80           u = 0x80010000 + ((u & 0x0f) << 6) + (*p++ & 0x3f);   \
  81         }                                                       \
  82       else                                                      \
  83         u += 0x80000000;                                        \
  84     }                                                           \
  85   else                                                          \
  86     p++;                                                        \
  87 } while (0)
  88
  89 #define SKIP_TAGGED_CHAR(p) do {                                \
  90   if (*p >= 0x80 && *p < 0xc0)                                  \
  91     {                                                           \
  92       uns u = *p++;                                             \
  93       if (u >= 0xa0 && u < 0xb0 && *p >= 0x80 && *p < 0xc0)     \
  94         p++;                                                    \
  95     }                                                           \
  96   else                                                          \
  97     UTF8_SKIP(p);                                               \
  98 } while (0)
  99
 100 static inline uns
 101 bget_tagged_char(struct fastbuf *f)
 102 {
 103   uns u = bgetc(f);
 104   if ((int)u < 0x80)
 105     ;
 106   else if (u < 0xc0)
 107     {
 108       if (u >= 0xb0)
 109         {
 110           ASSERT(u == 0xb0);
 111           u += 0x80020000;
 112         }
 113       else if (u >= 0xa0)
 114         {
 115           uns v = bgetc(f);
 116           ASSERT(v >= 0x80 && v <= 0xbf);
 117           u = 0x80010000 + ((u & 0x0f) << 6) + (v & 0x3f);
 118         }
 119       else
 120         u += 0x80000000;
 121     }
 122   else
 123     {
 124       bungetc(f);
 125       u = bget_utf8(f);
 126     }
 127   return u;
 128 }
 129
 130 /* Conversion of document age from seconds to our internal units */
 131
 132 static inline int
 133 convert_age(sh_time_t lastmod, sh_time_t reftime)
 134 {
 135   sh_time_t age;
 136   if (reftime < lastmod)                /* past times */
 137     return -1;
 138   age = (reftime - lastmod) / 3600;
 139   if (age < 48)                         /* last 2 days: 1 hour resolution */
 140     return age;
 141   age = (age-48) / 24;
 142   if (age < 64)                         /* next 64 days: 1 day resolution */
 143     return 48 + age;
 144   age = (age-64) / 7;
 145   if (age < 135)                        /* next 135 weeks: 1 week resolution */
 146     return 112 + age;
 147   age = (age-135) / 52;
 148   if (age < 8)                          /* next 8 years: 1 year resolution */
 149     return 247 + age;
 150   return 255;                           /* then just "infinite future" */
 151 }
 152
 153 #endif