X-Git-Url: http://mj.ucw.cz/gitweb/?a=blobdiff_plain;f=lib%2Findex.h;h=bab49b952dcaca54ecbcb43058eecfb494f91e09;hb=5bb0345810e5d0097488fffd1216b9ed0b4b9d05;hp=733995a1cf2b98ccae9ca738668ea54ad2c2ba42;hpb=69191ef580483146f13f9beccc118e4a24c4e607;p=libucw.git diff --git a/lib/index.h b/lib/index.h index 733995a1..bab49b95 100644 --- a/lib/index.h +++ b/lib/index.h @@ -1,92 +1,75 @@ /* - * Sherlock Gatherer: Data structures used in indices + * Sherlock: Data structures used in indices * - * (c) 2001 Martin Mares + * (c) 2001--2004 Martin Mares */ -/* Words */ - -#define MAX_WORD_LEN 64 - -/* Word types */ - -enum word_type { - WT_RESERVED, /* Reserved word type */ - WT_TEXT, /* Ordinary text */ - WT_EMPH, /* Emphasized text */ - WT_SMALL, /* Small font */ - WT_TITLE, /* Document title */ - WT_SMALL_HEADING, /* Heading */ - WT_BIG_HEADING, /* Larger heading */ - WT_KEYWORD, /* Explicitly marked keyword */ - WT_META, /* Various meta-information */ - WT_ALT, /* Alternate texts for graphical elements */ - WT_URL1, /* Word extracted from document URL (low and high weight) */ - WT_URL2, - WT_MAX -}; +#ifndef _SHERLOCK_INDEX_H +#define _SHERLOCK_INDEX_H -/* Descriptive names used for user output */ -#define WORD_TYPE_USER_NAMES \ - "reserved", "text", "emph", "small", "title", "hdr1", "hdr2", "keywd", \ - "meta", "alt", "urlword1", "urlword2", "type12", "type13", "type14", "type15" - -/* Keywords for word type names */ -#define WORD_TYPE_NAMES \ - T(WORD, ~0) \ - T(TEXT, 1 << WT_TEXT) \ - T(EMPH, 1 << WT_EMPH) \ - T(SMALL, 1 << WT_SMALL) \ - T(TITLE, 1 << WT_TITLE) \ - T(HDR, (1 << WT_SMALL_HEADING) | (1 << WT_BIG_HEADING)) \ - T(HDR1, 1 << WT_SMALL_HEADING) \ - T(HDR2, 1 << WT_BIG_HEADING) \ - T(KEYWD, 1 << WT_KEYWORD) \ - T(META, 1 << WT_META) \ - T(ALT, 1 << WT_ALT) \ - T(URLWORD, (1 << WT_URL1) | (1 << WT_URL2)) - -/* These types are not shown in document contexts */ -#define WORD_TYPES_HIDDEN ((1 << WT_URL1) | (1 << WT_URL2)) - -/* These types are always matched without accents if accent mode is set to "auto" */ -#define WORD_TYPES_NO_AUTO_ACCENT ((1 << WT_URL1) | (1 << WT_URL2)) - -/* String types */ - -enum string_type { - ST_RESERVED, /* Reserved string type */ - ST_URL, /* URL of the document */ - ST_HOST, /* Host name */ - ST_DOMAIN, /* Domain name */ - ST_REF, /* URL reference */ - ST_BACKREF, /* Back-reference (frame or redirect source) */ - ST_MAX -}; +#include "custom/lib/custom.h" -#define STRING_TYPE_USER_NAMES \ - "URL", "host", "domain", "ref", "backref", "type5", "type6", "type7", \ - "type8", "type9", "type10", "type11", "type12", "type13", "type14", "type15" +#define INDEX_VERSION (0x32260200+sizeof(struct card_attr)) /* Increase with each incompatible change in index format */ -#define STRING_TYPE_NAMES \ - T(URL, 1 << ST_URL) \ - T(HOST, 1 << ST_HOST) \ - T(DOMAIN, 1 << ST_DOMAIN) \ - T(REF, 1 << ST_REF) \ - T(BACKREF, 1 << ST_BACKREF) +/* + * Words + * + * MAX_WORD_LEN is the maximum length (measured in UTF-8 characters, excluding + * the terminating zero byte if there's any) of any word which may appear in the + * indices or in the bucket file. Naturally, the same constant also bounds + * the number of UCS-2 characters in a word. + * + * Caveat: If you are upcasing/downcasing the word, the UTF-8 encoding can + * expand, although at most twice, so you need to reserve 2*MAX_WORD_LEN bytes. + */ -#define STRING_TYPES_URL ((1 << ST_URL) | (1 << ST_REF) | (1 << ST_BACKREF)) -/* These must be indexed in lowercase form */ -#define STRING_TYPES_CASE_INSENSITIVE ((1 << ST_HOST) | (1 << ST_DOMAIN)) +#define MAX_WORD_LEN 64 /* a multiple of 4 */ + +/* Word and string types are defined in lib/custom.h */ + +/* Types used for storing contexts */ + +#ifdef CONFIG_CONTEXTS +#if CONFIG_MAX_CONTEXTS == 32768 +typedef u16 context_t; +#define bget_context bgetw +#define bput_context bputw +#define GET_CONTEXT GET_U16 +#define PUT_CONTEXT PUT_U16 +#elif CONFIG_MAX_CONTEXTS == 256 +typedef byte context_t; +#define bget_context bgetc +#define bput_context bputc +#define GET_CONTEXT GET_U8 +#define PUT_CONTEXT PUT_U8 +#else +#error CONFIG_MAX_CONTEXTS set to an invalid value. +#endif +#else +struct fastbuf; +typedef struct { } context_t; +static inline uns bget_context(struct fastbuf *b UNUSED) { return 0; } +static inline void bput_context(struct fastbuf *b UNUSED, uns context UNUSED) { } +#define GET_CONTEXT(p) 0 +#define PUT_CONTEXT(p,x) do {} while(0) +#endif /* Index card attributes */ struct card_attr { u32 card; /* Reference to card description (either oid or filepos) */ +#ifdef CONFIG_SITES u32 site_id; +#endif + CUSTOM_CARD_ATTRS /* Include all custom attributes */ byte weight; byte flags; - byte rfu[2]; +#ifdef CONFIG_LASTMOD + byte age; /* Document age in pseudo-logarithmic units wrt. reference time */ +#endif +#ifdef CONFIG_FILETYPE + byte type_flags; /* File type flags (see below) */ +#endif }; enum card_flag { @@ -94,9 +77,45 @@ enum card_flag { CARD_FLAG_ACCENTED = 2, /* Document contains accented characters [scanner] */ CARD_FLAG_DUP = 4, /* Removed as a duplicate [merger] */ CARD_FLAG_MERGED = 8, /* Destination of a merge [merger] */ + CARD_FLAG_IMAGE = 16, /* Is an image object [scanner] */ + CARD_FLAG_FRAMESET = 32, /* Contains a frameset to be ignored [scanner] */ + CARD_FLAG_OVERRIDEN = 64, /* Overriden by another index [sherlockd] */ }; -#define CARD_POS_SHIFT 5 /* Card positions are shifted this # of bytes to the right */ +#define CARD_POS_SHIFT 5 /* Card positions are shifted this # of bits to the right */ + +/* + * We store document type and several other properties in card_attr->type_flags. + * Here we define only the basic structure, the details are defined in custom.h + * (the list of type names custom_file_type_names[] and also setting of the file + * types in custom_create_attrs()). + * + * bits 7--5 file type: (0-3: text types, 4-7: other types, defined by custom.h) + * bits 4--0 type-dependent information, for text types it's document language code + */ + +#ifdef CONFIG_FILETYPE +#define CA_GET_FILE_TYPE(a) ((a)->type_flags >> 5) +#define CA_GET_FILE_INFO(a) ((a)->type_flags & 0x1f) +#define CA_GET_FILE_LANG(a) ((a)->type_flags & 0x80 ? 0 : CA_GET_FILE_INFO(a)) +#define FILETYPE_ATTRS SMALL_SET_ATTR(ftype, FILETYPE, CA_GET_FILE_TYPE, ext_ft_parse) +#define MAX_FILE_TYPES 8 +#define FILETYPE_IS_TEXT(f) ((f) < 4) +byte *ext_ft_parse(u32 *dest, byte *value, uns intval); +extern byte *custom_file_type_names[MAX_FILE_TYPES]; +#else +#define FILETYPE_ATTRS +#endif + +#ifdef CONFIG_LANG +/* You can use language matching without CONFIG_FILETYPE, but you have to define CA_GET_FILE_LANG yourself. */ +#define LANG_ATTRS SMALL_SET_ATTR(lang, LANG, CA_GET_FILE_LANG, ext_lang_parse) +byte *ext_lang_parse(u32 *dest, byte *value, uns intval); +#else +#define LANG_ATTRS +#endif + +#define EXTENDED_ATTRS CUSTOM_ATTRS LANG_ATTRS /* Beware, FILETYPE_ATTRS are handled separately */ /* String fingerprints */ @@ -112,29 +131,41 @@ fp_hash(struct fingerprint *fp) return (fp->hash[0] << 24) | (fp->hash[1] << 16) | (fp->hash[2] << 8) | fp->hash[3]; } -/* Reading of tagged text (Unicode values, tags mapped to 0x80000000 and higher) */ - -#define GET_TAGGED_CHAR(p,u) do { \ - u = *p; \ - if (u >= 0xc0) \ - GET_UTF8(p,u); \ - else if (u >= 0x80) \ - { \ - p++; \ - if (u >= 0xb0) \ - { \ - if (u != 0xb0) \ - ASSERT(0); \ - u += 0x80020000; \ - } \ - else if (u >= 0xa0) \ - { \ - ASSERT(*p >= 0x80 && *p <= 0xbf); \ - u = 0x80010000 + ((u & 0x0f) << 6) + (*p++ & 0x3f); \ - } \ - else \ - u += 0x80000000; \ - } \ - else \ - p++; \ -} while (0) +/* The card fingerprints */ + +struct card_print { + struct fingerprint fp; + u32 cardid; +}; + +/* URL keys */ + +#define URL_KEY_BUF_SIZE (3*MAX_URL_SIZE) +byte *url_key(byte *url, byte *buf); +void url_fingerprint(byte *url, struct fingerprint *fp); +void url_key_init(void); + +/* Conversion of document age from seconds to our internal units */ + +static inline int +convert_age(sh_time_t lastmod, sh_time_t reftime) +{ + sh_time_t age; + if (reftime < lastmod) /* past times */ + return -1; + age = (reftime - lastmod) / 3600; + if (age < 48) /* last 2 days: 1 hour resolution */ + return age; + age = (age-48) / 24; + if (age < 64) /* next 64 days: 1 day resolution */ + return 48 + age; + age = (age-64) / 7; + if (age < 135) /* next 135 weeks: 1 week resolution */ + return 112 + age; + age = (age-135) / 52; + if (age < 8) /* next 8 years: 1 year resolution */ + return 247 + age; + return 255; /* then just "infinite future" */ +} + +#endif