From: Martin Mares <mj@ucw.cz>
Date: Tue, 23 Sep 2003 16:20:10 +0000 (+0000)
Subject: Added a set of functions for sliding window mmapping of large files.
X-Git-Tag: holmes-import~1213
X-Git-Url: http://mj.ucw.cz/gitweb/?a=commitdiff_plain;h=bc6049e67fb8de5774b7f6d3160bc89f952f9428;p=libucw.git

Added a set of functions for sliding window mmapping of large files.
Will be used by the indexer to access the card notes array.
---

diff --git a/lib/Makefile b/lib/Makefile
index 302b2ddd..e571fe6e 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -8,7 +8,8 @@ LIBSH_MODS=alloc alloc_str ctmatch db fastbuf fb-file fb-mem lists \
 	prime random realloc regex timer url wildmatch \
 	wordsplit str_ctype str_upper bucket conf object sorter \
 	finger proctitle ipaccess profile bitsig randomkey \
-	hashfunc base64 base224 fb-temp fb-mmap fb-printf urlkey
+	hashfunc base64 base224 fb-temp fb-mmap fb-printf urlkey \
+	partmap
 LIBSH_MOD_PATHS=$(addprefix obj/lib/,$(LIBSH_MODS)) $(CUSTOM_LIB_MODULES)
 
 obj/lib/libsh.a: $(addsuffix .o,$(LIBSH_MOD_PATHS))
diff --git a/lib/lib.h b/lib/lib.h
index 125deaca..a7cf0417 100644
--- a/lib/lib.h
+++ b/lib/lib.h
@@ -150,6 +150,14 @@ uns random_max(uns);
 void *mmap_file(byte *name, unsigned *len, int writeable);
 void munmap_file(void *start, unsigned len);
 
+/* partmap.c */
+
+struct partmap;
+struct partmap *partmap_open(byte *name, int writeable);
+void *partmap_map(struct partmap *p, sh_off_t offset, uns size);
+void partmap_close(struct partmap *p);
+sh_off_t partmap_size(struct partmap *p);
+
 /* proctitle.c */
 
 void setproctitle_init(int argc, char **argv);
diff --git a/lib/partmap.c b/lib/partmap.c
new file mode 100644
index 00000000..2947bedf
--- /dev/null
+++ b/lib/partmap.c
@@ -0,0 +1,94 @@
+/*
+ *	Sherlock Library -- Mapping of File Parts
+ *
+ *	(c) 2003 Martin Mares <mj@ucw.cz>
+ *
+ *	This software may be freely distributed and used according to the terms
+ *	of the GNU Lesser General Public License.
+ */
+
+#include "lib/lib.h"
+#include "lib/lfs.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+
+struct partmap {
+  int fd;
+  sh_off_t file_size;
+  sh_off_t start_off, end_off;
+  byte *start_map;
+  int writeable;
+};
+
+#ifdef TEST
+#define PARTMAP_WINDOW 4096
+#else
+#define PARTMAP_WINDOW 16777216
+#endif
+
+struct partmap *
+partmap_open(byte *name, int writeable)
+{
+  struct partmap *p = xmalloc_zero(sizeof(struct partmap));
+
+  p->fd = sh_open(name, writeable ? O_RDWR : O_RDONLY);
+  if (p->fd < 0)
+    die("open(%s): %m", name);
+  if ((p->file_size = sh_seek(p->fd, 0, SEEK_END)) < 0)
+    die("lseek(%s): %m", name);
+  p->writeable = writeable;
+  return p;
+}
+
+sh_off_t
+partmap_size(struct partmap *p)
+{
+  return p->file_size;
+}
+
+void
+partmap_close(struct partmap *p)
+{
+  if (p->start_map)
+    munmap(p->start_map, p->end_off - p->start_off);
+  close(p->fd);
+  xfree(p);
+}
+
+void *
+partmap_map(struct partmap *p, sh_off_t start, uns size)
+{
+  if (!p->start_map || start < p->start_off || start+size > p->end_off)
+    {
+      if (p->start_map)
+	munmap(p->start_map, p->end_off - p->start_off);
+      uns win = PARTMAP_WINDOW;
+      ASSERT(win >= size);
+      if (start+win > p->file_size)
+	win = p->file_size - start;
+      p->start_map = sh_mmap(NULL, win, p->writeable ? (PROT_READ | PROT_WRITE) : PROT_READ, MAP_SHARED, p->fd, start);
+      if (p->start_map == MAP_FAILED)
+	die("mmap failed at position %Ld: %m", (long long)start);
+      p->start_off = start;
+      p->end_off = start+win;
+    }
+  return p->start_map + (start - p->start_off);
+}
+
+#ifdef TEST
+int main(int argc, char **argv)
+{
+  struct partmap *p = partmap_open(argv[1], 0);
+  uns l = partmap_size(p);
+  uns i;
+  for (i=0; i<l; i++)
+    putchar(*(char *)partmap_map(p, i, 1));
+  partmap_close(p);
+  return 0;
+}
+#endif