/*
* UCW Library -- Fast Buffered I/O
*
- * (c) 1997--2007 Martin Mares <mj@ucw.cz>
+ * (c) 1997--2008 Martin Mares <mj@ucw.cz>
* (c) 2004 Robert Spalek <robert@ucw.cz>
*
* This software may be freely distributed and used according to the terms
#include <alloca.h>
/***
- * Generic buffered I/O. You supply hooks to be called for low-level operations
- * (swapping of buffers, seeking and closing), we do the rest.
+ * === Internal structure [[internal]]
*
- * Buffer layout when reading:
+ * Generally speaking, a fastbuf consists of a buffer and a set of callbacks.
+ * All front-end functions operate on the buffer and if the buffer becomes
+ * empty or fills up, they ask the corresponding callback to solve the
+ * situation. Back-ends then differ just in the definition of the callbacks.
+ * The state of the fastbuf is represented by `struct fastbuf`, which
+ * is a simple structure describing the state of the buffer and pointers
+ * to the callback functions.
+ *
+ * When we are reading from the fastbuf, the buffer is laid out as follows:
*
* +----------------+---------------------------+
* | read data | free space |
* ^ ^ ^ ^
* buffer bptr bstop bufend
*
- * After the last character is read, +bptr == bstop+ and buffer refill
- * is deferred to the next read attempt. This gives us an easy way
- * how to implement bungetc().
+ * Here `bptr` points to the next character to be read. After the last character is
+ * read, `bptr == bstop` and the `refill` callback gets called upon the next read
+ * attempt to bring further data. This gives us an easy way how to implement bungetc().
*
- * When writing:
+ * When writing, the situation looks like:
*
* +--------+--------------+--------------------+
* | unused | written data | free space |
* ^ ^ ^ ^
* buffer bstop bptr bufend
*
- * Dirty tricks:
+ * In this case, the `bptr` points at the position where the next character
+ * will be written to. When we want to write, but `bptr == bufend`, we call
+ * the `spout` hook to flush the data and get an empty buffer.
+ *
+ * Several dirty tricks can be played:
*
- * - You can mix reads and writes on the same stream, but you must
- * call bflush() in between and remember that the file position
- * points after the flushed buffer which is not necessarily the same
- * as after the data you've read.
- * - The spout/refill hooks can change not only bptr and bstop, but also
- * the location of the buffer; +fb-mem.c+ takes advantage of it.
- * - In some cases, the user of the +bdirect+ interface can be allowed to modify
+ * - The `spout`/`refill` hooks can change not only `bptr` and `bstop`, but also
+ * the location and size of the buffer; the fb-mem back-end takes advantage of it.
+ * - In some cases, the user of the `bdirect` interface can be allowed to modify
* the data in the buffer to avoid unnecessary copying. If the back-end
- * allows such modifications, it can set +can_overwrite_buffer+ accordingly:
+ * allows such modifications, it can set `fastbuf->can_overwrite_buffer` accordingly:
* * 0 if no modification is allowed,
* * 1 if the user can modify the buffer on the condition that
* the modifications will be undone before calling the next
* In this case, the back-end must be prepared for trimming
* of the buffer which is done by the commit function.
*
- * Generic parts
- * ~~~~~~~~~~~~~
***/
/**
- * Fastbuf structure.
- * This structure is of main interest to fastbuf backends,
- * it can be considered a black box for use.
+ * This structure contains the state of the fastbuf. See the discussion above
+ * for how it works.
**/
struct fastbuf {
byte is_fastbuf[0]; /* Dummy field for checking of type casts */
- byte *bptr, *bstop; /* Access pointers */
+ byte *bptr, *bstop; /* State of the buffer */
byte *buffer, *bufend; /* Start and end of the buffer */
- char *name; /* File name for error messages */
+ char *name; /* File name (used for error messages) */
ucw_off_t pos; /* Position of bstop in the file */
- int (*refill)(struct fastbuf *); /* Get a buffer with new data */
+ int (*refill)(struct fastbuf *); /* Get a buffer with new data, returns 0 on EOF */
void (*spout)(struct fastbuf *); /* Write buffer data to the file */
int (*seek)(struct fastbuf *, ucw_off_t, int);/* Slow path for bseek(), buffer already flushed; returns success */
void (*close)(struct fastbuf *); /* Close the stream */
int (*config)(struct fastbuf *, uns, int); /* Configure the stream */
- int can_overwrite_buffer; /* Can the buffer be altered? (see discussion above) 0=never, 1=temporarily, 2=permanently */
+ int can_overwrite_buffer; /* Can the buffer be altered? 0=never, 1=temporarily, 2=permanently */
};
-/*** === FastIO on files with several configurable back-ends ***/
+/***
+ * === Fastbuf on files [[fbparam]]
+ *
+ * If you want to use fastbufs to access files, you can choose one of several
+ * back-ends and set their parameters.
+ ***/
/**
- * Which back-end do you want to use?
+ * Back-end types
*/
enum fb_type {
FB_STD, /* Standard buffered I/O */
- FB_DIRECT, /* Direct I/O bypassing system caches (see +fb-direct.c+ for a description) */
+ FB_DIRECT, /* Direct I/O bypassing system caches (see fb-direct.c for a description) */
FB_MMAP /* Memory mapped files */
};
/**
- * A way to configure created fastbuf.
+ * When you open a file fastbuf, you can use this structure to select a back-end
+ * and set its parameters. If you want just an "ordinary" file stream, you can
+ * happily pass NULL instead and the defaults from the configuration file (or
+ * hard-wired defaults if no config file has been read) will be used.
*/
struct fb_params {
- enum fb_type type;
- uns buffer_size; /* 0 for default size. */
- uns keep_back_buf; /* FB_STD: optimize for bi-directional access. */
- uns read_ahead; /* FB_DIRECT options. */
+ enum fb_type type; /* The chosen back-end */
+ uns buffer_size; /* 0 for default size */
+ uns keep_back_buf; /* FB_STD: optimize for bi-directional access */
+ uns read_ahead; /* FB_DIRECT options */
uns write_back;
struct asio_queue *asio;
};
struct cf_section;
-extern struct cf_section fbpar_cf; /** Config. Can be used by fastbuf systems. **/
-extern struct fb_params fbpar_def; /** Default parameters. **/
+extern struct cf_section fbpar_cf; /** Configuration section with which you can fill the `fb_params` **/
+extern struct fb_params fbpar_def; /** The default `fb_params` **/
/**
- * Opens a file.
- * Use +@params = NULL+ for defaults.
- * See standard unix open() for information about @mode.
+ * Opens a file with file mode @mode (see the man page of open()).
+ * Use @params to select the fastbuf back-end and its parameters or
+ * pass `NULL` if you are fine with defaults.
+ *
+ * Dies if the file does not exist.
**/
struct fastbuf *bopen_file(const char *name, int mode, struct fb_params *params);
-struct fastbuf *bopen_file_try(const char *name, int mode, struct fb_params *params); /** Tries to open a file (does not die, if unsuccessful). **/
+struct fastbuf *bopen_file_try(const char *name, int mode, struct fb_params *params); /** Like bopen_file(), but returns NULL on failure. **/
/**
* Opens a temporary file.
- * It is placed with other temp files and is deleted when closed.
+ * It is placed with other temp files and it is deleted when closed.
+ * Again, use `NULL` for @params if you want the defaults.
**/
struct fastbuf *bopen_tmp_file(struct fb_params *params);
+
/**
- * Creates a fastbuf (wrapper) from a file descriptor.
- * Sets it's filename to @name (used when outputting errors).
+ * Creates a fastbuf from a file descriptor @fd and sets its filename
+ * to @name (the name is used only in error messages).
+ * When the fastbuf is closed, the fd is closed as well. You can override
+ * this behavior by calling bconfig().
*/
struct fastbuf *bopen_fd_name(int fd, struct fb_params *params, const char *name);
-static inline struct fastbuf *bopen_fd(int fd, struct fb_params *params) /** Same as above, but with empty filename. **/
+static inline struct fastbuf *bopen_fd(int fd, struct fb_params *params) /** Same as above, but with an auto-generated filename. **/
{
return bopen_fd_name(fd, params, NULL);
}
+/**
+ * Flushes all buffers and makes sure that they are written to the disk.
+ **/
+void bfilesync(struct fastbuf *b);
+
/***
- * FastIO on standard files (shortcuts for FB_STD)
- * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ * === Fastbufs on regular files [[fbfile]]
+ *
+ * If you want to use the `FB_STD` back-end and not worry about setting
+ * up any parameters, there is a couple of shortcuts.
***/
-/**
- * Opens a file in a usual way (with system cache enabled).
- * Use +@buflen = 0+ for default buffer size.
- * Dies if unsuccessful.
- */
-struct fastbuf *bopen(const char *name, uns mode, uns buflen);
-struct fastbuf *bopen_try(const char *name, uns mode, uns buflen);/** Same as bopen(), but does not die when unsuccessful. **/
-struct fastbuf *bopen_tmp(uns buflen);/** Opens a temporary file (read-write). Deletes it, when closed. **/
-struct fastbuf *bfdopen(int fd, uns buflen);/** Wraps a filedescriptor into a fastbuf. **/
-struct fastbuf *bfdopen_shared(int fd, uns buflen);/** Wraps a filedescriptor and marks it as shared. **/
-void bfilesync(struct fastbuf *b);/** Sync file to disk. **/
+struct fastbuf *bopen(const char *name, uns mode, uns buflen); /** Equivalent to bopen_file() with `FB_STD` back-end. **/
+struct fastbuf *bopen_try(const char *name, uns mode, uns buflen); /** Equivalent to bopen_file_try() with `FB_STD` back-end. **/
+struct fastbuf *bopen_tmp(uns buflen); /** Equivalent to bopen_tmp_file() with `FB_STD` back-end. **/
+struct fastbuf *bfdopen(int fd, uns buflen); /** Equivalent to bopen_fd() with `FB_STD` back-end. **/
+struct fastbuf *bfdopen_shared(int fd, uns buflen); /** Like bfdopen(), but it does not close the @fd on bclose(). **/
+
+/***
+ * === Temporary files [[fbtemp]]
+ *
+ * Usually, bopen_tmp_file() is the best way how to come to a temporary file.
+ * However, in some specific cases you can need more, so there is also a set
+ * of more general functions.
+ ***/
-/*** === Temporary files ***/
+#define TEMP_FILE_NAME_LEN 256
-#define TEMP_FILE_NAME_LEN 256 /** Maximum length of temp file name. **/
/**
- * Generates a temporary filename.
- * Provide a buffer (as @name_buf, at last +TEMP_FILE_NAME_LEN+ long) to store the name into.
- * If @open_flags are not +NULL+, flags that should be ored with other flags to open() will be set.
+ * Generates a temporary filename and stores it to the @name_buf (of size
+ * at least * `TEMP_FILE_NAME_LEN`). If @open_flags are not `NULL`, flags that
+ * should be OR-ed with other flags to open() will be stored there.
+ *
+ * The location and style of temporary files is controlled by the configuration.
+ * By default, the system temp directory (`$TMPDIR` or `/tmp`) is used.
*
- * The provided name can already exist.
- * If it is not safe to overwrite existing files, +O_EXCL+ is specified in @open_flags.
- * Check for the result of open().
+ * If the location is a publicly writeable directory (like `/tmp`), the
+ * generated filename cannot be guaranteed to be unique, so @open_flags
+ * will include `O_EXCL` and you have to check the result of open() and
+ * iterate if needed.
*
- * This is not specific to fastbufs, can be used separately.
+ * This function is not specific to fastbufs, it can be used separately.
**/
void temp_file_name(char *name_buf, int *open_flags);
+
/**
- * Renames a temp fastbuf to given @name and marks it permanent (so it will not be deleted when closed).
- * The fastbuf is closed by this call.
- */
-void bfix_tmp_file(struct fastbuf *fb, const char *name);
-/**
- * Opens a temporary file and returns it as file descriptor.
- * You specify open @mode and @open_flags.
+ * Opens a temporary file and returns its file descriptor.
+ * You specify the file @mode and @open_flags passed to open().
*
- * If @name_buf (at last +TEMP_FILE_NAME_LEN+ long) is not +NULL+, the filename is stored there.
+ * If the @name_buf (of at last `TEMP_FILE_NAME_LEN` chars) is not `NULL`,
+ * the filename is also stored in it.
*
- * This is not specific to fastbufs, can be used separately.
+ * This function is not specific to fastbufs, it can be used separately.
*/
int open_tmp(char *name_buf, int open_flags, int mode);
+/**
+ * Sometimes, a file is created as temporary and then moved to a stable
+ * location. This function takes a fastbuf created by bopen_tmp_file()
+ * or bopen_tmp(), marks it as permanent, closes it and renames it to
+ * @name.
+ *
+ * Please note that it assumes that the temporary file and the @name
+ * are on the same volume (otherwise, rename() fails), so you might
+ * want to configure a special location for the temporary files
+ * beforehand.
+ */
+void bfix_tmp_file(struct fastbuf *fb, const char *name);
+
/* Internal functions of some file back-ends */
struct fastbuf *bfdopen_internal(int fd, const char *name, uns buflen);
void bclose_file_helper(struct fastbuf *f, int fd, int is_temp_file);
/***
- * FastIO on in-memory streams
- * ~~~~~~~~~~~~~~~~~~~~~~~~~~~
- * These behaves in a way similar to unix pipes.
- * You create one for writing and another for reading.
- * What you write to the first one can be read from the other.
+ * === Fastbufs on file fragments [[fblim]]
+ *
+ * The `fblim` back-end reads from a file handle, but at most a given
+ * number of bytes. This is frequently used for reading from sockets.
***/
-struct fastbuf *fbmem_create(uns blocksize); /** Create stream and return its writing fastbuf. **/
-struct fastbuf *fbmem_clone_read(struct fastbuf *); /** Create reading fastbuf. **/
+struct fastbuf *bopen_limited_fd(int fd, uns bufsize, uns limit); /** Create a fastbuf which reads at most @limit bytes from @fd. **/
-/*** === FastI on file descriptors with limit ***/
+/***
+ * === Fastbufs on in-memory streams [[fbmem]]
+ *
+ * The `fbmem` back-end keeps the whole contents of the stream
+ * in memory (as a linked list of memory blocks, so address space
+ * fragmentation is avoided).
+ *
+ * First, you use fbmem_create() to create the stream and the fastbuf
+ * used for writing to it. Then you can call fbmem_clone_read() to get
+ * an arbitrary number of fastbuf for reading from the stream.
+ ***/
-/**
- * Wrap a file descriptor @fd into a fastbuf.
- * No more than @limit bytes will be read/written in the lifetime of this fastbuf.
- **/
-struct fastbuf *bopen_limited_fd(int fd, uns bufsize, uns limit);
+struct fastbuf *fbmem_create(uns blocksize); /** Create stream and return its writing fastbuf. **/
+struct fastbuf *fbmem_clone_read(struct fastbuf *f); /** Given a writing fastbuf, create a new reading fastbuf. **/
-/*** === FastIO on static buffers ***/
+/***
+ * === Fastbufs on static buffers [[fbbuf]]
+ *
+ * The `fbbuf` back-end stores the stream in a given block of memory.
+ * This is useful for parsing and generating of complex data structures.
+ ***/
/**
- * Creates a fastbuf that takes data from a memory buffer.
- * The fastbuf is not allocated, it is initialized in @f.
- * @buffer and @size specify the buffer with data.
- * See top of this file for info about @can_overwrite.
+ * Creates a read-only fastbuf that takes its data from a given buffer.
+ * The fastbuf structure is allocated by the caller and pointed to by @f.
+ * The @buffer and @size specify the location and size of the buffer.
+ *
+ * In some cases, the front-ends can take advantage of rewriting the contents
+ * of the buffer temporarily. In this case, set @can_overwrite as described
+ * in xref:internal[Internals]. If you do not care, keep @can_overwrite zero.
*
- * No resources are alocated by this, so you do not need to free it.
* It is not possible to close this fastbuf.
*/
void fbbuf_init_read(struct fastbuf *f, byte *buffer, uns size, uns can_overwrite);
+
/**
- * Creates a fastbuf which writes into a provided memory buffer.
- * Write over the end dies.
+ * Creates a write-only fastbuf which writes into a provided memory buffer.
+ * The fastbuf structure is allocated by the caller and pointed to by @f.
+ * An attempt to write behind the end of the buffer dies.
*
- * No resources are allocated by this and you can not close this fastbuf.
+ * Data are written directly into the buffer, so it is not necessary to call bflush()
+ * at any moment.
*
- * Data are written directly into the buffer, no need for flushes.
+ * It is not possible to close this fastbuf.
*/
void fbbuf_init_write(struct fastbuf *f, byte *buffer, uns size);
-static inline uns fbbuf_count_written(struct fastbuf *f) /** How many bytes were written into the buffer already? **/
+
+static inline uns fbbuf_count_written(struct fastbuf *f) /** Calculates, how many bytes were already written into the buffer. **/
{
return f->bptr - f->bstop;
}
-/*** === FastIO on recyclable growing buffers ***/
+/***
+ * === Fastbuf on recyclable growing buffers [[fbgrow]]
+ *
+ * The `fbgrow` back-end keeps the stream in a contiguous buffer stored in the
+ * main memory, but unlike xref:fbmem[`fbmem`], the buffer does not have a fixed
+ * size and it is expanded to accomodate all data.
+ *
+ * At every moment, you can use `fastbuf->buffer` to gain access to the stream.
+ ***/
-struct fastbuf *fbgrow_create(unsigned basic_size); /** Create the growing buffer, with pre-allocated size @basic_size. **/
+struct fastbuf *fbgrow_create(unsigned basic_size); /** Create the growing buffer pre-allocated to @basic_size bytes. **/
void fbgrow_reset(struct fastbuf *b); /** Reset stream and prepare for writing. **/
void fbgrow_rewind(struct fastbuf *b); /** Prepare for reading (of already written data). **/
/***
- * FastO on memory pools
- * ~~~~~~~~~~~~~~~~~~~~~
- * You write to it and get buffers of written data.
+ * === Fastbuf on memory pools [[fbpool]]
+ *
+ * The write-only `fbpool` back-end also keeps the stream in a contiguous
+ * buffer, but this time the buffer is allocated from within a memory pool.
***/
struct mempool;
struct mempool *mp;
};
+/**
+ * Initialize a new `fbpool`. The structure is allocated by the caller.
+ **/
void fbpool_init(struct fbpool *fb); /** Initialize a new mempool fastbuf. **/
/**
* Start a new continuous block and prepare for writing (see mp_start()).
- * Provide the memory pool you want to use for this block (in @mp).
+ * Provide the memory pool you want to use for this block as @mp.
**/
void fbpool_start(struct fbpool *fb, struct mempool *mp, uns init_size);
/**
- * Close the block and return its address (see mp_end()).
- * The length can be determined with mp_size(mp, ptr).
+ * Close the block and return the address of its start (see mp_end()).
+ * The length can be determined by calling mp_size(mp, ptr).
**/
void *fbpool_end(struct fbpool *fb);
/***
- * === FastO with atomic writes for multi-threaded programs
- * Use them, when you need to write records into single file from several threads.
- * It does not ensure order of the records, but they will not intersect.
+ * === Atomic files for multi-threaded programs [[fbatomic]]
+ *
+ * This fastbuf backend is designed for cases when several threads
+ * of a single program append records to a common file and while the
+ * record can mix in an arbitrary way, the bytes inside a single
+ * record must remain uninterrupted.
+ *
+ * In case of files with fixed record size, we just allocate the
+ * buffer to hold a whole number of records and take advantage
+ * of the atomicity of the write() system call.
+ *
+ * With variable-sized records, we need another solution: when
+ * writing a record, we keep the fastbuf in a locked state, which
+ * prevents buffer flushing (and if the buffer becomes full, we extend it),
+ * and we wait for an explicit commit operation which write()s the buffer
+ * if the free space in the buffer falls below the expected maximum record
+ * length.
+ *
+ * Please note that initialization of the clones is not thread-safe,
+ * so you have to serialize it yourself.
***/
struct fb_atomic {
/**
* Open an atomic fastbuf.
- * If you specify @master, it is used to write into it (both the master
- * and the new one will be the same file, with separate buffers).
- * If @master is +NULL+, a file @name is opened.
+ * If @master is `NULL`, the file @name is opened. If it is non-null,
+ * a new clone of an existing atomic fastbuf is created.
+ *
+ * If the file has fixed record length, just set @record_len to it.
+ * Otherwise set @record_len to the expected maximum record length
+ * with a negative sign (you need not fit in this length, but as long
+ * as you do, the fastbuf is more efficient) and call fbatomic_commit()
+ * after each record.
*
* You can specify @record_len, if it is known (for optimisations).
*
void fbatomic_internal_write(struct fastbuf *b);
/**
- * Commit the last record.
- * It may not yet write it to the file, but it will stay together.
+ * Declare that you have finished writing a record. This is required only
+ * if a fixed record size was not specified.
**/
static inline void fbatomic_commit(struct fastbuf *b)
{
fbatomic_internal_write(b);
}
-/*** === Configuring stream parameters ***/
+/*** === Configuring stream parameters [[bconfig]] ***/
enum bconfig_type { /** Parameters that could be configured. **/
BCONFIG_IS_TEMP_FILE, /* 0=normal file, 1=temporary file, 2=shared fd */
int bconfig(struct fastbuf *f, uns type, int data); /** Configure a fastbuf. Returns previous value. **/
-/*** === Universal functions working on all fastbuf's ***/
+/*** === Universal functions working on all fastbuf's [[ffbasic]] ***/
/**
* Close and free fastbuf.
projects / libucw.git / commitdiff