From e1aae876430ba9497ba1a524f315c5dee6cf285a Mon Sep 17 00:00:00 2001 From: Martin Mares Date: Wed, 8 Jul 2015 23:11:04 +0200 Subject: [PATCH] JSON: Improved memory management with pushing/popping of states --- maint/libucw.abi | 8 +++++--- ucw-json/doc/json.txt | 5 ----- ucw-json/json-test.c | 31 +++++++++++++++++++++++++++++++ ucw-json/json-test.t | 9 +++++++++ ucw-json/json.c | 13 +++++++++++++ ucw-json/json.h | 37 +++++++++++++++++++++++++++++++++---- ucw-json/parse.c | 2 -- 7 files changed, 91 insertions(+), 14 deletions(-) diff --git a/maint/libucw.abi b/maint/libucw.abi index 9b8c2b53..0e4deffd 100644 --- a/maint/libucw.abi +++ b/maint/libucw.abi @@ -888,17 +888,19 @@ xml_ns_by_name json_new json_delete json_reset +json_push +json_pop json_new_node json_new_array json_array_append json_new_object json_object_set json_object_get +json_parse json_set_input -json_peek_token json_next_token +json_peek_token json_next_value -json_parse +json_write json_set_output json_write_value -json_write diff --git a/ucw-json/doc/json.txt b/ucw-json/doc/json.txt index a4e77ee2..de4725e2 100644 --- a/ucw-json/doc/json.txt +++ b/ucw-json/doc/json.txt @@ -4,9 +4,4 @@ JSON Parser and Generator ucw-json/json.h --------------- -To use the JSON library, you need to create a context (<>) -by calling <> first. - -FIXME - !!ucw-json/json.h diff --git a/ucw-json/json-test.c b/ucw-json/json-test.c index 0b6d7a06..5d4c2875 100644 --- a/ucw-json/json-test.c +++ b/ucw-json/json-test.c @@ -20,6 +20,7 @@ static int opt_escape; static int opt_indent; static int opt_read_hex; static int opt_write_hex; +static int opt_stream; static struct opt_section options = { OPT_ITEMS { @@ -34,6 +35,7 @@ static struct opt_section options = { OPT_BOOL('W', "write-hex", opt_write_hex, 0, "\tWrite JSON, print non-ASCII as hex escapes"), OPT_BOOL('e', "escape", opt_escape, 0, "\tEscape non-ASCII characters in strings"), OPT_BOOL('i', "indent", opt_indent, 0, "\tIndent output"), + OPT_BOOL('s', "stream", opt_stream, 0, "\tTest of streaming mode"), OPT_END } }; @@ -54,6 +56,28 @@ static struct json_node *do_parse(struct json_context *js, struct fastbuf *fb) return n; } +static void test_stream(struct json_context *js) +{ + struct fastbuf *in = bfdopen_shared(0, 65536); + struct fastbuf *out = bfdopen_shared(1, 65536); + json_set_input(js, in); + json_set_output(js, out); + + for (;;) + { + json_push(js); + struct json_node *n = json_next_value(js); + if (!n) + break; + json_write_value(js, n); + bputc(out, '\n'); + json_pop(js); + } + + bclose(out); + bclose(in); +} + int main(int argc UNUSED, char **argv) { opt_parse(&options, argv+1); @@ -66,6 +90,13 @@ int main(int argc UNUSED, char **argv) if (opt_indent) js->format_options |= JSON_FORMAT_INDENT; + if (opt_stream) + { + test_stream(js); + json_delete(js); + return 0; + } + if (opt_read || opt_read_hex) { struct fastbuf *fb = bfdopen_shared(0, 65536); diff --git a/ucw-json/json-test.t b/ucw-json/json-test.t index cd6f5c72..cece0d23 100644 --- a/ucw-json/json-test.t +++ b/ucw-json/json-test.t @@ -377,3 +377,12 @@ Name: Multiple values In: 1 2 Exit: 1 Err: ERROR: Only one top-level value allowed at line 1:4 + +### Streaming interface ### + +Name: Streaming +Run: ../obj/ucw-json/json-test -s +In: 123 [true, false] "Rincewind" +Out: 123 + [ true, false ] + "Rincewind" diff --git a/ucw-json/json.c b/ucw-json/json.c index 3661a735..fbcd5278 100644 --- a/ucw-json/json.c +++ b/ucw-json/json.c @@ -15,6 +15,7 @@ static void json_init(struct json_context *js) { mp_save(js->pool, &js->init_state); + js->trivial_token = json_new_node(js, JSON_INVALID); } struct json_context *json_new(void) @@ -40,6 +41,18 @@ void json_reset(struct json_context *js) json_init(js); } +void json_push(struct json_context *js) +{ + ASSERT(!js->next_token); + mp_push(js->pool); +} + +void json_pop(struct json_context *js) +{ + ASSERT(!js->next_token); + mp_pop(js->pool); +} + struct json_node *json_new_node(struct json_context *js, enum json_node_type type) { struct json_node *n = mp_alloc_fast(js->pool, sizeof(*n)); diff --git a/ucw-json/json.h b/ucw-json/json.h index aff1fccc..74620498 100644 --- a/ucw-json/json.h +++ b/ucw-json/json.h @@ -28,6 +28,8 @@ #define json_object_set ucw_json_object_set #define json_parse ucw_json_parse #define json_peek_token ucw_json_peek_token +#define json_pop ucw_json_pop +#define json_push ucw_json_push #define json_reset ucw_json_reset #define json_set_input ucw_json_set_input #define json_set_output ucw_json_set_output @@ -38,7 +40,15 @@ /*** * === JSON library context * - * FIXME: Document memory management + * The context structure remembers the whole state of the JSON + * library. All JSON values are allocated from a memory pool associated + * with the context. By default, their lifetime is the same as that + * of the context. + * + * Alternatively, you can mark the current state of the context + * with json_push() and return to the marked state later using + * json_pop(). All JSON values created between these two operations + * are released afterwards. See json_push() for details. ***/ /** @@ -78,6 +88,25 @@ void json_delete(struct json_context *js); **/ void json_reset(struct json_context *js); +/** + * Push the current state of the context onto state stack. + * + * Between json_push() and the associated json_pop(), only newly + * created JSON values can be modified. Older values can be only + * inspected, never modified. In particular, new values cannot be + * inserted to old arrays nor objects. + * + * If you are using json_peek_token(), the saved tokens cannot + * be carried over push/pop boundary. + **/ +void json_push(struct json_context *js); + +/** + * Pop state of the context off state stack. All JSON values created + * since the state was saved by json_push() are released. + **/ +void json_pop(struct json_context *js); + /*** * === JSON values * @@ -204,9 +233,9 @@ struct json_node *json_object_get(struct json_node *n, const char *key); * which returns a value tree representing the contents of the file. * * Alternatively, you can read the input token by token: call json_set_input() - * and then repeat json_next_token(). Note that even in this mode, the tokens - * are kept in memory. (If you need to parse huge JSON files, please ask the - * maintainer of this library to improve memory allocation.) + * and then repeat json_next_token(). If you are parsing huge JSON files, + * you probably want to do json_push() first, then scan and process some + * tokens, and then json_pop(). ***/ /** Parses a JSON file from the given fastbuf stream. **/ diff --git a/ucw-json/parse.c b/ucw-json/parse.c index 96c916f3..8ed97e3b 100644 --- a/ucw-json/parse.c +++ b/ucw-json/parse.c @@ -25,8 +25,6 @@ void json_set_input(struct json_context *js, struct fastbuf *in) js->next_char = -1; js->next_token = NULL; js->in_eof = 0; - if (!js->trivial_token) - js->trivial_token = json_new_node(js, JSON_INVALID); } static void NONRET json_parse_error(struct json_context *js, const char *msg) -- 2.39.2