JSON: Improved memory management with pushing/popping of states

diff --git a/maint/libucw.abi b/maint/libucw.abi
index 9b8c2b53a451a1d81ce56c66aad1ad4d287905b9..0e4deffd83cff302392048ce1686af1d22bc1ea2 100644 (file)
--- a/maint/libucw.abi
+++ b/maint/libucw.abi
@@ -888,17 +888,19 @@ xml_ns_by_name
 json_new
 json_delete
 json_reset
+json_push
+json_pop
 json_new_node
 json_new_array
 json_array_append
 json_new_object
 json_object_set
 json_object_get
+json_parse
 json_set_input
-json_peek_token
 json_next_token
+json_peek_token
 json_next_value
-json_parse
+json_write
 json_set_output
 json_write_value
-json_write

diff --git a/ucw-json/doc/json.txt b/ucw-json/doc/json.txt
index a4e77ee290614af4f488c85838959a6cb16adf9e..de4725e2a439643ff47e8a7aa488ff5bd4c2c874 100644 (file)
--- a/ucw-json/doc/json.txt
+++ b/ucw-json/doc/json.txt
@@ -4,9 +4,4 @@ JSON Parser and Generator
 ucw-json/json.h
 ---------------
 
-To use the JSON library, you need to create a context (<<struct json_context,struct json_context>>)
-by calling <<json_new()>> first.
-
-FIXME
-
 !!ucw-json/json.h

diff --git a/ucw-json/json-test.c b/ucw-json/json-test.c
index 0b6d7a0697c638cd34432dde3e03c2280dc8bdd1..5d4c28753dc5efcd0c2ea95ab01aa5b027e508ca 100644 (file)
--- a/ucw-json/json-test.c
+++ b/ucw-json/json-test.c
@@ -20,6 +20,7 @@ static int opt_escape;
 static int opt_indent;
 static int opt_read_hex;
 static int opt_write_hex;
+static int opt_stream;
 
 static struct opt_section options = {
   OPT_ITEMS {
@@ -34,6 +35,7 @@ static struct opt_section options = {
     OPT_BOOL('W', "write-hex", opt_write_hex, 0, "\tWrite JSON, print non-ASCII as hex escapes"),
     OPT_BOOL('e', "escape", opt_escape, 0, "\tEscape non-ASCII characters in strings"),
     OPT_BOOL('i', "indent", opt_indent, 0, "\tIndent output"),
+    OPT_BOOL('s', "stream", opt_stream, 0, "\tTest of streaming mode"),
     OPT_END
   }
 };
@@ -54,6 +56,28 @@ static struct json_node *do_parse(struct json_context *js, struct fastbuf *fb)
   return n;
 }
 
+static void test_stream(struct json_context *js)
+{
+  struct fastbuf *in = bfdopen_shared(0, 65536);
+  struct fastbuf *out = bfdopen_shared(1, 65536);
+  json_set_input(js, in);
+  json_set_output(js, out);
+
+  for (;;)
+    {
+      json_push(js);
+      struct json_node *n = json_next_value(js);
+      if (!n)
+       break;
+      json_write_value(js, n);
+      bputc(out, '\n');
+      json_pop(js);
+    }
+
+  bclose(out);
+  bclose(in);
+}
+
 int main(int argc UNUSED, char **argv)
 {
   opt_parse(&options, argv+1);
@@ -66,6 +90,13 @@ int main(int argc UNUSED, char **argv)
   if (opt_indent)
     js->format_options |= JSON_FORMAT_INDENT;
 
+  if (opt_stream)
+    {
+      test_stream(js);
+      json_delete(js);
+      return 0;
+    }
+
   if (opt_read || opt_read_hex)
     {
       struct fastbuf *fb = bfdopen_shared(0, 65536);

diff --git a/ucw-json/json-test.t b/ucw-json/json-test.t
index cd6f5c7277453bca2b0dec2a68eff2bb9e27fe50..cece0d233ab8db3a2f7e764ff01437e5de395d1c 100644 (file)
--- a/ucw-json/json-test.t
+++ b/ucw-json/json-test.t
@@ -377,3 +377,12 @@ Name:      Multiple values
 In:    1 2
 Exit:  1
 Err:   ERROR: Only one top-level value allowed at line 1:4
+
+### Streaming interface ###
+
+Name:  Streaming
+Run:   ../obj/ucw-json/json-test -s
+In:    123 [true, false] "Rincewind"
+Out:   123
+       [ true, false ]
+       "Rincewind"

diff --git a/ucw-json/json.c b/ucw-json/json.c
index 3661a735c3d07809ed6686595af0701ad3fbdfd8..fbcd5278c389056c0a8ab09a06c84b5df95f84db 100644 (file)
--- a/ucw-json/json.c
+++ b/ucw-json/json.c
@@ -15,6 +15,7 @@
 static void json_init(struct json_context *js)
 {
   mp_save(js->pool, &js->init_state);
+  js->trivial_token = json_new_node(js, JSON_INVALID);
 }
 
 struct json_context *json_new(void)
@@ -40,6 +41,18 @@ void json_reset(struct json_context *js)
   json_init(js);
 }
 
+void json_push(struct json_context *js)
+{
+  ASSERT(!js->next_token);
+  mp_push(js->pool);
+}
+
+void json_pop(struct json_context *js)
+{
+  ASSERT(!js->next_token);
+  mp_pop(js->pool);
+}
+
 struct json_node *json_new_node(struct json_context *js, enum json_node_type type)
 {
   struct json_node *n = mp_alloc_fast(js->pool, sizeof(*n));

diff --git a/ucw-json/json.h b/ucw-json/json.h
index aff1fccc9e8ab0e07449aaf2b4da45d1f39c773c..74620498387ba588bf822ce80bc1badda066fe5c 100644 (file)
--- a/ucw-json/json.h
+++ b/ucw-json/json.h
@@ -28,6 +28,8 @@
 #define json_object_set ucw_json_object_set
 #define json_parse ucw_json_parse
 #define json_peek_token ucw_json_peek_token
+#define json_pop ucw_json_pop
+#define json_push ucw_json_push
 #define json_reset ucw_json_reset
 #define json_set_input ucw_json_set_input
 #define json_set_output ucw_json_set_output
@@ -38,7 +40,15 @@
 /***
  * === JSON library context
  *
- * FIXME: Document memory management
+ * The context structure remembers the whole state of the JSON
+ * library. All JSON values are allocated from a memory pool associated
+ * with the context. By default, their lifetime is the same as that
+ * of the context.
+ *
+ * Alternatively, you can mark the current state of the context
+ * with json_push() and return to the marked state later using
+ * json_pop(). All JSON values created between these two operations
+ * are released afterwards. See json_push() for details.
  ***/
 
 /**
@@ -78,6 +88,25 @@ void json_delete(struct json_context *js);
  **/
 void json_reset(struct json_context *js);
 
+/**
+ * Push the current state of the context onto state stack.
+ *
+ * Between json_push() and the associated json_pop(), only newly
+ * created JSON values can be modified. Older values can be only
+ * inspected, never modified. In particular, new values cannot be
+ * inserted to old arrays nor objects.
+ *
+ * If you are using json_peek_token(), the saved tokens cannot
+ * be carried over push/pop boundary.
+ **/
+void json_push(struct json_context *js);
+
+/**
+ * Pop state of the context off state stack. All JSON values created
+ * since the state was saved by json_push() are released.
+ **/
+void json_pop(struct json_context *js);
+
 /***
  * === JSON values
  *
@@ -204,9 +233,9 @@ struct json_node *json_object_get(struct json_node *n, const char *key);
  * which returns a value tree representing the contents of the file.
  *
  * Alternatively, you can read the input token by token: call json_set_input()
- * and then repeat json_next_token(). Note that even in this mode, the tokens
- * are kept in memory. (If you need to parse huge JSON files, please ask the
- * maintainer of this library to improve memory allocation.)
+ * and then repeat json_next_token(). If you are parsing huge JSON files,
+ * you probably want to do json_push() first, then scan and process some
+ * tokens, and then json_pop().
  ***/
 
 /** Parses a JSON file from the given fastbuf stream. **/

diff --git a/ucw-json/parse.c b/ucw-json/parse.c
index 96c916f3c205fdac71738637ad89642013d5d943..8ed97e3b423fa31b5f4fbea9c8680a1274e53936 100644 (file)
--- a/ucw-json/parse.c
+++ b/ucw-json/parse.c
@@ -25,8 +25,6 @@ void json_set_input(struct json_context *js, struct fastbuf *in)
   js->next_char = -1;
   js->next_token = NULL;
   js->in_eof = 0;
-  if (!js->trivial_token)
-    js->trivial_token = json_new_node(js, JSON_INVALID);
 }
 
 static void NONRET json_parse_error(struct json_context *js, const char *msg)