tranfi 0.1.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -0
- package/NOTICE +8 -0
- package/README.md +443 -51
- package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
- package/app/index.html +1 -1
- package/binding.gyp +55 -3
- package/csrc/arena.c +7 -5
- package/csrc/batch.c +818 -71
- package/csrc/buffer.c +84 -8
- package/csrc/cJSON.c +262 -19
- package/csrc/cJSON.h +17 -1
- package/csrc/codec_csv.c +1074 -181
- package/csrc/codec_jsonl.c +830 -118
- package/csrc/codec_table.c +108 -78
- package/csrc/codec_text.c +286 -68
- package/csrc/compiler.c +31 -3
- package/csrc/config.h +21 -0
- package/csrc/dsl.c +4722 -485
- package/csrc/expr.c +363 -55
- package/csrc/expr.h +2 -0
- package/csrc/internal.h +316 -27
- package/csrc/ir.c +65 -18
- package/csrc/ir.h +41 -0
- package/csrc/ir_schema.c +20 -5
- package/csrc/ir_serialize.c +68 -6
- package/csrc/ir_sql.c +796 -185
- package/csrc/ir_validate.c +462 -6
- package/csrc/json_path.c +210 -0
- package/csrc/main.c +879 -30
- package/csrc/memory_estimate.c +477 -0
- package/csrc/op_acf.c +171 -21
- package/csrc/op_across.c +477 -0
- package/csrc/op_anomaly.c +167 -32
- package/csrc/op_assert.c +761 -0
- package/csrc/op_bin.c +168 -29
- package/csrc/op_cast.c +383 -55
- package/csrc/op_clip.c +30 -19
- package/csrc/op_date_trunc.c +208 -34
- package/csrc/op_datetime.c +259 -77
- package/csrc/op_derive.c +65 -97
- package/csrc/op_diff.c +146 -30
- package/csrc/op_ewma.c +149 -30
- package/csrc/op_explode.c +124 -26
- package/csrc/op_fill_down.c +125 -53
- package/csrc/op_fill_null.c +176 -31
- package/csrc/op_filter.c +89 -40
- package/csrc/op_frequency.c +571 -43
- package/csrc/op_grep.c +36 -18
- package/csrc/op_group_agg.c +1790 -119
- package/csrc/op_hash.c +48 -15
- package/csrc/op_head.c +21 -86
- package/csrc/op_interpolate.c +268 -62
- package/csrc/op_join.c +2700 -182
- package/csrc/op_json_extract.c +227 -0
- package/csrc/op_json_filter.c +384 -0
- package/csrc/op_json_flatten.c +293 -0
- package/csrc/op_json_schema.c +503 -0
- package/csrc/op_label_encode.c +328 -53
- package/csrc/op_lag.c +181 -0
- package/csrc/op_lead.c +141 -89
- package/csrc/op_normalize.c +363 -79
- package/csrc/op_onehot.c +345 -73
- package/csrc/op_pivot.c +1546 -162
- package/csrc/op_quarantine.c +189 -0
- package/csrc/op_registry.c +2062 -166
- package/csrc/op_rename.c +41 -50
- package/csrc/op_replace.c +270 -118
- package/csrc/op_rleid.c +297 -0
- package/csrc/op_rowid.c +559 -0
- package/csrc/op_sample.c +80 -23
- package/csrc/op_schema.c +1341 -0
- package/csrc/op_schema_infer.c +252 -0
- package/csrc/op_select.c +265 -65
- package/csrc/op_set.c +3449 -0
- package/csrc/op_skip.c +30 -87
- package/csrc/op_sort.c +670 -124
- package/csrc/op_source_name.c +120 -0
- package/csrc/op_split.c +65 -28
- package/csrc/op_split_data.c +41 -9
- package/csrc/op_stack.c +178 -222
- package/csrc/op_stats.c +206 -110
- package/csrc/op_step.c +217 -55
- package/csrc/op_tail.c +21 -12
- package/csrc/op_tee.c +338 -0
- package/csrc/op_top.c +260 -53
- package/csrc/op_trim.c +48 -19
- package/csrc/op_unique.c +1193 -150
- package/csrc/op_unpivot.c +100 -66
- package/csrc/op_validate.c +601 -24
- package/csrc/op_window.c +492 -51
- package/csrc/path_policy.c +85 -0
- package/csrc/pipeline.c +872 -99
- package/csrc/recipes.c +3 -1
- package/csrc/report.c +73 -30
- package/csrc/selector.c +1097 -0
- package/csrc/size_utils.c +352 -0
- package/csrc/spill.c +317 -0
- package/csrc/spill.h +21 -0
- package/csrc/tranfi.h +169 -1
- package/csrc/transform.h +209 -0
- package/csrc/transform_api.c +2237 -0
- package/csrc/transform_categorical.c +923 -0
- package/csrc/transform_internal.h +472 -0
- package/csrc/transform_json.c +3812 -0
- package/csrc/transform_numeric.c +1966 -0
- package/csrc/transform_sha256.c +154 -0
- package/csrc/transform_wasm.h +162 -0
- package/csrc/transform_wasm_api.c +1373 -0
- package/csrc/wasm_api.c +70 -9
- package/napi_api.c +219 -11
- package/napi_transform.c +1648 -0
- package/napi_transform.h +8 -0
- package/package.json +27 -11
- package/scripts/install-native.js +76 -0
- package/scripts/prepack.js +64 -0
- package/scripts/sync-csrc.js +23 -0
- package/src/cli.js +81 -41
- package/src/engines/duckdb.js +45 -12
- package/src/index.js +661 -42
- package/src/memory_policy.js +411 -0
- package/src/native.js +1 -5
- package/src/pipeline.js +454 -31
- package/src/recipe_json.js +80 -0
- package/src/server.js +10 -8
- package/src/transform.js +403 -0
- package/src/transform_error.js +10 -0
- package/src/wasm.js +6 -4
- package/wasm/index.js +498 -10
- package/wasm/tranfi_core.js +0 -0
- package/wasm/transform.js +1156 -0
- package/wasm/worker.js +786 -0
- package/csrc/plan.c +0 -206
package/csrc/buffer.c
CHANGED
|
@@ -4,10 +4,13 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
#include "internal.h"
|
|
7
|
+
#include "cJSON.h"
|
|
7
8
|
#include <stdlib.h>
|
|
8
9
|
#include <string.h>
|
|
9
10
|
|
|
10
11
|
#define INITIAL_CAP 4096
|
|
12
|
+
#define COMPACT_AFTER_READ_POS (64 * 1024)
|
|
13
|
+
#define MAX_RETAINED_EMPTY_CAP (1024 * 1024)
|
|
11
14
|
|
|
12
15
|
void tf_buffer_init(tf_buffer *b) {
|
|
13
16
|
b->data = NULL;
|
|
@@ -16,12 +19,11 @@ void tf_buffer_init(tf_buffer *b) {
|
|
|
16
19
|
b->read_pos = 0;
|
|
17
20
|
}
|
|
18
21
|
|
|
19
|
-
static int buffer_ensure(tf_buffer *b, size_t
|
|
20
|
-
size_t needed = b->len + extra;
|
|
22
|
+
static int buffer_ensure(tf_buffer *b, size_t needed) {
|
|
21
23
|
if (needed <= b->cap) return TF_OK;
|
|
22
|
-
size_t new_cap =
|
|
23
|
-
|
|
24
|
-
uint8_t *new_data =
|
|
24
|
+
size_t new_cap = 0;
|
|
25
|
+
if (tf_size_grow_pow2(b->cap, needed, INITIAL_CAP, &new_cap) != TF_OK) return TF_ERROR;
|
|
26
|
+
uint8_t *new_data = tf_reallocarray_checked(b->data, new_cap, sizeof(uint8_t));
|
|
25
27
|
if (!new_data) return TF_ERROR;
|
|
26
28
|
b->data = new_data;
|
|
27
29
|
b->cap = new_cap;
|
|
@@ -30,9 +32,11 @@ static int buffer_ensure(tf_buffer *b, size_t extra) {
|
|
|
30
32
|
|
|
31
33
|
int tf_buffer_write(tf_buffer *b, const uint8_t *data, size_t len) {
|
|
32
34
|
if (len == 0) return TF_OK;
|
|
33
|
-
|
|
35
|
+
size_t new_len = 0;
|
|
36
|
+
if (tf_size_add(b->len, len, &new_len) != TF_OK) return TF_ERROR;
|
|
37
|
+
if (buffer_ensure(b, new_len) != TF_OK) return TF_ERROR;
|
|
34
38
|
memcpy(b->data + b->len, data, len);
|
|
35
|
-
b->len
|
|
39
|
+
b->len = new_len;
|
|
36
40
|
return TF_OK;
|
|
37
41
|
}
|
|
38
42
|
|
|
@@ -40,6 +44,67 @@ int tf_buffer_write_str(tf_buffer *b, const char *s) {
|
|
|
40
44
|
return tf_buffer_write(b, (const uint8_t *)s, strlen(s));
|
|
41
45
|
}
|
|
42
46
|
|
|
47
|
+
int tf_buffer_write_line(tf_buffer *b, const char *s) {
|
|
48
|
+
int rc = tf_buffer_write_str(b, s);
|
|
49
|
+
if (rc == TF_OK) rc = tf_buffer_write(b, (const uint8_t *)"\n", 1);
|
|
50
|
+
return rc;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
int tf_buffer_write_json_line(tf_buffer *b, const cJSON *obj) {
|
|
54
|
+
if (!b || !obj) return TF_ERROR;
|
|
55
|
+
char *printed = cJSON_PrintUnformatted(obj);
|
|
56
|
+
if (!printed) return TF_ERROR;
|
|
57
|
+
int rc = tf_buffer_write_line(b, printed);
|
|
58
|
+
free(printed);
|
|
59
|
+
return rc;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
int tf_side_write_error(tf_side_channels *side, const char *msg) {
|
|
63
|
+
if (msg) tf_set_last_error(msg);
|
|
64
|
+
if (!side || !side->errors || !msg) return TF_OK;
|
|
65
|
+
return tf_buffer_write_line(side->errors, msg);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
int tf_json_add_string(cJSON *obj, const char *name, const char *value) {
|
|
69
|
+
return cJSON_AddStringToObject(obj, name, value ? value : "") ? TF_OK : TF_ERROR;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
int tf_json_add_number(cJSON *obj, const char *name, double value) {
|
|
73
|
+
return cJSON_AddNumberToObject(obj, name, value) ? TF_OK : TF_ERROR;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
int tf_json_add_bool(cJSON *obj, const char *name, int value) {
|
|
77
|
+
return cJSON_AddBoolToObject(obj, name, value ? 1 : 0) ? TF_OK : TF_ERROR;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
int tf_json_add_null(cJSON *obj, const char *name) {
|
|
81
|
+
return cJSON_AddNullToObject(obj, name) ? TF_OK : TF_ERROR;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
int tf_json_add_item(cJSON *obj, const char *name, cJSON *item) {
|
|
85
|
+
if (!obj || !name || !item) {
|
|
86
|
+
cJSON_Delete(item);
|
|
87
|
+
return TF_ERROR;
|
|
88
|
+
}
|
|
89
|
+
if (!cJSON_AddItemToObject(obj, name, item)) {
|
|
90
|
+
cJSON_Delete(item);
|
|
91
|
+
return TF_ERROR;
|
|
92
|
+
}
|
|
93
|
+
return TF_OK;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
int tf_json_add_array_item(cJSON *arr, cJSON *item) {
|
|
97
|
+
if (!arr || !item) {
|
|
98
|
+
cJSON_Delete(item);
|
|
99
|
+
return TF_ERROR;
|
|
100
|
+
}
|
|
101
|
+
if (!cJSON_AddItemToArray(arr, item)) {
|
|
102
|
+
cJSON_Delete(item);
|
|
103
|
+
return TF_ERROR;
|
|
104
|
+
}
|
|
105
|
+
return TF_OK;
|
|
106
|
+
}
|
|
107
|
+
|
|
43
108
|
size_t tf_buffer_read(tf_buffer *b, uint8_t *out, size_t len) {
|
|
44
109
|
size_t avail = b->len - b->read_pos;
|
|
45
110
|
if (len > avail) len = avail;
|
|
@@ -47,10 +112,21 @@ size_t tf_buffer_read(tf_buffer *b, uint8_t *out, size_t len) {
|
|
|
47
112
|
memcpy(out, b->data + b->read_pos, len);
|
|
48
113
|
b->read_pos += len;
|
|
49
114
|
}
|
|
50
|
-
/* Auto-compact when all data consumed
|
|
115
|
+
/* Auto-compact when all data consumed. Keep a modest scratch buffer for
|
|
116
|
+
* future chunks, but release one-off large output buffers after drains. */
|
|
51
117
|
if (b->read_pos == b->len) {
|
|
52
118
|
b->read_pos = 0;
|
|
53
119
|
b->len = 0;
|
|
120
|
+
if (b->cap > MAX_RETAINED_EMPTY_CAP) {
|
|
121
|
+
uint8_t *new_data = tf_reallocarray_checked(b->data, MAX_RETAINED_EMPTY_CAP,
|
|
122
|
+
sizeof(uint8_t));
|
|
123
|
+
if (new_data) {
|
|
124
|
+
b->data = new_data;
|
|
125
|
+
b->cap = MAX_RETAINED_EMPTY_CAP;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
} else if (b->read_pos >= COMPACT_AFTER_READ_POS && b->read_pos > b->cap / 2) {
|
|
129
|
+
tf_buffer_compact(b);
|
|
54
130
|
}
|
|
55
131
|
return len;
|
|
56
132
|
}
|
package/csrc/cJSON.c
CHANGED
|
@@ -42,6 +42,7 @@
|
|
|
42
42
|
#include <math.h>
|
|
43
43
|
#include <stdlib.h>
|
|
44
44
|
#include <limits.h>
|
|
45
|
+
#include <stdint.h>
|
|
45
46
|
#include <ctype.h>
|
|
46
47
|
#include <float.h>
|
|
47
48
|
|
|
@@ -89,13 +90,40 @@ typedef struct {
|
|
|
89
90
|
const unsigned char *json;
|
|
90
91
|
size_t position;
|
|
91
92
|
} error;
|
|
92
|
-
|
|
93
|
+
|
|
94
|
+
/* Tranfi local patch: parse errors are thread-local for concurrent embedders. */
|
|
95
|
+
#if defined(_MSC_VER)
|
|
96
|
+
#define CJSON_THREAD_LOCAL __declspec(thread)
|
|
97
|
+
#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
|
|
98
|
+
#define CJSON_THREAD_LOCAL _Thread_local
|
|
99
|
+
#elif defined(__GNUC__) || defined(__clang__)
|
|
100
|
+
#define CJSON_THREAD_LOCAL __thread
|
|
101
|
+
#else
|
|
102
|
+
#define CJSON_THREAD_LOCAL
|
|
103
|
+
#endif
|
|
104
|
+
|
|
105
|
+
static CJSON_THREAD_LOCAL error global_error = { NULL, 0 };
|
|
106
|
+
static CJSON_THREAD_LOCAL cJSON_bool global_parse_tracking = false;
|
|
107
|
+
static CJSON_THREAD_LOCAL cJSON_bool global_parse_allocation_failed = false;
|
|
108
|
+
|
|
109
|
+
static void cJSON_mark_parse_allocation_failure(void)
|
|
110
|
+
{
|
|
111
|
+
if (global_parse_tracking)
|
|
112
|
+
{
|
|
113
|
+
global_parse_allocation_failed = true;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
93
116
|
|
|
94
117
|
CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void)
|
|
95
118
|
{
|
|
96
119
|
return (const char*) (global_error.json + global_error.position);
|
|
97
120
|
}
|
|
98
121
|
|
|
122
|
+
CJSON_PUBLIC(cJSON_bool) cJSON_ParseHadAllocationFailure(void)
|
|
123
|
+
{
|
|
124
|
+
return global_parse_allocation_failed;
|
|
125
|
+
}
|
|
126
|
+
|
|
99
127
|
CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item)
|
|
100
128
|
{
|
|
101
129
|
if (!cJSON_IsString(item))
|
|
@@ -117,7 +145,7 @@ CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item)
|
|
|
117
145
|
}
|
|
118
146
|
|
|
119
147
|
/* This is a safeguard to prevent copy-pasters from using incompatible C and header files */
|
|
120
|
-
#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH !=
|
|
148
|
+
#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 19)
|
|
121
149
|
#error cJSON.h and cJSON.c have different versions. Make sure that both have the same.
|
|
122
150
|
#endif
|
|
123
151
|
|
|
@@ -199,6 +227,7 @@ static unsigned char* cJSON_strdup(const unsigned char* string, const internal_h
|
|
|
199
227
|
copy = (unsigned char*)hooks->allocate(length);
|
|
200
228
|
if (copy == NULL)
|
|
201
229
|
{
|
|
230
|
+
cJSON_mark_parse_allocation_failure();
|
|
202
231
|
return NULL;
|
|
203
232
|
}
|
|
204
233
|
memcpy(copy, string, length);
|
|
@@ -245,6 +274,10 @@ static cJSON *cJSON_New_Item(const internal_hooks * const hooks)
|
|
|
245
274
|
{
|
|
246
275
|
memset(node, '\0', sizeof(cJSON));
|
|
247
276
|
}
|
|
277
|
+
else
|
|
278
|
+
{
|
|
279
|
+
cJSON_mark_parse_allocation_failure();
|
|
280
|
+
}
|
|
248
281
|
|
|
249
282
|
return node;
|
|
250
283
|
}
|
|
@@ -293,8 +326,69 @@ typedef struct
|
|
|
293
326
|
size_t offset;
|
|
294
327
|
size_t depth; /* How deeply nested (in arrays/objects) is the input at the current offset. */
|
|
295
328
|
internal_hooks hooks;
|
|
329
|
+
cJSON_ParsePollFn poll;
|
|
330
|
+
void *poll_user_data;
|
|
331
|
+
size_t poll_byte_interval;
|
|
332
|
+
size_t poll_iteration_interval;
|
|
333
|
+
size_t poll_bytes;
|
|
334
|
+
size_t poll_iterations;
|
|
335
|
+
cJSON_bool poll_failed;
|
|
296
336
|
} parse_buffer;
|
|
297
337
|
|
|
338
|
+
static cJSON_bool parse_poll_progress(
|
|
339
|
+
parse_buffer * const buffer, size_t bytes, size_t iterations)
|
|
340
|
+
{
|
|
341
|
+
if ((buffer == NULL) || buffer->poll_failed)
|
|
342
|
+
{
|
|
343
|
+
return false;
|
|
344
|
+
}
|
|
345
|
+
if (bytes > (SIZE_MAX - buffer->poll_bytes)
|
|
346
|
+
|| iterations > (SIZE_MAX - buffer->poll_iterations))
|
|
347
|
+
{
|
|
348
|
+
buffer->poll_failed = true;
|
|
349
|
+
return false;
|
|
350
|
+
}
|
|
351
|
+
buffer->poll_bytes += bytes;
|
|
352
|
+
buffer->poll_iterations += iterations;
|
|
353
|
+
if (buffer->poll != NULL
|
|
354
|
+
&& ((buffer->poll_bytes >= buffer->poll_byte_interval)
|
|
355
|
+
|| (buffer->poll_iterations >= buffer->poll_iteration_interval)))
|
|
356
|
+
{
|
|
357
|
+
if (!buffer->poll(buffer->poll_user_data))
|
|
358
|
+
{
|
|
359
|
+
buffer->poll_failed = true;
|
|
360
|
+
return false;
|
|
361
|
+
}
|
|
362
|
+
buffer->poll_bytes = 0;
|
|
363
|
+
buffer->poll_iterations = 0;
|
|
364
|
+
}
|
|
365
|
+
return true;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
static void parse_delete(cJSON *item, parse_buffer *buffer)
|
|
369
|
+
{
|
|
370
|
+
cJSON *next = NULL;
|
|
371
|
+
while (item != NULL)
|
|
372
|
+
{
|
|
373
|
+
next = item->next;
|
|
374
|
+
if (!(item->type & cJSON_IsReference) && (item->child != NULL))
|
|
375
|
+
{
|
|
376
|
+
parse_delete(item->child, buffer);
|
|
377
|
+
}
|
|
378
|
+
if (!(item->type & cJSON_IsReference) && (item->valuestring != NULL))
|
|
379
|
+
{
|
|
380
|
+
buffer->hooks.deallocate(item->valuestring);
|
|
381
|
+
}
|
|
382
|
+
if (!(item->type & cJSON_StringIsConst) && (item->string != NULL))
|
|
383
|
+
{
|
|
384
|
+
buffer->hooks.deallocate(item->string);
|
|
385
|
+
}
|
|
386
|
+
buffer->hooks.deallocate(item);
|
|
387
|
+
(void)parse_poll_progress(buffer, 0, 1);
|
|
388
|
+
item = next;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
|
|
298
392
|
/* check if the given size is left to read in a given parse buffer (starting with 1) */
|
|
299
393
|
#define can_read(buffer, size) ((buffer != NULL) && (((buffer)->offset + size) <= (buffer)->length))
|
|
300
394
|
/* check if the buffer can be accessed at the given index (starting with 0) */
|
|
@@ -308,9 +402,11 @@ static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_bu
|
|
|
308
402
|
{
|
|
309
403
|
double number = 0;
|
|
310
404
|
unsigned char *after_end = NULL;
|
|
311
|
-
unsigned char number_c_string
|
|
405
|
+
unsigned char *number_c_string;
|
|
312
406
|
unsigned char decimal_point = get_decimal_point();
|
|
313
407
|
size_t i = 0;
|
|
408
|
+
size_t number_string_length = 0;
|
|
409
|
+
cJSON_bool has_decimal_point = false;
|
|
314
410
|
|
|
315
411
|
if ((input_buffer == NULL) || (input_buffer->content == NULL))
|
|
316
412
|
{
|
|
@@ -320,8 +416,12 @@ static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_bu
|
|
|
320
416
|
/* copy the number into a temporary buffer and replace '.' with the decimal point
|
|
321
417
|
* of the current locale (for strtod)
|
|
322
418
|
* This also takes care of '\0' not necessarily being available for marking the end of the input */
|
|
323
|
-
for (i = 0;
|
|
419
|
+
for (i = 0; can_access_at_index(input_buffer, i); i++)
|
|
324
420
|
{
|
|
421
|
+
if (!parse_poll_progress(input_buffer, 1, 0))
|
|
422
|
+
{
|
|
423
|
+
return false;
|
|
424
|
+
}
|
|
325
425
|
switch (buffer_at_offset(input_buffer)[i])
|
|
326
426
|
{
|
|
327
427
|
case '0':
|
|
@@ -338,11 +438,12 @@ static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_bu
|
|
|
338
438
|
case '-':
|
|
339
439
|
case 'e':
|
|
340
440
|
case 'E':
|
|
341
|
-
|
|
441
|
+
number_string_length++;
|
|
342
442
|
break;
|
|
343
443
|
|
|
344
444
|
case '.':
|
|
345
|
-
|
|
445
|
+
number_string_length++;
|
|
446
|
+
has_decimal_point = true;
|
|
346
447
|
break;
|
|
347
448
|
|
|
348
449
|
default:
|
|
@@ -350,11 +451,50 @@ static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_bu
|
|
|
350
451
|
}
|
|
351
452
|
}
|
|
352
453
|
loop_end:
|
|
353
|
-
|
|
454
|
+
if (!parse_poll_progress(input_buffer, 0, 1))
|
|
455
|
+
{
|
|
456
|
+
return false;
|
|
457
|
+
}
|
|
458
|
+
/* malloc for temporary buffer, add 1 for '\0' */
|
|
459
|
+
number_c_string = (unsigned char *) input_buffer->hooks.allocate(number_string_length + 1);
|
|
460
|
+
if (number_c_string == NULL)
|
|
461
|
+
{
|
|
462
|
+
cJSON_mark_parse_allocation_failure();
|
|
463
|
+
return false; /* allocation failure */
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
for (i = 0; i < number_string_length;)
|
|
467
|
+
{
|
|
468
|
+
size_t remaining = number_string_length - i;
|
|
469
|
+
size_t chunk = remaining < input_buffer->poll_byte_interval
|
|
470
|
+
? remaining : input_buffer->poll_byte_interval;
|
|
471
|
+
memcpy(number_c_string + i, buffer_at_offset(input_buffer) + i, chunk);
|
|
472
|
+
i += chunk;
|
|
473
|
+
if (!parse_poll_progress(input_buffer, chunk, 0))
|
|
474
|
+
{
|
|
475
|
+
input_buffer->hooks.deallocate(number_c_string);
|
|
476
|
+
return false;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
number_c_string[number_string_length] = '\0';
|
|
480
|
+
|
|
481
|
+
if (has_decimal_point)
|
|
482
|
+
{
|
|
483
|
+
for (i = 0; i < number_string_length; i++)
|
|
484
|
+
{
|
|
485
|
+
if (number_c_string[i] == '.')
|
|
486
|
+
{
|
|
487
|
+
/* replace '.' with the decimal point of the current locale (for strtod) */
|
|
488
|
+
number_c_string[i] = decimal_point;
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
}
|
|
354
492
|
|
|
355
493
|
number = strtod((const char*)number_c_string, (char**)&after_end);
|
|
356
494
|
if (number_c_string == after_end)
|
|
357
495
|
{
|
|
496
|
+
/* free the temporary buffer */
|
|
497
|
+
input_buffer->hooks.deallocate(number_c_string);
|
|
358
498
|
return false; /* parse_error */
|
|
359
499
|
}
|
|
360
500
|
|
|
@@ -377,6 +517,8 @@ loop_end:
|
|
|
377
517
|
item->type = cJSON_Number;
|
|
378
518
|
|
|
379
519
|
input_buffer->offset += (size_t)(after_end - number_c_string);
|
|
520
|
+
/* free the temporary buffer */
|
|
521
|
+
input_buffer->hooks.deallocate(number_c_string);
|
|
380
522
|
return true;
|
|
381
523
|
}
|
|
382
524
|
|
|
@@ -403,6 +545,8 @@ CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number)
|
|
|
403
545
|
CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring)
|
|
404
546
|
{
|
|
405
547
|
char *copy = NULL;
|
|
548
|
+
size_t v1_len;
|
|
549
|
+
size_t v2_len;
|
|
406
550
|
/* if object's type is not cJSON_String or is cJSON_IsReference, it should not set valuestring */
|
|
407
551
|
if ((object == NULL) || !(object->type & cJSON_String) || (object->type & cJSON_IsReference))
|
|
408
552
|
{
|
|
@@ -413,8 +557,17 @@ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring)
|
|
|
413
557
|
{
|
|
414
558
|
return NULL;
|
|
415
559
|
}
|
|
416
|
-
|
|
560
|
+
|
|
561
|
+
v1_len = strlen(valuestring);
|
|
562
|
+
v2_len = strlen(object->valuestring);
|
|
563
|
+
|
|
564
|
+
if (v1_len <= v2_len)
|
|
417
565
|
{
|
|
566
|
+
/* strcpy does not handle overlapping string: [X1, X2] [Y1, Y2] => X2 < Y1 or Y2 < X1 */
|
|
567
|
+
if (!( valuestring + v1_len < object->valuestring || object->valuestring + v2_len < valuestring ))
|
|
568
|
+
{
|
|
569
|
+
return NULL;
|
|
570
|
+
}
|
|
418
571
|
strcpy(object->valuestring, valuestring);
|
|
419
572
|
return object->valuestring;
|
|
420
573
|
}
|
|
@@ -570,10 +723,10 @@ static cJSON_bool print_number(const cJSON * const item, printbuffer * const out
|
|
|
570
723
|
{
|
|
571
724
|
length = sprintf((char*)number_buffer, "null");
|
|
572
725
|
}
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
726
|
+
else if(d == (double)item->valueint)
|
|
727
|
+
{
|
|
728
|
+
length = sprintf((char*)number_buffer, "%d", item->valueint);
|
|
729
|
+
}
|
|
577
730
|
else
|
|
578
731
|
{
|
|
579
732
|
/* Try 15 decimal places of precision to avoid nonsignificant nonzero digits */
|
|
@@ -797,6 +950,10 @@ static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_bu
|
|
|
797
950
|
size_t skipped_bytes = 0;
|
|
798
951
|
while (((size_t)(input_end - input_buffer->content) < input_buffer->length) && (*input_end != '\"'))
|
|
799
952
|
{
|
|
953
|
+
if (!parse_poll_progress(input_buffer, 1, 0))
|
|
954
|
+
{
|
|
955
|
+
goto fail;
|
|
956
|
+
}
|
|
800
957
|
/* is escape sequence */
|
|
801
958
|
if (input_end[0] == '\\')
|
|
802
959
|
{
|
|
@@ -806,6 +963,10 @@ static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_bu
|
|
|
806
963
|
goto fail;
|
|
807
964
|
}
|
|
808
965
|
skipped_bytes++;
|
|
966
|
+
if (!parse_poll_progress(input_buffer, 1, 0))
|
|
967
|
+
{
|
|
968
|
+
goto fail;
|
|
969
|
+
}
|
|
809
970
|
input_end++;
|
|
810
971
|
}
|
|
811
972
|
input_end++;
|
|
@@ -817,9 +978,14 @@ static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_bu
|
|
|
817
978
|
|
|
818
979
|
/* This is at most how much we need for the output */
|
|
819
980
|
allocation_length = (size_t) (input_end - buffer_at_offset(input_buffer)) - skipped_bytes;
|
|
981
|
+
if (!parse_poll_progress(input_buffer, 0, 1))
|
|
982
|
+
{
|
|
983
|
+
goto fail;
|
|
984
|
+
}
|
|
820
985
|
output = (unsigned char*)input_buffer->hooks.allocate(allocation_length + sizeof(""));
|
|
821
986
|
if (output == NULL)
|
|
822
987
|
{
|
|
988
|
+
cJSON_mark_parse_allocation_failure();
|
|
823
989
|
goto fail; /* allocation failure */
|
|
824
990
|
}
|
|
825
991
|
}
|
|
@@ -828,6 +994,10 @@ static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_bu
|
|
|
828
994
|
/* loop through the string literal */
|
|
829
995
|
while (input_pointer < input_end)
|
|
830
996
|
{
|
|
997
|
+
if (!parse_poll_progress(input_buffer, 1, 0))
|
|
998
|
+
{
|
|
999
|
+
goto fail;
|
|
1000
|
+
}
|
|
831
1001
|
if (*input_pointer != '\\')
|
|
832
1002
|
{
|
|
833
1003
|
*output_pointer++ = *input_pointer++;
|
|
@@ -877,6 +1047,11 @@ static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_bu
|
|
|
877
1047
|
default:
|
|
878
1048
|
goto fail;
|
|
879
1049
|
}
|
|
1050
|
+
if ((sequence_length > 1)
|
|
1051
|
+
&& !parse_poll_progress(input_buffer, sequence_length - 1, 0))
|
|
1052
|
+
{
|
|
1053
|
+
goto fail;
|
|
1054
|
+
}
|
|
880
1055
|
input_pointer += sequence_length;
|
|
881
1056
|
}
|
|
882
1057
|
}
|
|
@@ -1058,6 +1233,10 @@ static parse_buffer *buffer_skip_whitespace(parse_buffer * const buffer)
|
|
|
1058
1233
|
|
|
1059
1234
|
while (can_access_at_index(buffer, 0) && (buffer_at_offset(buffer)[0] <= 32))
|
|
1060
1235
|
{
|
|
1236
|
+
if (!parse_poll_progress(buffer, 1, 0))
|
|
1237
|
+
{
|
|
1238
|
+
return buffer;
|
|
1239
|
+
}
|
|
1061
1240
|
buffer->offset++;
|
|
1062
1241
|
}
|
|
1063
1242
|
|
|
@@ -1101,14 +1280,19 @@ CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return
|
|
|
1101
1280
|
}
|
|
1102
1281
|
|
|
1103
1282
|
/* Parse an object - create a new root, and populate. */
|
|
1104
|
-
|
|
1283
|
+
static cJSON *cJSON_ParseWithLengthOptsInternal(
|
|
1284
|
+
const char *value, size_t buffer_length, const char **return_parse_end,
|
|
1285
|
+
cJSON_bool require_null_terminated, cJSON_ParsePollFn poll,
|
|
1286
|
+
void *poll_user_data, size_t byte_interval, size_t iteration_interval)
|
|
1105
1287
|
{
|
|
1106
|
-
parse_buffer buffer = { 0
|
|
1288
|
+
parse_buffer buffer = { 0 };
|
|
1107
1289
|
cJSON *item = NULL;
|
|
1108
1290
|
|
|
1109
1291
|
/* reset error position */
|
|
1110
1292
|
global_error.json = NULL;
|
|
1111
1293
|
global_error.position = 0;
|
|
1294
|
+
global_parse_allocation_failed = false;
|
|
1295
|
+
global_parse_tracking = true;
|
|
1112
1296
|
|
|
1113
1297
|
if (value == NULL || 0 == buffer_length)
|
|
1114
1298
|
{
|
|
@@ -1119,6 +1303,17 @@ CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer
|
|
|
1119
1303
|
buffer.length = buffer_length;
|
|
1120
1304
|
buffer.offset = 0;
|
|
1121
1305
|
buffer.hooks = global_hooks;
|
|
1306
|
+
buffer.poll = poll;
|
|
1307
|
+
buffer.poll_user_data = poll_user_data;
|
|
1308
|
+
buffer.poll_byte_interval = byte_interval == 0 ? SIZE_MAX : byte_interval;
|
|
1309
|
+
buffer.poll_iteration_interval = iteration_interval == 0
|
|
1310
|
+
? SIZE_MAX : iteration_interval;
|
|
1311
|
+
|
|
1312
|
+
if (buffer.poll != NULL && !buffer.poll(buffer.poll_user_data))
|
|
1313
|
+
{
|
|
1314
|
+
buffer.poll_failed = true;
|
|
1315
|
+
goto fail;
|
|
1316
|
+
}
|
|
1122
1317
|
|
|
1123
1318
|
item = cJSON_New_Item(&global_hooks);
|
|
1124
1319
|
if (item == NULL) /* memory fail */
|
|
@@ -1141,17 +1336,25 @@ CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer
|
|
|
1141
1336
|
goto fail;
|
|
1142
1337
|
}
|
|
1143
1338
|
}
|
|
1339
|
+
if (buffer.poll_failed
|
|
1340
|
+
|| (buffer.poll != NULL && !buffer.poll(buffer.poll_user_data)))
|
|
1341
|
+
{
|
|
1342
|
+
buffer.poll_failed = true;
|
|
1343
|
+
goto fail;
|
|
1344
|
+
}
|
|
1144
1345
|
if (return_parse_end)
|
|
1145
1346
|
{
|
|
1146
1347
|
*return_parse_end = (const char*)buffer_at_offset(&buffer);
|
|
1147
1348
|
}
|
|
1148
1349
|
|
|
1350
|
+
global_parse_tracking = false;
|
|
1149
1351
|
return item;
|
|
1150
1352
|
|
|
1151
1353
|
fail:
|
|
1354
|
+
global_parse_tracking = false;
|
|
1152
1355
|
if (item != NULL)
|
|
1153
1356
|
{
|
|
1154
|
-
|
|
1357
|
+
parse_delete(item, &buffer);
|
|
1155
1358
|
}
|
|
1156
1359
|
|
|
1157
1360
|
if (value != NULL)
|
|
@@ -1180,6 +1383,23 @@ fail:
|
|
|
1180
1383
|
return NULL;
|
|
1181
1384
|
}
|
|
1182
1385
|
|
|
1386
|
+
CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer_length, const char **return_parse_end, cJSON_bool require_null_terminated)
|
|
1387
|
+
{
|
|
1388
|
+
return cJSON_ParseWithLengthOptsInternal(
|
|
1389
|
+
value, buffer_length, return_parse_end, require_null_terminated,
|
|
1390
|
+
NULL, NULL, 0, 0);
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOptsAndPoll(
|
|
1394
|
+
const char *value, size_t buffer_length, const char **return_parse_end,
|
|
1395
|
+
cJSON_bool require_null_terminated, cJSON_ParsePollFn poll,
|
|
1396
|
+
void *poll_user_data, size_t byte_interval, size_t iteration_interval)
|
|
1397
|
+
{
|
|
1398
|
+
return cJSON_ParseWithLengthOptsInternal(
|
|
1399
|
+
value, buffer_length, return_parse_end, require_null_terminated,
|
|
1400
|
+
poll, poll_user_data, byte_interval, iteration_interval);
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1183
1403
|
/* Default options for cJSON_Parse */
|
|
1184
1404
|
CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value)
|
|
1185
1405
|
{
|
|
@@ -1329,6 +1549,11 @@ static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buf
|
|
|
1329
1549
|
return false; /* no input */
|
|
1330
1550
|
}
|
|
1331
1551
|
|
|
1552
|
+
if (!parse_poll_progress(input_buffer, 0, 1))
|
|
1553
|
+
{
|
|
1554
|
+
return false;
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1332
1557
|
/* parse the different types of values */
|
|
1333
1558
|
/* null */
|
|
1334
1559
|
if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "null", 4) == 0))
|
|
@@ -1488,6 +1713,10 @@ static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buf
|
|
|
1488
1713
|
/* loop through the comma separated array elements */
|
|
1489
1714
|
do
|
|
1490
1715
|
{
|
|
1716
|
+
if (!parse_poll_progress(input_buffer, 0, 1))
|
|
1717
|
+
{
|
|
1718
|
+
goto fail;
|
|
1719
|
+
}
|
|
1491
1720
|
/* allocate next item */
|
|
1492
1721
|
cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks));
|
|
1493
1722
|
if (new_item == NULL)
|
|
@@ -1542,7 +1771,7 @@ success:
|
|
|
1542
1771
|
fail:
|
|
1543
1772
|
if (head != NULL)
|
|
1544
1773
|
{
|
|
1545
|
-
|
|
1774
|
+
parse_delete(head, input_buffer);
|
|
1546
1775
|
}
|
|
1547
1776
|
|
|
1548
1777
|
return false;
|
|
@@ -1646,6 +1875,10 @@ static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_bu
|
|
|
1646
1875
|
/* loop through the comma separated array elements */
|
|
1647
1876
|
do
|
|
1648
1877
|
{
|
|
1878
|
+
if (!parse_poll_progress(input_buffer, 0, 1))
|
|
1879
|
+
{
|
|
1880
|
+
goto fail;
|
|
1881
|
+
}
|
|
1649
1882
|
/* allocate next item */
|
|
1650
1883
|
cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks));
|
|
1651
1884
|
if (new_item == NULL)
|
|
@@ -1722,7 +1955,7 @@ success:
|
|
|
1722
1955
|
fail:
|
|
1723
1956
|
if (head != NULL)
|
|
1724
1957
|
{
|
|
1725
|
-
|
|
1958
|
+
parse_delete(head, input_buffer);
|
|
1726
1959
|
}
|
|
1727
1960
|
|
|
1728
1961
|
return false;
|
|
@@ -2204,7 +2437,7 @@ CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * c
|
|
|
2204
2437
|
|
|
2205
2438
|
CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item)
|
|
2206
2439
|
{
|
|
2207
|
-
if ((parent == NULL) || (item == NULL))
|
|
2440
|
+
if ((parent == NULL) || (item == NULL) || (item != parent->child && item->prev == NULL))
|
|
2208
2441
|
{
|
|
2209
2442
|
return NULL;
|
|
2210
2443
|
}
|
|
@@ -2726,7 +2959,14 @@ CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char *const *strings, int co
|
|
|
2726
2959
|
}
|
|
2727
2960
|
|
|
2728
2961
|
/* Duplication */
|
|
2962
|
+
cJSON * cJSON_Duplicate_rec(const cJSON *item, size_t depth, cJSON_bool recurse);
|
|
2963
|
+
|
|
2729
2964
|
CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse)
|
|
2965
|
+
{
|
|
2966
|
+
return cJSON_Duplicate_rec(item, 0, recurse );
|
|
2967
|
+
}
|
|
2968
|
+
|
|
2969
|
+
cJSON * cJSON_Duplicate_rec(const cJSON *item, size_t depth, cJSON_bool recurse)
|
|
2730
2970
|
{
|
|
2731
2971
|
cJSON *newitem = NULL;
|
|
2732
2972
|
cJSON *child = NULL;
|
|
@@ -2773,7 +3013,10 @@ CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse)
|
|
|
2773
3013
|
child = item->child;
|
|
2774
3014
|
while (child != NULL)
|
|
2775
3015
|
{
|
|
2776
|
-
|
|
3016
|
+
if(depth >= CJSON_CIRCULAR_LIMIT) {
|
|
3017
|
+
goto fail;
|
|
3018
|
+
}
|
|
3019
|
+
newchild = cJSON_Duplicate_rec(child, depth + 1, true); /* Duplicate (with recurse) each item in the ->next chain */
|
|
2777
3020
|
if (!newchild)
|
|
2778
3021
|
{
|
|
2779
3022
|
goto fail;
|