tranfi 0.1.2 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/LICENSE +177 -0
  2. package/NOTICE +8 -0
  3. package/README.md +443 -51
  4. package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
  5. package/app/index.html +1 -1
  6. package/binding.gyp +55 -3
  7. package/csrc/arena.c +7 -5
  8. package/csrc/batch.c +818 -71
  9. package/csrc/buffer.c +84 -8
  10. package/csrc/cJSON.c +262 -19
  11. package/csrc/cJSON.h +17 -1
  12. package/csrc/codec_csv.c +1074 -181
  13. package/csrc/codec_jsonl.c +830 -118
  14. package/csrc/codec_table.c +108 -78
  15. package/csrc/codec_text.c +286 -68
  16. package/csrc/compiler.c +31 -3
  17. package/csrc/config.h +21 -0
  18. package/csrc/dsl.c +4722 -485
  19. package/csrc/expr.c +363 -55
  20. package/csrc/expr.h +2 -0
  21. package/csrc/internal.h +316 -27
  22. package/csrc/ir.c +65 -18
  23. package/csrc/ir.h +41 -0
  24. package/csrc/ir_schema.c +20 -5
  25. package/csrc/ir_serialize.c +68 -6
  26. package/csrc/ir_sql.c +796 -185
  27. package/csrc/ir_validate.c +462 -6
  28. package/csrc/json_path.c +210 -0
  29. package/csrc/main.c +879 -30
  30. package/csrc/memory_estimate.c +477 -0
  31. package/csrc/op_acf.c +171 -21
  32. package/csrc/op_across.c +477 -0
  33. package/csrc/op_anomaly.c +167 -32
  34. package/csrc/op_assert.c +761 -0
  35. package/csrc/op_bin.c +168 -29
  36. package/csrc/op_cast.c +383 -55
  37. package/csrc/op_clip.c +30 -19
  38. package/csrc/op_date_trunc.c +208 -34
  39. package/csrc/op_datetime.c +259 -77
  40. package/csrc/op_derive.c +65 -97
  41. package/csrc/op_diff.c +146 -30
  42. package/csrc/op_ewma.c +149 -30
  43. package/csrc/op_explode.c +124 -26
  44. package/csrc/op_fill_down.c +125 -53
  45. package/csrc/op_fill_null.c +176 -31
  46. package/csrc/op_filter.c +89 -40
  47. package/csrc/op_frequency.c +571 -43
  48. package/csrc/op_grep.c +36 -18
  49. package/csrc/op_group_agg.c +1790 -119
  50. package/csrc/op_hash.c +48 -15
  51. package/csrc/op_head.c +21 -86
  52. package/csrc/op_interpolate.c +268 -62
  53. package/csrc/op_join.c +2700 -182
  54. package/csrc/op_json_extract.c +227 -0
  55. package/csrc/op_json_filter.c +384 -0
  56. package/csrc/op_json_flatten.c +293 -0
  57. package/csrc/op_json_schema.c +503 -0
  58. package/csrc/op_label_encode.c +328 -53
  59. package/csrc/op_lag.c +181 -0
  60. package/csrc/op_lead.c +141 -89
  61. package/csrc/op_normalize.c +363 -79
  62. package/csrc/op_onehot.c +345 -73
  63. package/csrc/op_pivot.c +1546 -162
  64. package/csrc/op_quarantine.c +189 -0
  65. package/csrc/op_registry.c +2062 -166
  66. package/csrc/op_rename.c +41 -50
  67. package/csrc/op_replace.c +270 -118
  68. package/csrc/op_rleid.c +297 -0
  69. package/csrc/op_rowid.c +559 -0
  70. package/csrc/op_sample.c +80 -23
  71. package/csrc/op_schema.c +1341 -0
  72. package/csrc/op_schema_infer.c +252 -0
  73. package/csrc/op_select.c +265 -65
  74. package/csrc/op_set.c +3449 -0
  75. package/csrc/op_skip.c +30 -87
  76. package/csrc/op_sort.c +670 -124
  77. package/csrc/op_source_name.c +120 -0
  78. package/csrc/op_split.c +65 -28
  79. package/csrc/op_split_data.c +41 -9
  80. package/csrc/op_stack.c +178 -222
  81. package/csrc/op_stats.c +206 -110
  82. package/csrc/op_step.c +217 -55
  83. package/csrc/op_tail.c +21 -12
  84. package/csrc/op_tee.c +338 -0
  85. package/csrc/op_top.c +260 -53
  86. package/csrc/op_trim.c +48 -19
  87. package/csrc/op_unique.c +1193 -150
  88. package/csrc/op_unpivot.c +100 -66
  89. package/csrc/op_validate.c +601 -24
  90. package/csrc/op_window.c +492 -51
  91. package/csrc/path_policy.c +85 -0
  92. package/csrc/pipeline.c +872 -99
  93. package/csrc/recipes.c +3 -1
  94. package/csrc/report.c +73 -30
  95. package/csrc/selector.c +1097 -0
  96. package/csrc/size_utils.c +352 -0
  97. package/csrc/spill.c +317 -0
  98. package/csrc/spill.h +21 -0
  99. package/csrc/tranfi.h +169 -1
  100. package/csrc/transform.h +209 -0
  101. package/csrc/transform_api.c +2237 -0
  102. package/csrc/transform_categorical.c +923 -0
  103. package/csrc/transform_internal.h +472 -0
  104. package/csrc/transform_json.c +3812 -0
  105. package/csrc/transform_numeric.c +1966 -0
  106. package/csrc/transform_sha256.c +154 -0
  107. package/csrc/transform_wasm.h +162 -0
  108. package/csrc/transform_wasm_api.c +1373 -0
  109. package/csrc/wasm_api.c +70 -9
  110. package/napi_api.c +219 -11
  111. package/napi_transform.c +1648 -0
  112. package/napi_transform.h +8 -0
  113. package/package.json +27 -11
  114. package/scripts/install-native.js +76 -0
  115. package/scripts/prepack.js +64 -0
  116. package/scripts/sync-csrc.js +23 -0
  117. package/src/cli.js +81 -41
  118. package/src/engines/duckdb.js +45 -12
  119. package/src/index.js +661 -42
  120. package/src/memory_policy.js +411 -0
  121. package/src/native.js +1 -5
  122. package/src/pipeline.js +454 -31
  123. package/src/recipe_json.js +80 -0
  124. package/src/server.js +10 -8
  125. package/src/transform.js +403 -0
  126. package/src/transform_error.js +10 -0
  127. package/src/wasm.js +6 -4
  128. package/wasm/index.js +498 -10
  129. package/wasm/tranfi_core.js +0 -0
  130. package/wasm/transform.js +1156 -0
  131. package/wasm/worker.js +786 -0
  132. package/csrc/plan.c +0 -206
package/csrc/buffer.c CHANGED
@@ -4,10 +4,13 @@
4
4
  */
5
5
 
6
6
  #include "internal.h"
7
+ #include "cJSON.h"
7
8
  #include <stdlib.h>
8
9
  #include <string.h>
9
10
 
10
11
  #define INITIAL_CAP 4096
12
+ #define COMPACT_AFTER_READ_POS (64 * 1024)
13
+ #define MAX_RETAINED_EMPTY_CAP (1024 * 1024)
11
14
 
12
15
  void tf_buffer_init(tf_buffer *b) {
13
16
  b->data = NULL;
@@ -16,12 +19,11 @@ void tf_buffer_init(tf_buffer *b) {
16
19
  b->read_pos = 0;
17
20
  }
18
21
 
19
- static int buffer_ensure(tf_buffer *b, size_t extra) {
20
- size_t needed = b->len + extra;
22
+ static int buffer_ensure(tf_buffer *b, size_t needed) {
21
23
  if (needed <= b->cap) return TF_OK;
22
- size_t new_cap = b->cap ? b->cap : INITIAL_CAP;
23
- while (new_cap < needed) new_cap *= 2;
24
- uint8_t *new_data = realloc(b->data, new_cap);
24
+ size_t new_cap = 0;
25
+ if (tf_size_grow_pow2(b->cap, needed, INITIAL_CAP, &new_cap) != TF_OK) return TF_ERROR;
26
+ uint8_t *new_data = tf_reallocarray_checked(b->data, new_cap, sizeof(uint8_t));
25
27
  if (!new_data) return TF_ERROR;
26
28
  b->data = new_data;
27
29
  b->cap = new_cap;
@@ -30,9 +32,11 @@ static int buffer_ensure(tf_buffer *b, size_t extra) {
30
32
 
31
33
  int tf_buffer_write(tf_buffer *b, const uint8_t *data, size_t len) {
32
34
  if (len == 0) return TF_OK;
33
- if (buffer_ensure(b, len) != TF_OK) return TF_ERROR;
35
+ size_t new_len = 0;
36
+ if (tf_size_add(b->len, len, &new_len) != TF_OK) return TF_ERROR;
37
+ if (buffer_ensure(b, new_len) != TF_OK) return TF_ERROR;
34
38
  memcpy(b->data + b->len, data, len);
35
- b->len += len;
39
+ b->len = new_len;
36
40
  return TF_OK;
37
41
  }
38
42
 
@@ -40,6 +44,67 @@ int tf_buffer_write_str(tf_buffer *b, const char *s) {
40
44
  return tf_buffer_write(b, (const uint8_t *)s, strlen(s));
41
45
  }
42
46
 
47
+ int tf_buffer_write_line(tf_buffer *b, const char *s) {
48
+ int rc = tf_buffer_write_str(b, s);
49
+ if (rc == TF_OK) rc = tf_buffer_write(b, (const uint8_t *)"\n", 1);
50
+ return rc;
51
+ }
52
+
53
+ int tf_buffer_write_json_line(tf_buffer *b, const cJSON *obj) {
54
+ if (!b || !obj) return TF_ERROR;
55
+ char *printed = cJSON_PrintUnformatted(obj);
56
+ if (!printed) return TF_ERROR;
57
+ int rc = tf_buffer_write_line(b, printed);
58
+ free(printed);
59
+ return rc;
60
+ }
61
+
62
+ int tf_side_write_error(tf_side_channels *side, const char *msg) {
63
+ if (msg) tf_set_last_error(msg);
64
+ if (!side || !side->errors || !msg) return TF_OK;
65
+ return tf_buffer_write_line(side->errors, msg);
66
+ }
67
+
68
+ int tf_json_add_string(cJSON *obj, const char *name, const char *value) {
69
+ return cJSON_AddStringToObject(obj, name, value ? value : "") ? TF_OK : TF_ERROR;
70
+ }
71
+
72
+ int tf_json_add_number(cJSON *obj, const char *name, double value) {
73
+ return cJSON_AddNumberToObject(obj, name, value) ? TF_OK : TF_ERROR;
74
+ }
75
+
76
+ int tf_json_add_bool(cJSON *obj, const char *name, int value) {
77
+ return cJSON_AddBoolToObject(obj, name, value ? 1 : 0) ? TF_OK : TF_ERROR;
78
+ }
79
+
80
+ int tf_json_add_null(cJSON *obj, const char *name) {
81
+ return cJSON_AddNullToObject(obj, name) ? TF_OK : TF_ERROR;
82
+ }
83
+
84
+ int tf_json_add_item(cJSON *obj, const char *name, cJSON *item) {
85
+ if (!obj || !name || !item) {
86
+ cJSON_Delete(item);
87
+ return TF_ERROR;
88
+ }
89
+ if (!cJSON_AddItemToObject(obj, name, item)) {
90
+ cJSON_Delete(item);
91
+ return TF_ERROR;
92
+ }
93
+ return TF_OK;
94
+ }
95
+
96
+ int tf_json_add_array_item(cJSON *arr, cJSON *item) {
97
+ if (!arr || !item) {
98
+ cJSON_Delete(item);
99
+ return TF_ERROR;
100
+ }
101
+ if (!cJSON_AddItemToArray(arr, item)) {
102
+ cJSON_Delete(item);
103
+ return TF_ERROR;
104
+ }
105
+ return TF_OK;
106
+ }
107
+
43
108
  size_t tf_buffer_read(tf_buffer *b, uint8_t *out, size_t len) {
44
109
  size_t avail = b->len - b->read_pos;
45
110
  if (len > avail) len = avail;
@@ -47,10 +112,21 @@ size_t tf_buffer_read(tf_buffer *b, uint8_t *out, size_t len) {
47
112
  memcpy(out, b->data + b->read_pos, len);
48
113
  b->read_pos += len;
49
114
  }
50
- /* Auto-compact when all data consumed */
115
+ /* Auto-compact when all data consumed. Keep a modest scratch buffer for
116
+ * future chunks, but release one-off large output buffers after drains. */
51
117
  if (b->read_pos == b->len) {
52
118
  b->read_pos = 0;
53
119
  b->len = 0;
120
+ if (b->cap > MAX_RETAINED_EMPTY_CAP) {
121
+ uint8_t *new_data = tf_reallocarray_checked(b->data, MAX_RETAINED_EMPTY_CAP,
122
+ sizeof(uint8_t));
123
+ if (new_data) {
124
+ b->data = new_data;
125
+ b->cap = MAX_RETAINED_EMPTY_CAP;
126
+ }
127
+ }
128
+ } else if (b->read_pos >= COMPACT_AFTER_READ_POS && b->read_pos > b->cap / 2) {
129
+ tf_buffer_compact(b);
54
130
  }
55
131
  return len;
56
132
  }
package/csrc/cJSON.c CHANGED
@@ -42,6 +42,7 @@
42
42
  #include <math.h>
43
43
  #include <stdlib.h>
44
44
  #include <limits.h>
45
+ #include <stdint.h>
45
46
  #include <ctype.h>
46
47
  #include <float.h>
47
48
 
@@ -89,13 +90,40 @@ typedef struct {
89
90
  const unsigned char *json;
90
91
  size_t position;
91
92
  } error;
92
- static error global_error = { NULL, 0 };
93
+
94
+ /* Tranfi local patch: parse errors are thread-local for concurrent embedders. */
95
+ #if defined(_MSC_VER)
96
+ #define CJSON_THREAD_LOCAL __declspec(thread)
97
+ #elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
98
+ #define CJSON_THREAD_LOCAL _Thread_local
99
+ #elif defined(__GNUC__) || defined(__clang__)
100
+ #define CJSON_THREAD_LOCAL __thread
101
+ #else
102
+ #define CJSON_THREAD_LOCAL
103
+ #endif
104
+
105
+ static CJSON_THREAD_LOCAL error global_error = { NULL, 0 };
106
+ static CJSON_THREAD_LOCAL cJSON_bool global_parse_tracking = false;
107
+ static CJSON_THREAD_LOCAL cJSON_bool global_parse_allocation_failed = false;
108
+
109
+ static void cJSON_mark_parse_allocation_failure(void)
110
+ {
111
+ if (global_parse_tracking)
112
+ {
113
+ global_parse_allocation_failed = true;
114
+ }
115
+ }
93
116
 
94
117
  CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void)
95
118
  {
96
119
  return (const char*) (global_error.json + global_error.position);
97
120
  }
98
121
 
122
+ CJSON_PUBLIC(cJSON_bool) cJSON_ParseHadAllocationFailure(void)
123
+ {
124
+ return global_parse_allocation_failed;
125
+ }
126
+
99
127
  CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item)
100
128
  {
101
129
  if (!cJSON_IsString(item))
@@ -117,7 +145,7 @@ CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item)
117
145
  }
118
146
 
119
147
  /* This is a safeguard to prevent copy-pasters from using incompatible C and header files */
120
- #if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 18)
148
+ #if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 19)
121
149
  #error cJSON.h and cJSON.c have different versions. Make sure that both have the same.
122
150
  #endif
123
151
 
@@ -199,6 +227,7 @@ static unsigned char* cJSON_strdup(const unsigned char* string, const internal_h
199
227
  copy = (unsigned char*)hooks->allocate(length);
200
228
  if (copy == NULL)
201
229
  {
230
+ cJSON_mark_parse_allocation_failure();
202
231
  return NULL;
203
232
  }
204
233
  memcpy(copy, string, length);
@@ -245,6 +274,10 @@ static cJSON *cJSON_New_Item(const internal_hooks * const hooks)
245
274
  {
246
275
  memset(node, '\0', sizeof(cJSON));
247
276
  }
277
+ else
278
+ {
279
+ cJSON_mark_parse_allocation_failure();
280
+ }
248
281
 
249
282
  return node;
250
283
  }
@@ -293,8 +326,69 @@ typedef struct
293
326
  size_t offset;
294
327
  size_t depth; /* How deeply nested (in arrays/objects) is the input at the current offset. */
295
328
  internal_hooks hooks;
329
+ cJSON_ParsePollFn poll;
330
+ void *poll_user_data;
331
+ size_t poll_byte_interval;
332
+ size_t poll_iteration_interval;
333
+ size_t poll_bytes;
334
+ size_t poll_iterations;
335
+ cJSON_bool poll_failed;
296
336
  } parse_buffer;
297
337
 
338
+ static cJSON_bool parse_poll_progress(
339
+ parse_buffer * const buffer, size_t bytes, size_t iterations)
340
+ {
341
+ if ((buffer == NULL) || buffer->poll_failed)
342
+ {
343
+ return false;
344
+ }
345
+ if (bytes > (SIZE_MAX - buffer->poll_bytes)
346
+ || iterations > (SIZE_MAX - buffer->poll_iterations))
347
+ {
348
+ buffer->poll_failed = true;
349
+ return false;
350
+ }
351
+ buffer->poll_bytes += bytes;
352
+ buffer->poll_iterations += iterations;
353
+ if (buffer->poll != NULL
354
+ && ((buffer->poll_bytes >= buffer->poll_byte_interval)
355
+ || (buffer->poll_iterations >= buffer->poll_iteration_interval)))
356
+ {
357
+ if (!buffer->poll(buffer->poll_user_data))
358
+ {
359
+ buffer->poll_failed = true;
360
+ return false;
361
+ }
362
+ buffer->poll_bytes = 0;
363
+ buffer->poll_iterations = 0;
364
+ }
365
+ return true;
366
+ }
367
+
368
+ static void parse_delete(cJSON *item, parse_buffer *buffer)
369
+ {
370
+ cJSON *next = NULL;
371
+ while (item != NULL)
372
+ {
373
+ next = item->next;
374
+ if (!(item->type & cJSON_IsReference) && (item->child != NULL))
375
+ {
376
+ parse_delete(item->child, buffer);
377
+ }
378
+ if (!(item->type & cJSON_IsReference) && (item->valuestring != NULL))
379
+ {
380
+ buffer->hooks.deallocate(item->valuestring);
381
+ }
382
+ if (!(item->type & cJSON_StringIsConst) && (item->string != NULL))
383
+ {
384
+ buffer->hooks.deallocate(item->string);
385
+ }
386
+ buffer->hooks.deallocate(item);
387
+ (void)parse_poll_progress(buffer, 0, 1);
388
+ item = next;
389
+ }
390
+ }
391
+
298
392
  /* check if the given size is left to read in a given parse buffer (starting with 1) */
299
393
  #define can_read(buffer, size) ((buffer != NULL) && (((buffer)->offset + size) <= (buffer)->length))
300
394
  /* check if the buffer can be accessed at the given index (starting with 0) */
@@ -308,9 +402,11 @@ static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_bu
308
402
  {
309
403
  double number = 0;
310
404
  unsigned char *after_end = NULL;
311
- unsigned char number_c_string[64];
405
+ unsigned char *number_c_string;
312
406
  unsigned char decimal_point = get_decimal_point();
313
407
  size_t i = 0;
408
+ size_t number_string_length = 0;
409
+ cJSON_bool has_decimal_point = false;
314
410
 
315
411
  if ((input_buffer == NULL) || (input_buffer->content == NULL))
316
412
  {
@@ -320,8 +416,12 @@ static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_bu
320
416
  /* copy the number into a temporary buffer and replace '.' with the decimal point
321
417
  * of the current locale (for strtod)
322
418
  * This also takes care of '\0' not necessarily being available for marking the end of the input */
323
- for (i = 0; (i < (sizeof(number_c_string) - 1)) && can_access_at_index(input_buffer, i); i++)
419
+ for (i = 0; can_access_at_index(input_buffer, i); i++)
324
420
  {
421
+ if (!parse_poll_progress(input_buffer, 1, 0))
422
+ {
423
+ return false;
424
+ }
325
425
  switch (buffer_at_offset(input_buffer)[i])
326
426
  {
327
427
  case '0':
@@ -338,11 +438,12 @@ static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_bu
338
438
  case '-':
339
439
  case 'e':
340
440
  case 'E':
341
- number_c_string[i] = buffer_at_offset(input_buffer)[i];
441
+ number_string_length++;
342
442
  break;
343
443
 
344
444
  case '.':
345
- number_c_string[i] = decimal_point;
445
+ number_string_length++;
446
+ has_decimal_point = true;
346
447
  break;
347
448
 
348
449
  default:
@@ -350,11 +451,50 @@ static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_bu
350
451
  }
351
452
  }
352
453
  loop_end:
353
- number_c_string[i] = '\0';
454
+ if (!parse_poll_progress(input_buffer, 0, 1))
455
+ {
456
+ return false;
457
+ }
458
+ /* malloc for temporary buffer, add 1 for '\0' */
459
+ number_c_string = (unsigned char *) input_buffer->hooks.allocate(number_string_length + 1);
460
+ if (number_c_string == NULL)
461
+ {
462
+ cJSON_mark_parse_allocation_failure();
463
+ return false; /* allocation failure */
464
+ }
465
+
466
+ for (i = 0; i < number_string_length;)
467
+ {
468
+ size_t remaining = number_string_length - i;
469
+ size_t chunk = remaining < input_buffer->poll_byte_interval
470
+ ? remaining : input_buffer->poll_byte_interval;
471
+ memcpy(number_c_string + i, buffer_at_offset(input_buffer) + i, chunk);
472
+ i += chunk;
473
+ if (!parse_poll_progress(input_buffer, chunk, 0))
474
+ {
475
+ input_buffer->hooks.deallocate(number_c_string);
476
+ return false;
477
+ }
478
+ }
479
+ number_c_string[number_string_length] = '\0';
480
+
481
+ if (has_decimal_point)
482
+ {
483
+ for (i = 0; i < number_string_length; i++)
484
+ {
485
+ if (number_c_string[i] == '.')
486
+ {
487
+ /* replace '.' with the decimal point of the current locale (for strtod) */
488
+ number_c_string[i] = decimal_point;
489
+ }
490
+ }
491
+ }
354
492
 
355
493
  number = strtod((const char*)number_c_string, (char**)&after_end);
356
494
  if (number_c_string == after_end)
357
495
  {
496
+ /* free the temporary buffer */
497
+ input_buffer->hooks.deallocate(number_c_string);
358
498
  return false; /* parse_error */
359
499
  }
360
500
 
@@ -377,6 +517,8 @@ loop_end:
377
517
  item->type = cJSON_Number;
378
518
 
379
519
  input_buffer->offset += (size_t)(after_end - number_c_string);
520
+ /* free the temporary buffer */
521
+ input_buffer->hooks.deallocate(number_c_string);
380
522
  return true;
381
523
  }
382
524
 
@@ -403,6 +545,8 @@ CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number)
403
545
  CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring)
404
546
  {
405
547
  char *copy = NULL;
548
+ size_t v1_len;
549
+ size_t v2_len;
406
550
  /* if object's type is not cJSON_String or is cJSON_IsReference, it should not set valuestring */
407
551
  if ((object == NULL) || !(object->type & cJSON_String) || (object->type & cJSON_IsReference))
408
552
  {
@@ -413,8 +557,17 @@ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring)
413
557
  {
414
558
  return NULL;
415
559
  }
416
- if (strlen(valuestring) <= strlen(object->valuestring))
560
+
561
+ v1_len = strlen(valuestring);
562
+ v2_len = strlen(object->valuestring);
563
+
564
+ if (v1_len <= v2_len)
417
565
  {
566
+ /* strcpy does not handle overlapping string: [X1, X2] [Y1, Y2] => X2 < Y1 or Y2 < X1 */
567
+ if (!( valuestring + v1_len < object->valuestring || object->valuestring + v2_len < valuestring ))
568
+ {
569
+ return NULL;
570
+ }
418
571
  strcpy(object->valuestring, valuestring);
419
572
  return object->valuestring;
420
573
  }
@@ -570,10 +723,10 @@ static cJSON_bool print_number(const cJSON * const item, printbuffer * const out
570
723
  {
571
724
  length = sprintf((char*)number_buffer, "null");
572
725
  }
573
- else if(d == (double)item->valueint)
574
- {
575
- length = sprintf((char*)number_buffer, "%d", item->valueint);
576
- }
726
+ else if(d == (double)item->valueint)
727
+ {
728
+ length = sprintf((char*)number_buffer, "%d", item->valueint);
729
+ }
577
730
  else
578
731
  {
579
732
  /* Try 15 decimal places of precision to avoid nonsignificant nonzero digits */
@@ -797,6 +950,10 @@ static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_bu
797
950
  size_t skipped_bytes = 0;
798
951
  while (((size_t)(input_end - input_buffer->content) < input_buffer->length) && (*input_end != '\"'))
799
952
  {
953
+ if (!parse_poll_progress(input_buffer, 1, 0))
954
+ {
955
+ goto fail;
956
+ }
800
957
  /* is escape sequence */
801
958
  if (input_end[0] == '\\')
802
959
  {
@@ -806,6 +963,10 @@ static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_bu
806
963
  goto fail;
807
964
  }
808
965
  skipped_bytes++;
966
+ if (!parse_poll_progress(input_buffer, 1, 0))
967
+ {
968
+ goto fail;
969
+ }
809
970
  input_end++;
810
971
  }
811
972
  input_end++;
@@ -817,9 +978,14 @@ static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_bu
817
978
 
818
979
  /* This is at most how much we need for the output */
819
980
  allocation_length = (size_t) (input_end - buffer_at_offset(input_buffer)) - skipped_bytes;
981
+ if (!parse_poll_progress(input_buffer, 0, 1))
982
+ {
983
+ goto fail;
984
+ }
820
985
  output = (unsigned char*)input_buffer->hooks.allocate(allocation_length + sizeof(""));
821
986
  if (output == NULL)
822
987
  {
988
+ cJSON_mark_parse_allocation_failure();
823
989
  goto fail; /* allocation failure */
824
990
  }
825
991
  }
@@ -828,6 +994,10 @@ static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_bu
828
994
  /* loop through the string literal */
829
995
  while (input_pointer < input_end)
830
996
  {
997
+ if (!parse_poll_progress(input_buffer, 1, 0))
998
+ {
999
+ goto fail;
1000
+ }
831
1001
  if (*input_pointer != '\\')
832
1002
  {
833
1003
  *output_pointer++ = *input_pointer++;
@@ -877,6 +1047,11 @@ static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_bu
877
1047
  default:
878
1048
  goto fail;
879
1049
  }
1050
+ if ((sequence_length > 1)
1051
+ && !parse_poll_progress(input_buffer, sequence_length - 1, 0))
1052
+ {
1053
+ goto fail;
1054
+ }
880
1055
  input_pointer += sequence_length;
881
1056
  }
882
1057
  }
@@ -1058,6 +1233,10 @@ static parse_buffer *buffer_skip_whitespace(parse_buffer * const buffer)
1058
1233
 
1059
1234
  while (can_access_at_index(buffer, 0) && (buffer_at_offset(buffer)[0] <= 32))
1060
1235
  {
1236
+ if (!parse_poll_progress(buffer, 1, 0))
1237
+ {
1238
+ return buffer;
1239
+ }
1061
1240
  buffer->offset++;
1062
1241
  }
1063
1242
 
@@ -1101,14 +1280,19 @@ CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return
1101
1280
  }
1102
1281
 
1103
1282
  /* Parse an object - create a new root, and populate. */
1104
- CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer_length, const char **return_parse_end, cJSON_bool require_null_terminated)
1283
+ static cJSON *cJSON_ParseWithLengthOptsInternal(
1284
+ const char *value, size_t buffer_length, const char **return_parse_end,
1285
+ cJSON_bool require_null_terminated, cJSON_ParsePollFn poll,
1286
+ void *poll_user_data, size_t byte_interval, size_t iteration_interval)
1105
1287
  {
1106
- parse_buffer buffer = { 0, 0, 0, 0, { 0, 0, 0 } };
1288
+ parse_buffer buffer = { 0 };
1107
1289
  cJSON *item = NULL;
1108
1290
 
1109
1291
  /* reset error position */
1110
1292
  global_error.json = NULL;
1111
1293
  global_error.position = 0;
1294
+ global_parse_allocation_failed = false;
1295
+ global_parse_tracking = true;
1112
1296
 
1113
1297
  if (value == NULL || 0 == buffer_length)
1114
1298
  {
@@ -1119,6 +1303,17 @@ CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer
1119
1303
  buffer.length = buffer_length;
1120
1304
  buffer.offset = 0;
1121
1305
  buffer.hooks = global_hooks;
1306
+ buffer.poll = poll;
1307
+ buffer.poll_user_data = poll_user_data;
1308
+ buffer.poll_byte_interval = byte_interval == 0 ? SIZE_MAX : byte_interval;
1309
+ buffer.poll_iteration_interval = iteration_interval == 0
1310
+ ? SIZE_MAX : iteration_interval;
1311
+
1312
+ if (buffer.poll != NULL && !buffer.poll(buffer.poll_user_data))
1313
+ {
1314
+ buffer.poll_failed = true;
1315
+ goto fail;
1316
+ }
1122
1317
 
1123
1318
  item = cJSON_New_Item(&global_hooks);
1124
1319
  if (item == NULL) /* memory fail */
@@ -1141,17 +1336,25 @@ CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer
1141
1336
  goto fail;
1142
1337
  }
1143
1338
  }
1339
+ if (buffer.poll_failed
1340
+ || (buffer.poll != NULL && !buffer.poll(buffer.poll_user_data)))
1341
+ {
1342
+ buffer.poll_failed = true;
1343
+ goto fail;
1344
+ }
1144
1345
  if (return_parse_end)
1145
1346
  {
1146
1347
  *return_parse_end = (const char*)buffer_at_offset(&buffer);
1147
1348
  }
1148
1349
 
1350
+ global_parse_tracking = false;
1149
1351
  return item;
1150
1352
 
1151
1353
  fail:
1354
+ global_parse_tracking = false;
1152
1355
  if (item != NULL)
1153
1356
  {
1154
- cJSON_Delete(item);
1357
+ parse_delete(item, &buffer);
1155
1358
  }
1156
1359
 
1157
1360
  if (value != NULL)
@@ -1180,6 +1383,23 @@ fail:
1180
1383
  return NULL;
1181
1384
  }
1182
1385
 
1386
+ CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer_length, const char **return_parse_end, cJSON_bool require_null_terminated)
1387
+ {
1388
+ return cJSON_ParseWithLengthOptsInternal(
1389
+ value, buffer_length, return_parse_end, require_null_terminated,
1390
+ NULL, NULL, 0, 0);
1391
+ }
1392
+
1393
+ CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOptsAndPoll(
1394
+ const char *value, size_t buffer_length, const char **return_parse_end,
1395
+ cJSON_bool require_null_terminated, cJSON_ParsePollFn poll,
1396
+ void *poll_user_data, size_t byte_interval, size_t iteration_interval)
1397
+ {
1398
+ return cJSON_ParseWithLengthOptsInternal(
1399
+ value, buffer_length, return_parse_end, require_null_terminated,
1400
+ poll, poll_user_data, byte_interval, iteration_interval);
1401
+ }
1402
+
1183
1403
  /* Default options for cJSON_Parse */
1184
1404
  CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value)
1185
1405
  {
@@ -1329,6 +1549,11 @@ static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buf
1329
1549
  return false; /* no input */
1330
1550
  }
1331
1551
 
1552
+ if (!parse_poll_progress(input_buffer, 0, 1))
1553
+ {
1554
+ return false;
1555
+ }
1556
+
1332
1557
  /* parse the different types of values */
1333
1558
  /* null */
1334
1559
  if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "null", 4) == 0))
@@ -1488,6 +1713,10 @@ static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buf
1488
1713
  /* loop through the comma separated array elements */
1489
1714
  do
1490
1715
  {
1716
+ if (!parse_poll_progress(input_buffer, 0, 1))
1717
+ {
1718
+ goto fail;
1719
+ }
1491
1720
  /* allocate next item */
1492
1721
  cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks));
1493
1722
  if (new_item == NULL)
@@ -1542,7 +1771,7 @@ success:
1542
1771
  fail:
1543
1772
  if (head != NULL)
1544
1773
  {
1545
- cJSON_Delete(head);
1774
+ parse_delete(head, input_buffer);
1546
1775
  }
1547
1776
 
1548
1777
  return false;
@@ -1646,6 +1875,10 @@ static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_bu
1646
1875
  /* loop through the comma separated array elements */
1647
1876
  do
1648
1877
  {
1878
+ if (!parse_poll_progress(input_buffer, 0, 1))
1879
+ {
1880
+ goto fail;
1881
+ }
1649
1882
  /* allocate next item */
1650
1883
  cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks));
1651
1884
  if (new_item == NULL)
@@ -1722,7 +1955,7 @@ success:
1722
1955
  fail:
1723
1956
  if (head != NULL)
1724
1957
  {
1725
- cJSON_Delete(head);
1958
+ parse_delete(head, input_buffer);
1726
1959
  }
1727
1960
 
1728
1961
  return false;
@@ -2204,7 +2437,7 @@ CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * c
2204
2437
 
2205
2438
  CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item)
2206
2439
  {
2207
- if ((parent == NULL) || (item == NULL))
2440
+ if ((parent == NULL) || (item == NULL) || (item != parent->child && item->prev == NULL))
2208
2441
  {
2209
2442
  return NULL;
2210
2443
  }
@@ -2726,7 +2959,14 @@ CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char *const *strings, int co
2726
2959
  }
2727
2960
 
2728
2961
  /* Duplication */
2962
+ cJSON * cJSON_Duplicate_rec(const cJSON *item, size_t depth, cJSON_bool recurse);
2963
+
2729
2964
  CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse)
2965
+ {
2966
+ return cJSON_Duplicate_rec(item, 0, recurse );
2967
+ }
2968
+
2969
+ cJSON * cJSON_Duplicate_rec(const cJSON *item, size_t depth, cJSON_bool recurse)
2730
2970
  {
2731
2971
  cJSON *newitem = NULL;
2732
2972
  cJSON *child = NULL;
@@ -2773,7 +3013,10 @@ CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse)
2773
3013
  child = item->child;
2774
3014
  while (child != NULL)
2775
3015
  {
2776
- newchild = cJSON_Duplicate(child, true); /* Duplicate (with recurse) each item in the ->next chain */
3016
+ if(depth >= CJSON_CIRCULAR_LIMIT) {
3017
+ goto fail;
3018
+ }
3019
+ newchild = cJSON_Duplicate_rec(child, depth + 1, true); /* Duplicate (with recurse) each item in the ->next chain */
2777
3020
  if (!newchild)
2778
3021
  {
2779
3022
  goto fail;