tranfi 0.1.2 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/LICENSE +177 -0
  2. package/NOTICE +8 -0
  3. package/README.md +443 -51
  4. package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
  5. package/app/index.html +1 -1
  6. package/binding.gyp +55 -3
  7. package/csrc/arena.c +7 -5
  8. package/csrc/batch.c +818 -71
  9. package/csrc/buffer.c +84 -8
  10. package/csrc/cJSON.c +262 -19
  11. package/csrc/cJSON.h +17 -1
  12. package/csrc/codec_csv.c +1074 -181
  13. package/csrc/codec_jsonl.c +830 -118
  14. package/csrc/codec_table.c +108 -78
  15. package/csrc/codec_text.c +286 -68
  16. package/csrc/compiler.c +31 -3
  17. package/csrc/config.h +21 -0
  18. package/csrc/dsl.c +4722 -485
  19. package/csrc/expr.c +363 -55
  20. package/csrc/expr.h +2 -0
  21. package/csrc/internal.h +316 -27
  22. package/csrc/ir.c +65 -18
  23. package/csrc/ir.h +41 -0
  24. package/csrc/ir_schema.c +20 -5
  25. package/csrc/ir_serialize.c +68 -6
  26. package/csrc/ir_sql.c +796 -185
  27. package/csrc/ir_validate.c +462 -6
  28. package/csrc/json_path.c +210 -0
  29. package/csrc/main.c +879 -30
  30. package/csrc/memory_estimate.c +477 -0
  31. package/csrc/op_acf.c +171 -21
  32. package/csrc/op_across.c +477 -0
  33. package/csrc/op_anomaly.c +167 -32
  34. package/csrc/op_assert.c +761 -0
  35. package/csrc/op_bin.c +168 -29
  36. package/csrc/op_cast.c +383 -55
  37. package/csrc/op_clip.c +30 -19
  38. package/csrc/op_date_trunc.c +208 -34
  39. package/csrc/op_datetime.c +259 -77
  40. package/csrc/op_derive.c +65 -97
  41. package/csrc/op_diff.c +146 -30
  42. package/csrc/op_ewma.c +149 -30
  43. package/csrc/op_explode.c +124 -26
  44. package/csrc/op_fill_down.c +125 -53
  45. package/csrc/op_fill_null.c +176 -31
  46. package/csrc/op_filter.c +89 -40
  47. package/csrc/op_frequency.c +571 -43
  48. package/csrc/op_grep.c +36 -18
  49. package/csrc/op_group_agg.c +1790 -119
  50. package/csrc/op_hash.c +48 -15
  51. package/csrc/op_head.c +21 -86
  52. package/csrc/op_interpolate.c +268 -62
  53. package/csrc/op_join.c +2700 -182
  54. package/csrc/op_json_extract.c +227 -0
  55. package/csrc/op_json_filter.c +384 -0
  56. package/csrc/op_json_flatten.c +293 -0
  57. package/csrc/op_json_schema.c +503 -0
  58. package/csrc/op_label_encode.c +328 -53
  59. package/csrc/op_lag.c +181 -0
  60. package/csrc/op_lead.c +141 -89
  61. package/csrc/op_normalize.c +363 -79
  62. package/csrc/op_onehot.c +345 -73
  63. package/csrc/op_pivot.c +1546 -162
  64. package/csrc/op_quarantine.c +189 -0
  65. package/csrc/op_registry.c +2062 -166
  66. package/csrc/op_rename.c +41 -50
  67. package/csrc/op_replace.c +270 -118
  68. package/csrc/op_rleid.c +297 -0
  69. package/csrc/op_rowid.c +559 -0
  70. package/csrc/op_sample.c +80 -23
  71. package/csrc/op_schema.c +1341 -0
  72. package/csrc/op_schema_infer.c +252 -0
  73. package/csrc/op_select.c +265 -65
  74. package/csrc/op_set.c +3449 -0
  75. package/csrc/op_skip.c +30 -87
  76. package/csrc/op_sort.c +670 -124
  77. package/csrc/op_source_name.c +120 -0
  78. package/csrc/op_split.c +65 -28
  79. package/csrc/op_split_data.c +41 -9
  80. package/csrc/op_stack.c +178 -222
  81. package/csrc/op_stats.c +206 -110
  82. package/csrc/op_step.c +217 -55
  83. package/csrc/op_tail.c +21 -12
  84. package/csrc/op_tee.c +338 -0
  85. package/csrc/op_top.c +260 -53
  86. package/csrc/op_trim.c +48 -19
  87. package/csrc/op_unique.c +1193 -150
  88. package/csrc/op_unpivot.c +100 -66
  89. package/csrc/op_validate.c +601 -24
  90. package/csrc/op_window.c +492 -51
  91. package/csrc/path_policy.c +85 -0
  92. package/csrc/pipeline.c +872 -99
  93. package/csrc/recipes.c +3 -1
  94. package/csrc/report.c +73 -30
  95. package/csrc/selector.c +1097 -0
  96. package/csrc/size_utils.c +352 -0
  97. package/csrc/spill.c +317 -0
  98. package/csrc/spill.h +21 -0
  99. package/csrc/tranfi.h +169 -1
  100. package/csrc/transform.h +209 -0
  101. package/csrc/transform_api.c +2237 -0
  102. package/csrc/transform_categorical.c +923 -0
  103. package/csrc/transform_internal.h +472 -0
  104. package/csrc/transform_json.c +3812 -0
  105. package/csrc/transform_numeric.c +1966 -0
  106. package/csrc/transform_sha256.c +154 -0
  107. package/csrc/transform_wasm.h +162 -0
  108. package/csrc/transform_wasm_api.c +1373 -0
  109. package/csrc/wasm_api.c +70 -9
  110. package/napi_api.c +219 -11
  111. package/napi_transform.c +1648 -0
  112. package/napi_transform.h +8 -0
  113. package/package.json +27 -11
  114. package/scripts/install-native.js +76 -0
  115. package/scripts/prepack.js +64 -0
  116. package/scripts/sync-csrc.js +23 -0
  117. package/src/cli.js +81 -41
  118. package/src/engines/duckdb.js +45 -12
  119. package/src/index.js +661 -42
  120. package/src/memory_policy.js +411 -0
  121. package/src/native.js +1 -5
  122. package/src/pipeline.js +454 -31
  123. package/src/recipe_json.js +80 -0
  124. package/src/server.js +10 -8
  125. package/src/transform.js +403 -0
  126. package/src/transform_error.js +10 -0
  127. package/src/wasm.js +6 -4
  128. package/wasm/index.js +498 -10
  129. package/wasm/tranfi_core.js +0 -0
  130. package/wasm/transform.js +1156 -0
  131. package/wasm/worker.js +786 -0
  132. package/csrc/plan.c +0 -206
@@ -0,0 +1,210 @@
1
+ /*
2
+ * json_path.c -- Small internal JSON Pointer / simple JSONPath resolver.
3
+ *
4
+ * This is intentionally not a full JSONPath implementation. It supports:
5
+ * - JSON Pointer: /a/b/0 with ~0 and ~1 escapes
6
+ * - Simple path: $.a[0].b, $['a'], $["a"], or a[0].b
7
+ */
8
+
9
+ #include "internal.h"
10
+ #include "cJSON.h"
11
+ #include <ctype.h>
12
+ #include <stdlib.h>
13
+ #include <string.h>
14
+
15
+ #define TF_JSON_PATH_MAX_INDEX 2147483647
16
+ #define TF_JSON_PATH_MAX_DEPTH 256
17
+
18
+ static int json_path_depth_fail(void) {
19
+ tf_set_last_error("json path nesting too deep");
20
+ return TF_ERROR;
21
+ }
22
+
23
+ int tf_json_path_validate(const char *path) {
24
+ if (!path) return TF_ERROR;
25
+ if (path[0] == '\0') return TF_OK;
26
+
27
+ size_t depth = 0;
28
+ if (path[0] == '/') {
29
+ for (const char *p = path; *p; p++) {
30
+ if (*p == '/' && ++depth > TF_JSON_PATH_MAX_DEPTH) return json_path_depth_fail();
31
+ }
32
+ return TF_OK;
33
+ }
34
+
35
+ const char *p = path;
36
+ if (*p == '$') p++;
37
+ while (*p) {
38
+ if (*p == '.') {
39
+ p++;
40
+ if (++depth > TF_JSON_PATH_MAX_DEPTH) return json_path_depth_fail();
41
+ while (*p && *p != '.' && *p != '[') p++;
42
+ continue;
43
+ }
44
+ if (*p == '[') {
45
+ if (++depth > TF_JSON_PATH_MAX_DEPTH) return json_path_depth_fail();
46
+ p++;
47
+ if (*p == '\'' || *p == '"') {
48
+ char quote = *p++;
49
+ while (*p && *p != quote) p++;
50
+ if (*p == quote) p++;
51
+ }
52
+ while (*p && *p != ']') p++;
53
+ if (*p == ']') p++;
54
+ continue;
55
+ }
56
+ if (++depth > TF_JSON_PATH_MAX_DEPTH) return json_path_depth_fail();
57
+ while (*p && *p != '.' && *p != '[') p++;
58
+ }
59
+ return TF_OK;
60
+ }
61
+
62
+ static char *dup_range(const char *start, size_t len) {
63
+ char *s = malloc(len + 1);
64
+ if (!s) return NULL;
65
+ memcpy(s, start, len);
66
+ s[len] = '\0';
67
+ return s;
68
+ }
69
+
70
+ static int parse_array_index(const char *s, int *out) {
71
+ if (!s || !*s) return 0;
72
+ long idx = 0;
73
+ for (const char *p = s; *p; p++) {
74
+ if (!isdigit((unsigned char)*p)) return 0;
75
+ idx = idx * 10 + (*p - '0');
76
+ if (idx > TF_JSON_PATH_MAX_INDEX) return 0;
77
+ }
78
+ *out = (int)idx;
79
+ return 1;
80
+ }
81
+
82
+ static char *decode_pointer_segment(const char *start, size_t len) {
83
+ char *s = malloc(len + 1);
84
+ if (!s) return NULL;
85
+ size_t j = 0;
86
+ for (size_t i = 0; i < len; i++) {
87
+ if (start[i] == '~' && i + 1 < len) {
88
+ if (start[i + 1] == '0') { s[j++] = '~'; i++; continue; }
89
+ if (start[i + 1] == '1') { s[j++] = '/'; i++; continue; }
90
+ }
91
+ s[j++] = start[i];
92
+ }
93
+ s[j] = '\0';
94
+ return s;
95
+ }
96
+
97
+ static const cJSON *descend_key_or_index(const cJSON *cur, const char *segment) {
98
+ if (!cur || !segment) return NULL;
99
+ if (cJSON_IsObject(cur)) {
100
+ return cJSON_GetObjectItemCaseSensitive((cJSON *)cur, segment);
101
+ }
102
+ if (cJSON_IsArray(cur)) {
103
+ int idx = -1;
104
+ if (!parse_array_index(segment, &idx)) return NULL;
105
+ return cJSON_GetArrayItem((cJSON *)cur, idx);
106
+ }
107
+ return NULL;
108
+ }
109
+
110
+ static const cJSON *resolve_json_pointer(const cJSON *root, const char *path) {
111
+ if (!path || path[0] == '\0') return root;
112
+ if (path[0] != '/') return NULL;
113
+
114
+ const cJSON *cur = root;
115
+ const char *p = path;
116
+ size_t depth = 0;
117
+ while (*p == '/') {
118
+ if (++depth > TF_JSON_PATH_MAX_DEPTH) {
119
+ tf_set_last_error("json path nesting too deep");
120
+ return NULL;
121
+ }
122
+ p++;
123
+ const char *start = p;
124
+ while (*p && *p != '/') p++;
125
+ char *segment = decode_pointer_segment(start, (size_t)(p - start));
126
+ if (!segment) return NULL;
127
+ cur = descend_key_or_index(cur, segment);
128
+ free(segment);
129
+ if (!cur) return NULL;
130
+ }
131
+ return *p == '\0' ? cur : NULL;
132
+ }
133
+
134
+ static const cJSON *resolve_simple_path(const cJSON *root, const char *path) {
135
+ if (!path || path[0] == '\0') return root;
136
+
137
+ const cJSON *cur = root;
138
+ const char *p = path;
139
+ if (*p == '$') p++;
140
+ if (*p == '\0') return cur;
141
+
142
+ size_t depth = 0;
143
+ while (*p) {
144
+ if (++depth > TF_JSON_PATH_MAX_DEPTH) {
145
+ tf_set_last_error("json path nesting too deep");
146
+ return NULL;
147
+ }
148
+ if (*p == '.') {
149
+ p++;
150
+ if (*p == '\0' || *p == '.' || *p == '[') return NULL;
151
+ const char *start = p;
152
+ while (*p && *p != '.' && *p != '[') p++;
153
+ char *key = dup_range(start, (size_t)(p - start));
154
+ if (!key) return NULL;
155
+ cur = cJSON_IsObject(cur) ? cJSON_GetObjectItemCaseSensitive((cJSON *)cur, key) : NULL;
156
+ free(key);
157
+ if (!cur) return NULL;
158
+ continue;
159
+ }
160
+
161
+ if (*p == '[') {
162
+ p++;
163
+ if (*p == '\'' || *p == '"') {
164
+ char quote = *p++;
165
+ const char *start = p;
166
+ while (*p && *p != quote) p++;
167
+ if (*p != quote) return NULL;
168
+ char *key = dup_range(start, (size_t)(p - start));
169
+ if (!key) return NULL;
170
+ p++;
171
+ if (*p != ']') { free(key); return NULL; }
172
+ p++;
173
+ cur = cJSON_IsObject(cur) ? cJSON_GetObjectItemCaseSensitive((cJSON *)cur, key) : NULL;
174
+ free(key);
175
+ if (!cur) return NULL;
176
+ continue;
177
+ }
178
+
179
+ const char *start = p;
180
+ while (*p && *p != ']') p++;
181
+ if (*p != ']') return NULL;
182
+ char *idx_s = dup_range(start, (size_t)(p - start));
183
+ if (!idx_s) return NULL;
184
+ int idx = -1;
185
+ int ok = parse_array_index(idx_s, &idx);
186
+ free(idx_s);
187
+ if (!ok || !cJSON_IsArray(cur)) return NULL;
188
+ cur = cJSON_GetArrayItem((cJSON *)cur, idx);
189
+ if (!cur) return NULL;
190
+ p++;
191
+ continue;
192
+ }
193
+
194
+ const char *start = p;
195
+ while (*p && *p != '.' && *p != '[') p++;
196
+ if (p == start) return NULL;
197
+ char *key = dup_range(start, (size_t)(p - start));
198
+ if (!key) return NULL;
199
+ cur = cJSON_IsObject(cur) ? cJSON_GetObjectItemCaseSensitive((cJSON *)cur, key) : NULL;
200
+ free(key);
201
+ if (!cur) return NULL;
202
+ }
203
+ return cur;
204
+ }
205
+
206
+ const cJSON *tf_json_path_resolve(const cJSON *root, const char *path) {
207
+ if (!path) return NULL;
208
+ if (path[0] == '\0' || path[0] == '/') return resolve_json_pointer(root, path);
209
+ return resolve_simple_path(root, path);
210
+ }