tranfi 0.1.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -0
- package/NOTICE +8 -0
- package/README.md +443 -51
- package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
- package/app/index.html +1 -1
- package/binding.gyp +55 -3
- package/csrc/arena.c +7 -5
- package/csrc/batch.c +818 -71
- package/csrc/buffer.c +84 -8
- package/csrc/cJSON.c +262 -19
- package/csrc/cJSON.h +17 -1
- package/csrc/codec_csv.c +1074 -181
- package/csrc/codec_jsonl.c +830 -118
- package/csrc/codec_table.c +108 -78
- package/csrc/codec_text.c +286 -68
- package/csrc/compiler.c +31 -3
- package/csrc/config.h +21 -0
- package/csrc/dsl.c +4722 -485
- package/csrc/expr.c +363 -55
- package/csrc/expr.h +2 -0
- package/csrc/internal.h +316 -27
- package/csrc/ir.c +65 -18
- package/csrc/ir.h +41 -0
- package/csrc/ir_schema.c +20 -5
- package/csrc/ir_serialize.c +68 -6
- package/csrc/ir_sql.c +796 -185
- package/csrc/ir_validate.c +462 -6
- package/csrc/json_path.c +210 -0
- package/csrc/main.c +879 -30
- package/csrc/memory_estimate.c +477 -0
- package/csrc/op_acf.c +171 -21
- package/csrc/op_across.c +477 -0
- package/csrc/op_anomaly.c +167 -32
- package/csrc/op_assert.c +761 -0
- package/csrc/op_bin.c +168 -29
- package/csrc/op_cast.c +383 -55
- package/csrc/op_clip.c +30 -19
- package/csrc/op_date_trunc.c +208 -34
- package/csrc/op_datetime.c +259 -77
- package/csrc/op_derive.c +65 -97
- package/csrc/op_diff.c +146 -30
- package/csrc/op_ewma.c +149 -30
- package/csrc/op_explode.c +124 -26
- package/csrc/op_fill_down.c +125 -53
- package/csrc/op_fill_null.c +176 -31
- package/csrc/op_filter.c +89 -40
- package/csrc/op_frequency.c +571 -43
- package/csrc/op_grep.c +36 -18
- package/csrc/op_group_agg.c +1790 -119
- package/csrc/op_hash.c +48 -15
- package/csrc/op_head.c +21 -86
- package/csrc/op_interpolate.c +268 -62
- package/csrc/op_join.c +2700 -182
- package/csrc/op_json_extract.c +227 -0
- package/csrc/op_json_filter.c +384 -0
- package/csrc/op_json_flatten.c +293 -0
- package/csrc/op_json_schema.c +503 -0
- package/csrc/op_label_encode.c +328 -53
- package/csrc/op_lag.c +181 -0
- package/csrc/op_lead.c +141 -89
- package/csrc/op_normalize.c +363 -79
- package/csrc/op_onehot.c +345 -73
- package/csrc/op_pivot.c +1546 -162
- package/csrc/op_quarantine.c +189 -0
- package/csrc/op_registry.c +2062 -166
- package/csrc/op_rename.c +41 -50
- package/csrc/op_replace.c +270 -118
- package/csrc/op_rleid.c +297 -0
- package/csrc/op_rowid.c +559 -0
- package/csrc/op_sample.c +80 -23
- package/csrc/op_schema.c +1341 -0
- package/csrc/op_schema_infer.c +252 -0
- package/csrc/op_select.c +265 -65
- package/csrc/op_set.c +3449 -0
- package/csrc/op_skip.c +30 -87
- package/csrc/op_sort.c +670 -124
- package/csrc/op_source_name.c +120 -0
- package/csrc/op_split.c +65 -28
- package/csrc/op_split_data.c +41 -9
- package/csrc/op_stack.c +178 -222
- package/csrc/op_stats.c +206 -110
- package/csrc/op_step.c +217 -55
- package/csrc/op_tail.c +21 -12
- package/csrc/op_tee.c +338 -0
- package/csrc/op_top.c +260 -53
- package/csrc/op_trim.c +48 -19
- package/csrc/op_unique.c +1193 -150
- package/csrc/op_unpivot.c +100 -66
- package/csrc/op_validate.c +601 -24
- package/csrc/op_window.c +492 -51
- package/csrc/path_policy.c +85 -0
- package/csrc/pipeline.c +872 -99
- package/csrc/recipes.c +3 -1
- package/csrc/report.c +73 -30
- package/csrc/selector.c +1097 -0
- package/csrc/size_utils.c +352 -0
- package/csrc/spill.c +317 -0
- package/csrc/spill.h +21 -0
- package/csrc/tranfi.h +169 -1
- package/csrc/transform.h +209 -0
- package/csrc/transform_api.c +2237 -0
- package/csrc/transform_categorical.c +923 -0
- package/csrc/transform_internal.h +472 -0
- package/csrc/transform_json.c +3812 -0
- package/csrc/transform_numeric.c +1966 -0
- package/csrc/transform_sha256.c +154 -0
- package/csrc/transform_wasm.h +162 -0
- package/csrc/transform_wasm_api.c +1373 -0
- package/csrc/wasm_api.c +70 -9
- package/napi_api.c +219 -11
- package/napi_transform.c +1648 -0
- package/napi_transform.h +8 -0
- package/package.json +27 -11
- package/scripts/install-native.js +76 -0
- package/scripts/prepack.js +64 -0
- package/scripts/sync-csrc.js +23 -0
- package/src/cli.js +81 -41
- package/src/engines/duckdb.js +45 -12
- package/src/index.js +661 -42
- package/src/memory_policy.js +411 -0
- package/src/native.js +1 -5
- package/src/pipeline.js +454 -31
- package/src/recipe_json.js +80 -0
- package/src/server.js +10 -8
- package/src/transform.js +403 -0
- package/src/transform_error.js +10 -0
- package/src/wasm.js +6 -4
- package/wasm/index.js +498 -10
- package/wasm/tranfi_core.js +0 -0
- package/wasm/transform.js +1156 -0
- package/wasm/worker.js +786 -0
- package/csrc/plan.c +0 -206
package/csrc/json_path.c
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* json_path.c -- Small internal JSON Pointer / simple JSONPath resolver.
|
|
3
|
+
*
|
|
4
|
+
* This is intentionally not a full JSONPath implementation. It supports:
|
|
5
|
+
* - JSON Pointer: /a/b/0 with ~0 and ~1 escapes
|
|
6
|
+
* - Simple path: $.a[0].b, $['a'], $["a"], or a[0].b
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
#include "internal.h"
|
|
10
|
+
#include "cJSON.h"
|
|
11
|
+
#include <ctype.h>
|
|
12
|
+
#include <stdlib.h>
|
|
13
|
+
#include <string.h>
|
|
14
|
+
|
|
15
|
+
#define TF_JSON_PATH_MAX_INDEX 2147483647
|
|
16
|
+
#define TF_JSON_PATH_MAX_DEPTH 256
|
|
17
|
+
|
|
18
|
+
static int json_path_depth_fail(void) {
|
|
19
|
+
tf_set_last_error("json path nesting too deep");
|
|
20
|
+
return TF_ERROR;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
int tf_json_path_validate(const char *path) {
|
|
24
|
+
if (!path) return TF_ERROR;
|
|
25
|
+
if (path[0] == '\0') return TF_OK;
|
|
26
|
+
|
|
27
|
+
size_t depth = 0;
|
|
28
|
+
if (path[0] == '/') {
|
|
29
|
+
for (const char *p = path; *p; p++) {
|
|
30
|
+
if (*p == '/' && ++depth > TF_JSON_PATH_MAX_DEPTH) return json_path_depth_fail();
|
|
31
|
+
}
|
|
32
|
+
return TF_OK;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const char *p = path;
|
|
36
|
+
if (*p == '$') p++;
|
|
37
|
+
while (*p) {
|
|
38
|
+
if (*p == '.') {
|
|
39
|
+
p++;
|
|
40
|
+
if (++depth > TF_JSON_PATH_MAX_DEPTH) return json_path_depth_fail();
|
|
41
|
+
while (*p && *p != '.' && *p != '[') p++;
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
if (*p == '[') {
|
|
45
|
+
if (++depth > TF_JSON_PATH_MAX_DEPTH) return json_path_depth_fail();
|
|
46
|
+
p++;
|
|
47
|
+
if (*p == '\'' || *p == '"') {
|
|
48
|
+
char quote = *p++;
|
|
49
|
+
while (*p && *p != quote) p++;
|
|
50
|
+
if (*p == quote) p++;
|
|
51
|
+
}
|
|
52
|
+
while (*p && *p != ']') p++;
|
|
53
|
+
if (*p == ']') p++;
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
if (++depth > TF_JSON_PATH_MAX_DEPTH) return json_path_depth_fail();
|
|
57
|
+
while (*p && *p != '.' && *p != '[') p++;
|
|
58
|
+
}
|
|
59
|
+
return TF_OK;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
static char *dup_range(const char *start, size_t len) {
|
|
63
|
+
char *s = malloc(len + 1);
|
|
64
|
+
if (!s) return NULL;
|
|
65
|
+
memcpy(s, start, len);
|
|
66
|
+
s[len] = '\0';
|
|
67
|
+
return s;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
static int parse_array_index(const char *s, int *out) {
|
|
71
|
+
if (!s || !*s) return 0;
|
|
72
|
+
long idx = 0;
|
|
73
|
+
for (const char *p = s; *p; p++) {
|
|
74
|
+
if (!isdigit((unsigned char)*p)) return 0;
|
|
75
|
+
idx = idx * 10 + (*p - '0');
|
|
76
|
+
if (idx > TF_JSON_PATH_MAX_INDEX) return 0;
|
|
77
|
+
}
|
|
78
|
+
*out = (int)idx;
|
|
79
|
+
return 1;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
static char *decode_pointer_segment(const char *start, size_t len) {
|
|
83
|
+
char *s = malloc(len + 1);
|
|
84
|
+
if (!s) return NULL;
|
|
85
|
+
size_t j = 0;
|
|
86
|
+
for (size_t i = 0; i < len; i++) {
|
|
87
|
+
if (start[i] == '~' && i + 1 < len) {
|
|
88
|
+
if (start[i + 1] == '0') { s[j++] = '~'; i++; continue; }
|
|
89
|
+
if (start[i + 1] == '1') { s[j++] = '/'; i++; continue; }
|
|
90
|
+
}
|
|
91
|
+
s[j++] = start[i];
|
|
92
|
+
}
|
|
93
|
+
s[j] = '\0';
|
|
94
|
+
return s;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
static const cJSON *descend_key_or_index(const cJSON *cur, const char *segment) {
|
|
98
|
+
if (!cur || !segment) return NULL;
|
|
99
|
+
if (cJSON_IsObject(cur)) {
|
|
100
|
+
return cJSON_GetObjectItemCaseSensitive((cJSON *)cur, segment);
|
|
101
|
+
}
|
|
102
|
+
if (cJSON_IsArray(cur)) {
|
|
103
|
+
int idx = -1;
|
|
104
|
+
if (!parse_array_index(segment, &idx)) return NULL;
|
|
105
|
+
return cJSON_GetArrayItem((cJSON *)cur, idx);
|
|
106
|
+
}
|
|
107
|
+
return NULL;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
static const cJSON *resolve_json_pointer(const cJSON *root, const char *path) {
|
|
111
|
+
if (!path || path[0] == '\0') return root;
|
|
112
|
+
if (path[0] != '/') return NULL;
|
|
113
|
+
|
|
114
|
+
const cJSON *cur = root;
|
|
115
|
+
const char *p = path;
|
|
116
|
+
size_t depth = 0;
|
|
117
|
+
while (*p == '/') {
|
|
118
|
+
if (++depth > TF_JSON_PATH_MAX_DEPTH) {
|
|
119
|
+
tf_set_last_error("json path nesting too deep");
|
|
120
|
+
return NULL;
|
|
121
|
+
}
|
|
122
|
+
p++;
|
|
123
|
+
const char *start = p;
|
|
124
|
+
while (*p && *p != '/') p++;
|
|
125
|
+
char *segment = decode_pointer_segment(start, (size_t)(p - start));
|
|
126
|
+
if (!segment) return NULL;
|
|
127
|
+
cur = descend_key_or_index(cur, segment);
|
|
128
|
+
free(segment);
|
|
129
|
+
if (!cur) return NULL;
|
|
130
|
+
}
|
|
131
|
+
return *p == '\0' ? cur : NULL;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
static const cJSON *resolve_simple_path(const cJSON *root, const char *path) {
|
|
135
|
+
if (!path || path[0] == '\0') return root;
|
|
136
|
+
|
|
137
|
+
const cJSON *cur = root;
|
|
138
|
+
const char *p = path;
|
|
139
|
+
if (*p == '$') p++;
|
|
140
|
+
if (*p == '\0') return cur;
|
|
141
|
+
|
|
142
|
+
size_t depth = 0;
|
|
143
|
+
while (*p) {
|
|
144
|
+
if (++depth > TF_JSON_PATH_MAX_DEPTH) {
|
|
145
|
+
tf_set_last_error("json path nesting too deep");
|
|
146
|
+
return NULL;
|
|
147
|
+
}
|
|
148
|
+
if (*p == '.') {
|
|
149
|
+
p++;
|
|
150
|
+
if (*p == '\0' || *p == '.' || *p == '[') return NULL;
|
|
151
|
+
const char *start = p;
|
|
152
|
+
while (*p && *p != '.' && *p != '[') p++;
|
|
153
|
+
char *key = dup_range(start, (size_t)(p - start));
|
|
154
|
+
if (!key) return NULL;
|
|
155
|
+
cur = cJSON_IsObject(cur) ? cJSON_GetObjectItemCaseSensitive((cJSON *)cur, key) : NULL;
|
|
156
|
+
free(key);
|
|
157
|
+
if (!cur) return NULL;
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
if (*p == '[') {
|
|
162
|
+
p++;
|
|
163
|
+
if (*p == '\'' || *p == '"') {
|
|
164
|
+
char quote = *p++;
|
|
165
|
+
const char *start = p;
|
|
166
|
+
while (*p && *p != quote) p++;
|
|
167
|
+
if (*p != quote) return NULL;
|
|
168
|
+
char *key = dup_range(start, (size_t)(p - start));
|
|
169
|
+
if (!key) return NULL;
|
|
170
|
+
p++;
|
|
171
|
+
if (*p != ']') { free(key); return NULL; }
|
|
172
|
+
p++;
|
|
173
|
+
cur = cJSON_IsObject(cur) ? cJSON_GetObjectItemCaseSensitive((cJSON *)cur, key) : NULL;
|
|
174
|
+
free(key);
|
|
175
|
+
if (!cur) return NULL;
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const char *start = p;
|
|
180
|
+
while (*p && *p != ']') p++;
|
|
181
|
+
if (*p != ']') return NULL;
|
|
182
|
+
char *idx_s = dup_range(start, (size_t)(p - start));
|
|
183
|
+
if (!idx_s) return NULL;
|
|
184
|
+
int idx = -1;
|
|
185
|
+
int ok = parse_array_index(idx_s, &idx);
|
|
186
|
+
free(idx_s);
|
|
187
|
+
if (!ok || !cJSON_IsArray(cur)) return NULL;
|
|
188
|
+
cur = cJSON_GetArrayItem((cJSON *)cur, idx);
|
|
189
|
+
if (!cur) return NULL;
|
|
190
|
+
p++;
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const char *start = p;
|
|
195
|
+
while (*p && *p != '.' && *p != '[') p++;
|
|
196
|
+
if (p == start) return NULL;
|
|
197
|
+
char *key = dup_range(start, (size_t)(p - start));
|
|
198
|
+
if (!key) return NULL;
|
|
199
|
+
cur = cJSON_IsObject(cur) ? cJSON_GetObjectItemCaseSensitive((cJSON *)cur, key) : NULL;
|
|
200
|
+
free(key);
|
|
201
|
+
if (!cur) return NULL;
|
|
202
|
+
}
|
|
203
|
+
return cur;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
const cJSON *tf_json_path_resolve(const cJSON *root, const char *path) {
|
|
207
|
+
if (!path) return NULL;
|
|
208
|
+
if (path[0] == '\0' || path[0] == '/') return resolve_json_pointer(root, path);
|
|
209
|
+
return resolve_simple_path(root, path);
|
|
210
|
+
}
|