tranfi 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -0
- package/NOTICE +8 -0
- package/README.md +272 -40
- package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
- package/app/index.html +1 -1
- package/binding.gyp +55 -3
- package/csrc/arena.c +7 -5
- package/csrc/batch.c +818 -71
- package/csrc/buffer.c +84 -8
- package/csrc/cJSON.c +262 -19
- package/csrc/cJSON.h +17 -1
- package/csrc/codec_csv.c +1074 -181
- package/csrc/codec_jsonl.c +830 -118
- package/csrc/codec_table.c +108 -78
- package/csrc/codec_text.c +286 -68
- package/csrc/compiler.c +31 -3
- package/csrc/config.h +21 -0
- package/csrc/dsl.c +4722 -485
- package/csrc/expr.c +363 -55
- package/csrc/expr.h +2 -0
- package/csrc/internal.h +316 -27
- package/csrc/ir.c +65 -18
- package/csrc/ir.h +41 -0
- package/csrc/ir_schema.c +20 -5
- package/csrc/ir_serialize.c +68 -6
- package/csrc/ir_sql.c +796 -185
- package/csrc/ir_validate.c +462 -6
- package/csrc/json_path.c +210 -0
- package/csrc/main.c +879 -30
- package/csrc/memory_estimate.c +477 -0
- package/csrc/op_acf.c +171 -21
- package/csrc/op_across.c +477 -0
- package/csrc/op_anomaly.c +167 -32
- package/csrc/op_assert.c +761 -0
- package/csrc/op_bin.c +168 -29
- package/csrc/op_cast.c +383 -55
- package/csrc/op_clip.c +30 -19
- package/csrc/op_date_trunc.c +208 -34
- package/csrc/op_datetime.c +259 -77
- package/csrc/op_derive.c +65 -97
- package/csrc/op_diff.c +146 -30
- package/csrc/op_ewma.c +149 -30
- package/csrc/op_explode.c +124 -26
- package/csrc/op_fill_down.c +125 -53
- package/csrc/op_fill_null.c +176 -31
- package/csrc/op_filter.c +89 -40
- package/csrc/op_frequency.c +571 -43
- package/csrc/op_grep.c +36 -18
- package/csrc/op_group_agg.c +1790 -119
- package/csrc/op_hash.c +48 -15
- package/csrc/op_head.c +21 -86
- package/csrc/op_interpolate.c +268 -62
- package/csrc/op_join.c +2700 -182
- package/csrc/op_json_extract.c +227 -0
- package/csrc/op_json_filter.c +384 -0
- package/csrc/op_json_flatten.c +293 -0
- package/csrc/op_json_schema.c +503 -0
- package/csrc/op_label_encode.c +328 -53
- package/csrc/op_lag.c +181 -0
- package/csrc/op_lead.c +141 -89
- package/csrc/op_normalize.c +363 -79
- package/csrc/op_onehot.c +345 -73
- package/csrc/op_pivot.c +1546 -162
- package/csrc/op_quarantine.c +189 -0
- package/csrc/op_registry.c +2062 -166
- package/csrc/op_rename.c +41 -50
- package/csrc/op_replace.c +270 -118
- package/csrc/op_rleid.c +297 -0
- package/csrc/op_rowid.c +559 -0
- package/csrc/op_sample.c +80 -23
- package/csrc/op_schema.c +1341 -0
- package/csrc/op_schema_infer.c +252 -0
- package/csrc/op_select.c +265 -65
- package/csrc/op_set.c +3449 -0
- package/csrc/op_skip.c +30 -87
- package/csrc/op_sort.c +670 -124
- package/csrc/op_source_name.c +120 -0
- package/csrc/op_split.c +65 -28
- package/csrc/op_split_data.c +41 -9
- package/csrc/op_stack.c +178 -222
- package/csrc/op_stats.c +206 -110
- package/csrc/op_step.c +217 -55
- package/csrc/op_tail.c +21 -12
- package/csrc/op_tee.c +338 -0
- package/csrc/op_top.c +260 -53
- package/csrc/op_trim.c +48 -19
- package/csrc/op_unique.c +1193 -150
- package/csrc/op_unpivot.c +100 -66
- package/csrc/op_validate.c +601 -24
- package/csrc/op_window.c +492 -51
- package/csrc/path_policy.c +85 -0
- package/csrc/pipeline.c +872 -99
- package/csrc/recipes.c +3 -1
- package/csrc/report.c +73 -30
- package/csrc/selector.c +1097 -0
- package/csrc/size_utils.c +348 -0
- package/csrc/spill.c +317 -0
- package/csrc/spill.h +21 -0
- package/csrc/tranfi.h +169 -1
- package/csrc/transform.h +209 -0
- package/csrc/transform_api.c +2237 -0
- package/csrc/transform_categorical.c +923 -0
- package/csrc/transform_internal.h +472 -0
- package/csrc/transform_json.c +3812 -0
- package/csrc/transform_numeric.c +1966 -0
- package/csrc/transform_sha256.c +154 -0
- package/csrc/transform_wasm.h +162 -0
- package/csrc/transform_wasm_api.c +1373 -0
- package/csrc/wasm_api.c +70 -9
- package/napi_api.c +219 -11
- package/napi_transform.c +1648 -0
- package/napi_transform.h +8 -0
- package/package.json +27 -11
- package/scripts/install-native.js +76 -0
- package/scripts/prepack.js +64 -0
- package/scripts/sync-csrc.js +23 -0
- package/src/cli.js +8 -11
- package/src/engines/duckdb.js +45 -12
- package/src/index.js +661 -42
- package/src/memory_policy.js +411 -0
- package/src/native.js +1 -5
- package/src/pipeline.js +454 -31
- package/src/recipe_json.js +80 -0
- package/src/server.js +10 -8
- package/src/transform.js +403 -0
- package/src/transform_error.js +10 -0
- package/src/wasm.js +6 -4
- package/wasm/index.js +498 -10
- package/wasm/tranfi_core.js +0 -0
- package/wasm/transform.js +1156 -0
- package/wasm/worker.js +786 -0
- package/csrc/plan.c +0 -206
package/csrc/ir_validate.c
CHANGED
|
@@ -10,15 +10,12 @@
|
|
|
10
10
|
* 6. No multiple decoders or encoders
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
#include "
|
|
13
|
+
#include "internal.h"
|
|
14
14
|
#include "cJSON.h"
|
|
15
15
|
#include <stdlib.h>
|
|
16
16
|
#include <string.h>
|
|
17
17
|
#include <stdio.h>
|
|
18
18
|
|
|
19
|
-
#define TF_OK 0
|
|
20
|
-
#define TF_ERROR (-1)
|
|
21
|
-
|
|
22
19
|
static void set_plan_error(tf_ir_plan *plan, const char *msg) {
|
|
23
20
|
free(plan->error);
|
|
24
21
|
plan->error = strdup(msg);
|
|
@@ -31,7 +28,422 @@ static void set_plan_errorf(tf_ir_plan *plan, const char *fmt, const char *detai
|
|
|
31
28
|
plan->error = strdup(buf);
|
|
32
29
|
}
|
|
33
30
|
|
|
34
|
-
int
|
|
31
|
+
static int node_bool_arg_true(const tf_ir_node *node, const char *name) {
|
|
32
|
+
cJSON *item = node && node->args ? cJSON_GetObjectItemCaseSensitive(node->args, name) : NULL;
|
|
33
|
+
return cJSON_IsTrue(item);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
static int node_array_arg_nonempty(const tf_ir_node *node, const char *name) {
|
|
37
|
+
cJSON *item = node && node->args ? cJSON_GetObjectItemCaseSensitive(node->args, name) : NULL;
|
|
38
|
+
return cJSON_IsArray(item) && cJSON_GetArraySize(item) > 0;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
#define TF_POLICY_PATH_MAX 4096
|
|
43
|
+
|
|
44
|
+
static int path_is_absolute(const char *path) {
|
|
45
|
+
if (!path || !path[0]) return 0;
|
|
46
|
+
if (path[0] == '/' || path[0] == '\\') return 1;
|
|
47
|
+
return ((path[0] >= 'A' && path[0] <= 'Z') || (path[0] >= 'a' && path[0] <= 'z')) &&
|
|
48
|
+
path[1] == ':';
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
static int append_char(char *out, size_t out_sz, size_t *len, char ch) {
|
|
52
|
+
if (*len + 1 >= out_sz) return TF_ERROR;
|
|
53
|
+
out[(*len)++] = ch;
|
|
54
|
+
out[*len] = '\0';
|
|
55
|
+
return TF_OK;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
static int append_segment(char *out, size_t out_sz, size_t *len,
|
|
59
|
+
const char *seg, size_t seg_len, size_t prefix_len) {
|
|
60
|
+
if (seg_len == 0) return TF_OK;
|
|
61
|
+
if (*len > 0 && out[*len - 1] != '/' &&
|
|
62
|
+
!(*len == prefix_len && prefix_len > 0 && out[prefix_len - 1] == '/')) {
|
|
63
|
+
if (append_char(out, out_sz, len, '/') != TF_OK) return TF_ERROR;
|
|
64
|
+
}
|
|
65
|
+
if (*len + seg_len >= out_sz) return TF_ERROR;
|
|
66
|
+
memcpy(out + *len, seg, seg_len);
|
|
67
|
+
*len += seg_len;
|
|
68
|
+
out[*len] = '\0';
|
|
69
|
+
return TF_OK;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
static int pop_segment(char *out, size_t *len, size_t prefix_len) {
|
|
73
|
+
if (*len <= prefix_len) return TF_ERROR;
|
|
74
|
+
while (*len > prefix_len && out[*len - 1] != '/') (*len)--;
|
|
75
|
+
if (*len > prefix_len && out[*len - 1] == '/') (*len)--;
|
|
76
|
+
if (*len < prefix_len) *len = prefix_len;
|
|
77
|
+
out[*len] = '\0';
|
|
78
|
+
return TF_OK;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
static int normalize_path_lexical(const char *in, char *out, size_t out_sz) {
|
|
82
|
+
if (!in || !in[0] || !out || out_sz == 0) return TF_ERROR;
|
|
83
|
+
size_t len = 0;
|
|
84
|
+
size_t prefix_len = 0;
|
|
85
|
+
const char *p = in;
|
|
86
|
+
|
|
87
|
+
if (((p[0] >= 'A' && p[0] <= 'Z') || (p[0] >= 'a' && p[0] <= 'z')) && p[1] == ':') {
|
|
88
|
+
if (out_sz < 3) return TF_ERROR;
|
|
89
|
+
out[len++] = p[0];
|
|
90
|
+
out[len++] = ':';
|
|
91
|
+
out[len] = '\0';
|
|
92
|
+
p += 2;
|
|
93
|
+
if (*p == '/' || *p == '\\') {
|
|
94
|
+
if (append_char(out, out_sz, &len, '/') != TF_OK) return TF_ERROR;
|
|
95
|
+
while (*p == '/' || *p == '\\') p++;
|
|
96
|
+
}
|
|
97
|
+
prefix_len = len;
|
|
98
|
+
} else if (*p == '/' || *p == '\\') {
|
|
99
|
+
if (append_char(out, out_sz, &len, '/') != TF_OK) return TF_ERROR;
|
|
100
|
+
while (*p == '/' || *p == '\\') p++;
|
|
101
|
+
prefix_len = len;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
while (*p) {
|
|
105
|
+
while (*p == '/' || *p == '\\') p++;
|
|
106
|
+
const char *seg = p;
|
|
107
|
+
while (*p && *p != '/' && *p != '\\') p++;
|
|
108
|
+
size_t seg_len = (size_t)(p - seg);
|
|
109
|
+
if (seg_len == 0 || (seg_len == 1 && seg[0] == '.')) continue;
|
|
110
|
+
if (seg_len == 2 && seg[0] == '.' && seg[1] == '.') {
|
|
111
|
+
if (pop_segment(out, &len, prefix_len) != TF_OK) return TF_ERROR;
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
if (append_segment(out, out_sz, &len, seg, seg_len, prefix_len) != TF_OK) return TF_ERROR;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
if (len == 0) {
|
|
118
|
+
if (out_sz < 2) return TF_ERROR;
|
|
119
|
+
out[0] = '.';
|
|
120
|
+
out[1] = '\0';
|
|
121
|
+
}
|
|
122
|
+
return TF_OK;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
static int path_is_under_root(const char *root, const char *path) {
|
|
126
|
+
size_t n = strlen(root);
|
|
127
|
+
if (strcmp(root, "/") == 0) return path[0] == '/';
|
|
128
|
+
return strcmp(root, path) == 0 || (strncmp(root, path, n) == 0 && path[n] == '/');
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
static int copy_path_checked(const char *src, char *out, size_t out_sz) {
|
|
132
|
+
if (!src || !out || out_sz == 0) return TF_ERROR;
|
|
133
|
+
size_t n = strlen(src);
|
|
134
|
+
if (n + 1 > out_sz) return TF_ERROR;
|
|
135
|
+
memcpy(out, src, n + 1);
|
|
136
|
+
return TF_OK;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
static int canonicalize_existing_path(const char *path, char *out, size_t out_sz) {
|
|
140
|
+
#ifdef _WIN32
|
|
141
|
+
return normalize_path_lexical(path, out, out_sz);
|
|
142
|
+
#else
|
|
143
|
+
char *resolved = realpath(path, NULL);
|
|
144
|
+
if (!resolved) return TF_ERROR;
|
|
145
|
+
int rc = copy_path_checked(resolved, out, out_sz);
|
|
146
|
+
free(resolved);
|
|
147
|
+
return rc;
|
|
148
|
+
#endif
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
static int resolve_workspace_path(const char *workspace_root, const char *logical,
|
|
152
|
+
char *out, size_t out_sz) {
|
|
153
|
+
char root[TF_POLICY_PATH_MAX];
|
|
154
|
+
char joined[TF_POLICY_PATH_MAX];
|
|
155
|
+
char candidate[TF_POLICY_PATH_MAX];
|
|
156
|
+
char resolved[TF_POLICY_PATH_MAX];
|
|
157
|
+
if (canonicalize_existing_path(workspace_root, root, sizeof(root)) != TF_OK) return TF_ERROR;
|
|
158
|
+
if (path_is_absolute(logical)) {
|
|
159
|
+
if (normalize_path_lexical(logical, candidate, sizeof(candidate)) != TF_OK) return TF_ERROR;
|
|
160
|
+
} else {
|
|
161
|
+
int n = snprintf(joined, sizeof(joined), "%s/%s", root, logical);
|
|
162
|
+
if (n < 0 || (size_t)n >= sizeof(joined)) return TF_ERROR;
|
|
163
|
+
if (normalize_path_lexical(joined, candidate, sizeof(candidate)) != TF_OK) return TF_ERROR;
|
|
164
|
+
}
|
|
165
|
+
if (canonicalize_existing_path(candidate, resolved, sizeof(resolved)) != TF_OK) return TF_ERROR;
|
|
166
|
+
if (!path_is_under_root(root, resolved)) return TF_ERROR;
|
|
167
|
+
if (copy_path_checked(resolved, out, out_sz) != TF_OK) return TF_ERROR;
|
|
168
|
+
return TF_OK;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
static void set_policy_error(tf_ir_plan *plan, const tf_ir_node *node,
|
|
172
|
+
const char *arg, const char *reason) {
|
|
173
|
+
free(plan->error);
|
|
174
|
+
char buf[384];
|
|
175
|
+
snprintf(buf, sizeof(buf), "capability denied: op '%s' arg '%s' %s",
|
|
176
|
+
node && node->op ? node->op : "unknown", arg ? arg : "?", reason ? reason : "rejected");
|
|
177
|
+
plan->error = strdup(buf);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
static int policy_rewrite_path_arg(tf_ir_plan *plan, tf_ir_node *node,
|
|
181
|
+
const tf_host_policy *policy,
|
|
182
|
+
const char *arg, int is_spill, int is_rules_file) {
|
|
183
|
+
cJSON *item = node && node->args ? cJSON_GetObjectItemCaseSensitive(node->args, arg) : NULL;
|
|
184
|
+
if (!cJSON_IsString(item) || !item->valuestring || item->valuestring[0] == '\0') return TF_OK;
|
|
185
|
+
if (!policy) return TF_OK;
|
|
186
|
+
|
|
187
|
+
if (!policy->allow_fs) {
|
|
188
|
+
set_policy_error(plan, node, arg, "requires filesystem access (allow_fs=false)");
|
|
189
|
+
return TF_ERROR;
|
|
190
|
+
}
|
|
191
|
+
if (is_spill && !policy->allow_spill) {
|
|
192
|
+
set_policy_error(plan, node, arg, "requires spill access (allow_spill=false)");
|
|
193
|
+
return TF_ERROR;
|
|
194
|
+
}
|
|
195
|
+
if (is_rules_file && !policy->allow_rules_file) {
|
|
196
|
+
set_policy_error(plan, node, arg, "requires rules_file access (allow_rules_file=false)");
|
|
197
|
+
return TF_ERROR;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if (policy->resolve_path || (policy->workspace_root && policy->workspace_root[0])) {
|
|
201
|
+
char resolved[TF_POLICY_PATH_MAX];
|
|
202
|
+
int rc = policy->resolve_path
|
|
203
|
+
? policy->resolve_path(item->valuestring, resolved, sizeof(resolved), policy->user)
|
|
204
|
+
: resolve_workspace_path(policy->workspace_root, item->valuestring, resolved, sizeof(resolved));
|
|
205
|
+
if (rc != TF_OK || resolved[0] == '\0') {
|
|
206
|
+
set_policy_error(plan, node, arg, "was rejected by host path resolver");
|
|
207
|
+
return TF_ERROR;
|
|
208
|
+
}
|
|
209
|
+
if (!cJSON_SetValuestring(item, resolved)) {
|
|
210
|
+
set_plan_error(plan, "out of memory while resolving host path");
|
|
211
|
+
return TF_ERROR;
|
|
212
|
+
}
|
|
213
|
+
if (!policy->resolve_path && !is_spill) {
|
|
214
|
+
const char *hidden = NULL;
|
|
215
|
+
if (strcmp(arg, "file") == 0) {
|
|
216
|
+
hidden = TF_POLICY_VALIDATED_FILE_PATH_ARG;
|
|
217
|
+
} else if (strcmp(arg, "rules_file") == 0 || strcmp(arg, "rulesFile") == 0) {
|
|
218
|
+
hidden = TF_POLICY_VALIDATED_RULES_FILE_PATH_ARG;
|
|
219
|
+
}
|
|
220
|
+
if (hidden) {
|
|
221
|
+
cJSON_DeleteItemFromObjectCaseSensitive(node->args, hidden);
|
|
222
|
+
if (tf_json_add_string(node->args, hidden, resolved) != TF_OK) {
|
|
223
|
+
set_plan_error(plan, "out of memory while recording host path");
|
|
224
|
+
return TF_ERROR;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return TF_OK;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
static int enforce_host_policy(tf_ir_plan *plan, tf_ir_node *node, const tf_host_policy *policy) {
|
|
233
|
+
if (!policy) return TF_OK;
|
|
234
|
+
if (!policy->allow_net && (node->caps & TF_CAP_NET)) {
|
|
235
|
+
set_policy_error(plan, node, "network", "requires network access (allow_net=false)");
|
|
236
|
+
return TF_ERROR;
|
|
237
|
+
}
|
|
238
|
+
if (!policy->allow_blocking && node->memory_class == TF_MEM_BLOCKING) {
|
|
239
|
+
set_policy_error(plan, node, "memory", "is blocking (allow_blocking=false)");
|
|
240
|
+
return TF_ERROR;
|
|
241
|
+
}
|
|
242
|
+
if (policy_rewrite_path_arg(plan, node, policy, "file", 0, 0) != TF_OK) return TF_ERROR;
|
|
243
|
+
if (policy_rewrite_path_arg(plan, node, policy, "spill_dir", 1, 0) != TF_OK) return TF_ERROR;
|
|
244
|
+
if (policy_rewrite_path_arg(plan, node, policy, "rules_file", 0, 1) != TF_OK) return TF_ERROR;
|
|
245
|
+
if (policy_rewrite_path_arg(plan, node, policy, "rulesFile", 0, 1) != TF_OK) return TF_ERROR;
|
|
246
|
+
return TF_OK;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
static int node_string_arg_nonempty(const tf_ir_node *node, const char *name) {
|
|
250
|
+
cJSON *item = node && node->args ? cJSON_GetObjectItemCaseSensitive(node->args, name) : NULL;
|
|
251
|
+
return cJSON_IsString(item) && item->valuestring && item->valuestring[0] != '\0';
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
static int node_string_arg_equals(const tf_ir_node *node, const char *name, const char *value) {
|
|
255
|
+
cJSON *item = node && node->args ? cJSON_GetObjectItemCaseSensitive(node->args, name) : NULL;
|
|
256
|
+
return cJSON_IsString(item) && item->valuestring && value &&
|
|
257
|
+
strcmp(item->valuestring, value) == 0;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
static int node_op_is_unique(const tf_ir_node *node) {
|
|
261
|
+
return node && node->op &&
|
|
262
|
+
(strcmp(node->op, "unique") == 0 || strcmp(node->op, "dedup") == 0);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
static int node_op_is_group_agg(const tf_ir_node *node) {
|
|
266
|
+
return node && node->op && strcmp(node->op, "group-agg") == 0;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
static int node_array_arg_missing_or_empty(const tf_ir_node *node, const char *name) {
|
|
270
|
+
cJSON *item = node && node->args ? cJSON_GetObjectItemCaseSensitive(node->args, name) : NULL;
|
|
271
|
+
if (!item) return 1;
|
|
272
|
+
if (!cJSON_IsArray(item)) return 0;
|
|
273
|
+
return cJSON_GetArraySize(item) <= 0;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
static int node_number_arg_positive(const tf_ir_node *node, const char *name) {
|
|
277
|
+
cJSON *item = node && node->args ? cJSON_GetObjectItemCaseSensitive(node->args, name) : NULL;
|
|
278
|
+
return cJSON_IsNumber(item) && item->valuedouble > 0.0;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
static int node_op_is_row_set_spillable(const tf_ir_node *node) {
|
|
282
|
+
return node && node->op &&
|
|
283
|
+
(strcmp(node->op, "intersect") == 0 || strcmp(node->op, "setdiff") == 0 ||
|
|
284
|
+
strcmp(node->op, "intersect-all") == 0 || strcmp(node->op, "setdiff-all") == 0 ||
|
|
285
|
+
strcmp(node->op, "union") == 0);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
static int node_op_is_sorted_set_bounded(const tf_ir_node *node) {
|
|
289
|
+
return node && node->op && node_bool_arg_true(node, "sorted") &&
|
|
290
|
+
(strcmp(node->op, "intersect") == 0 || strcmp(node->op, "setdiff") == 0 ||
|
|
291
|
+
strcmp(node->op, "intersect-all") == 0 || strcmp(node->op, "setdiff-all") == 0 ||
|
|
292
|
+
strcmp(node->op, "union") == 0);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
static int node_op_is_filtering_join(const tf_ir_node *node) {
|
|
296
|
+
if (!node || !node->op) return 0;
|
|
297
|
+
if (strcmp(node->op, "semi-join") == 0 || strcmp(node->op, "anti-join") == 0) return 1;
|
|
298
|
+
if (strcmp(node->op, "join") != 0 || !node->args) return 0;
|
|
299
|
+
cJSON *how = cJSON_GetObjectItemCaseSensitive(node->args, "how");
|
|
300
|
+
return cJSON_IsString(how) && how->valuestring &&
|
|
301
|
+
(strcmp(how->valuestring, "semi") == 0 || strcmp(how->valuestring, "anti") == 0);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
static int node_op_is_join_spillable(const tf_ir_node *node) {
|
|
305
|
+
if (!node || !node->op) return 0;
|
|
306
|
+
if (strcmp(node->op, "semi-join") == 0 || strcmp(node->op, "anti-join") == 0) return 1;
|
|
307
|
+
if (strcmp(node->op, "join") != 0) return 0;
|
|
308
|
+
if (!node->args) return 1;
|
|
309
|
+
cJSON *how = cJSON_GetObjectItemCaseSensitive(node->args, "how");
|
|
310
|
+
if (!cJSON_IsString(how) || !how->valuestring) return 1;
|
|
311
|
+
return strcmp(how->valuestring, "inner") == 0 || strcmp(how->valuestring, "left") == 0 ||
|
|
312
|
+
strcmp(how->valuestring, "semi") == 0 || strcmp(how->valuestring, "anti") == 0;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
static void apply_dynamic_contract(tf_ir_node *node) {
|
|
316
|
+
if (!node || !node->op) return;
|
|
317
|
+
if (strcmp(node->op, "validate") == 0 &&
|
|
318
|
+
(node_string_arg_nonempty(node, "rules_file") || node_string_arg_nonempty(node, "rulesFile"))) {
|
|
319
|
+
node->caps |= TF_CAP_FS;
|
|
320
|
+
node->caps &= ~TF_CAP_BROWSER_SAFE;
|
|
321
|
+
}
|
|
322
|
+
if (strcmp(node->op, "pivot") == 0) {
|
|
323
|
+
if (node_string_arg_nonempty(node, "spill_dir")) {
|
|
324
|
+
node->caps |= TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_FS | TF_CAP_DETERMINISTIC;
|
|
325
|
+
node->caps &= ~TF_CAP_BROWSER_SAFE;
|
|
326
|
+
node->memory_class = TF_MEM_EXTERNAL;
|
|
327
|
+
node->emit_class = TF_EMIT_ON_FLUSH;
|
|
328
|
+
node->schema_class = node_array_arg_nonempty(node, "categories") ?
|
|
329
|
+
TF_SCHEMA_PARAMETRIC : TF_SCHEMA_DATA_DEPENDENT;
|
|
330
|
+
node->state_estimate = "O(spill_run_rows * columns + categories) RAM + O(input_rows + output_rows) spill";
|
|
331
|
+
} else if (node_bool_arg_true(node, "sorted") && node_array_arg_nonempty(node, "categories")) {
|
|
332
|
+
node->caps |= TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC;
|
|
333
|
+
node->memory_class = TF_MEM_BOUNDED_STATE;
|
|
334
|
+
node->emit_class = TF_EMIT_MIXED;
|
|
335
|
+
node->schema_class = TF_SCHEMA_PARAMETRIC;
|
|
336
|
+
node->state_estimate = "O(current_group + categories)";
|
|
337
|
+
} else if (node_array_arg_nonempty(node, "categories")) {
|
|
338
|
+
node->schema_class = TF_SCHEMA_PARAMETRIC;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
if (strcmp(node->op, "rowid") == 0 &&
|
|
342
|
+
(node_array_arg_missing_or_empty(node, "columns") || node_bool_arg_true(node, "sorted"))) {
|
|
343
|
+
node->caps |= TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC;
|
|
344
|
+
node->memory_class = TF_MEM_BOUNDED_STATE;
|
|
345
|
+
node->emit_class = TF_EMIT_PER_BATCH;
|
|
346
|
+
node->schema_class = TF_SCHEMA_PARAMETRIC;
|
|
347
|
+
node->state_estimate = node_array_arg_missing_or_empty(node, "columns") ?
|
|
348
|
+
"O(1)" : "O(previous_key + counter)";
|
|
349
|
+
}
|
|
350
|
+
if (node_op_is_unique(node)) {
|
|
351
|
+
if (node_string_arg_nonempty(node, "spill_dir")) {
|
|
352
|
+
node->caps |= TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_FS | TF_CAP_DETERMINISTIC;
|
|
353
|
+
node->caps &= ~TF_CAP_BROWSER_SAFE;
|
|
354
|
+
node->memory_class = TF_MEM_EXTERNAL;
|
|
355
|
+
node->emit_class = TF_EMIT_ON_FLUSH;
|
|
356
|
+
node->schema_class = TF_SCHEMA_STABLE;
|
|
357
|
+
node->state_estimate = "O(spill_run_rows * columns) RAM + O(input_rows) spill";
|
|
358
|
+
} else if (node_string_arg_equals(node, "mode", "approx") || node_bool_arg_true(node, "approx")) {
|
|
359
|
+
node->caps |= TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC;
|
|
360
|
+
node->memory_class = TF_MEM_BOUNDED_STATE;
|
|
361
|
+
node->emit_class = TF_EMIT_PER_BATCH;
|
|
362
|
+
node->schema_class = TF_SCHEMA_STABLE;
|
|
363
|
+
node->state_estimate = "O(bloom_bytes)";
|
|
364
|
+
} else if (node_bool_arg_true(node, "sorted")) {
|
|
365
|
+
node->caps |= TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC;
|
|
366
|
+
node->memory_class = TF_MEM_BOUNDED_STATE;
|
|
367
|
+
node->emit_class = TF_EMIT_PER_BATCH;
|
|
368
|
+
node->schema_class = TF_SCHEMA_STABLE;
|
|
369
|
+
node->state_estimate = "O(previous_key)";
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
if (node_op_is_group_agg(node)) {
|
|
373
|
+
if (node_string_arg_nonempty(node, "spill_dir")) {
|
|
374
|
+
node->caps |= TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_FS | TF_CAP_DETERMINISTIC;
|
|
375
|
+
node->caps &= ~TF_CAP_BROWSER_SAFE;
|
|
376
|
+
node->memory_class = TF_MEM_EXTERNAL;
|
|
377
|
+
node->emit_class = TF_EMIT_ON_FLUSH;
|
|
378
|
+
node->schema_class = TF_SCHEMA_PARAMETRIC;
|
|
379
|
+
node->state_estimate = "O(spill_run_rows * columns) RAM + O(input_rows) spill";
|
|
380
|
+
} else if (node_bool_arg_true(node, "sorted")) {
|
|
381
|
+
node->caps |= TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC;
|
|
382
|
+
node->memory_class = TF_MEM_BOUNDED_STATE;
|
|
383
|
+
node->emit_class = TF_EMIT_MIXED;
|
|
384
|
+
node->schema_class = TF_SCHEMA_PARAMETRIC;
|
|
385
|
+
node->state_estimate = "O(current_group + aggregate_accumulators)";
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
if (node_op_is_sorted_set_bounded(node)) {
|
|
389
|
+
node->caps |= TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_DETERMINISTIC;
|
|
390
|
+
if (node->caps & (TF_CAP_FS | TF_CAP_NET)) node->caps &= ~TF_CAP_BROWSER_SAFE;
|
|
391
|
+
else node->caps |= TF_CAP_BROWSER_SAFE;
|
|
392
|
+
node->memory_class = TF_MEM_BOUNDED_STATE;
|
|
393
|
+
node->emit_class = strcmp(node->op, "union") == 0 ? TF_EMIT_MIXED : TF_EMIT_PER_BATCH;
|
|
394
|
+
node->schema_class = TF_SCHEMA_STABLE;
|
|
395
|
+
node->state_estimate = strcmp(node->op, "union") == 0
|
|
396
|
+
? "O(previous_left_key + current_file_key)"
|
|
397
|
+
: ((strcmp(node->op, "intersect-all") == 0 || strcmp(node->op, "setdiff-all") == 0)
|
|
398
|
+
? "O(current_lookup_run + current_left_run)"
|
|
399
|
+
: "O(previous_left_key + current_lookup_key)");
|
|
400
|
+
}
|
|
401
|
+
if (node_op_is_join_spillable(node) && node_bool_arg_true(node, "sorted") &&
|
|
402
|
+
!node_string_arg_nonempty(node, "spill_dir")) {
|
|
403
|
+
int filtering = node_op_is_filtering_join(node);
|
|
404
|
+
node->caps |= TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_DETERMINISTIC;
|
|
405
|
+
if (node->caps & (TF_CAP_FS | TF_CAP_NET)) node->caps &= ~TF_CAP_BROWSER_SAFE;
|
|
406
|
+
else node->caps |= TF_CAP_BROWSER_SAFE;
|
|
407
|
+
node->memory_class = TF_MEM_BOUNDED_STATE;
|
|
408
|
+
node->emit_class = TF_EMIT_PER_BATCH;
|
|
409
|
+
node->schema_class = filtering ? TF_SCHEMA_STABLE : TF_SCHEMA_DATA_DEPENDENT;
|
|
410
|
+
node->state_estimate = filtering
|
|
411
|
+
? "O(current_lookup_run)"
|
|
412
|
+
: "O(current_lookup_run + max_matches_per_row)";
|
|
413
|
+
}
|
|
414
|
+
if (node_op_is_join_spillable(node) && node_string_arg_nonempty(node, "spill_dir")) {
|
|
415
|
+
int filtering = node_op_is_filtering_join(node);
|
|
416
|
+
node->caps |= TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_FS | TF_CAP_DETERMINISTIC;
|
|
417
|
+
node->caps &= ~TF_CAP_BROWSER_SAFE;
|
|
418
|
+
node->memory_class = TF_MEM_EXTERNAL;
|
|
419
|
+
node->emit_class = TF_EMIT_ON_FLUSH;
|
|
420
|
+
node->schema_class = filtering ? TF_SCHEMA_STABLE : TF_SCHEMA_DATA_DEPENDENT;
|
|
421
|
+
node->state_estimate = filtering
|
|
422
|
+
? "O(spill_run_rows * columns) RAM + O(left_rows + lookup_keys) spill"
|
|
423
|
+
: "O(spill_run_rows * columns + max_matches_per_row) RAM + O(left_rows + lookup_rows + output_rows) spill";
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
if (strcmp(node->op, "assert") == 0 && node_string_arg_nonempty(node, "aggregate")) {
|
|
427
|
+
node->memory_class = TF_MEM_BOUNDED_STATE;
|
|
428
|
+
node->emit_class = TF_EMIT_MIXED;
|
|
429
|
+
node->schema_class = TF_SCHEMA_STABLE;
|
|
430
|
+
node->state_estimate = "O(1) aggregate counters";
|
|
431
|
+
}
|
|
432
|
+
if (node_op_is_row_set_spillable(node) && node_string_arg_nonempty(node, "spill_dir")) {
|
|
433
|
+
node->caps |= TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_FS | TF_CAP_DETERMINISTIC;
|
|
434
|
+
node->caps &= ~TF_CAP_BROWSER_SAFE;
|
|
435
|
+
node->memory_class = TF_MEM_EXTERNAL;
|
|
436
|
+
node->emit_class = TF_EMIT_ON_FLUSH;
|
|
437
|
+
node->schema_class = TF_SCHEMA_STABLE;
|
|
438
|
+
node->state_estimate = strcmp(node->op, "union") == 0
|
|
439
|
+
? "O(spill_run_rows * columns) RAM + O(left_rows + file_rows) spill"
|
|
440
|
+
: ((strcmp(node->op, "intersect-all") == 0 || strcmp(node->op, "setdiff-all") == 0)
|
|
441
|
+
? "O(spill_run_rows * columns) RAM + O(left_rows + lookup_rows) spill"
|
|
442
|
+
: "O(spill_run_rows * columns) RAM + O(left_rows + lookup_keys) spill");
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
static int tf_ir_validate_impl(tf_ir_plan *plan, const tf_host_policy *policy) {
|
|
35
447
|
plan->validated = false;
|
|
36
448
|
free(plan->error);
|
|
37
449
|
plan->error = NULL;
|
|
@@ -55,8 +467,13 @@ int tf_ir_validate(tf_ir_plan *plan) {
|
|
|
55
467
|
return TF_ERROR;
|
|
56
468
|
}
|
|
57
469
|
|
|
58
|
-
/* Populate
|
|
470
|
+
/* Populate execution contract metadata from registry. */
|
|
59
471
|
node->caps = entry->caps;
|
|
472
|
+
node->memory_class = entry->memory_class;
|
|
473
|
+
node->emit_class = entry->emit_class;
|
|
474
|
+
node->schema_class = entry->schema_class;
|
|
475
|
+
node->state_estimate = entry->state_estimate ? entry->state_estimate : "unknown";
|
|
476
|
+
apply_dynamic_contract(node);
|
|
60
477
|
|
|
61
478
|
/* Check decoder/encoder placement */
|
|
62
479
|
if (entry->kind == TF_OP_DECODER) {
|
|
@@ -98,6 +515,35 @@ int tf_ir_validate(tf_ir_plan *plan) {
|
|
|
98
515
|
return TF_ERROR;
|
|
99
516
|
}
|
|
100
517
|
}
|
|
518
|
+
if (strcmp(node->op, "validate") == 0) {
|
|
519
|
+
cJSON *expr = cJSON_GetObjectItemCaseSensitive(node->args, "expr");
|
|
520
|
+
cJSON *rules = cJSON_GetObjectItemCaseSensitive(node->args, "rules");
|
|
521
|
+
cJSON *rules_file = cJSON_GetObjectItemCaseSensitive(node->args, "rules_file");
|
|
522
|
+
if (!rules_file) rules_file = cJSON_GetObjectItemCaseSensitive(node->args, "rulesFile");
|
|
523
|
+
if (!expr && !rules && !rules_file) {
|
|
524
|
+
set_plan_error(plan, "op 'validate' requires expr, rules, or rules_file");
|
|
525
|
+
return TF_ERROR;
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
if (strcmp(node->op, "assert") == 0) {
|
|
529
|
+
cJSON *expr = cJSON_GetObjectItemCaseSensitive(node->args, "expr");
|
|
530
|
+
cJSON *agg = cJSON_GetObjectItemCaseSensitive(node->args, "aggregate");
|
|
531
|
+
if (!agg) agg = cJSON_GetObjectItemCaseSensitive(node->args, "agg");
|
|
532
|
+
if (!expr && !agg) {
|
|
533
|
+
set_plan_error(plan, "op 'assert' requires expr or aggregate");
|
|
534
|
+
return TF_ERROR;
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
if (node_op_is_join_spillable(node) && !node_op_is_filtering_join(node) &&
|
|
538
|
+
(node_bool_arg_true(node, "sorted") || node_string_arg_nonempty(node, "spill_dir")) &&
|
|
539
|
+
!node_number_arg_positive(node, "max_matches_per_row")) {
|
|
540
|
+
set_plan_error(plan,
|
|
541
|
+
"op 'join' sorted=true or spill_dir for inner/left joins requires max_matches_per_row");
|
|
542
|
+
return TF_ERROR;
|
|
543
|
+
}
|
|
544
|
+
if (enforce_host_policy(plan, node, policy) != TF_OK) {
|
|
545
|
+
return TF_ERROR;
|
|
546
|
+
}
|
|
101
547
|
}
|
|
102
548
|
|
|
103
549
|
if (!has_decoder) {
|
|
@@ -118,3 +564,13 @@ int tf_ir_validate(tf_ir_plan *plan) {
|
|
|
118
564
|
plan->validated = true;
|
|
119
565
|
return TF_OK;
|
|
120
566
|
}
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
int tf_ir_validate_with_host_policy(tf_ir_plan *plan, const tf_host_policy *policy) {
|
|
570
|
+
if (!plan) return TF_ERROR;
|
|
571
|
+
return tf_ir_validate_impl(plan, policy);
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
int tf_ir_validate(tf_ir_plan *plan) {
|
|
575
|
+
return tf_ir_validate_with_host_policy(plan, NULL);
|
|
576
|
+
}
|