tranfi 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -0
- package/NOTICE +8 -0
- package/README.md +272 -40
- package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
- package/app/index.html +1 -1
- package/binding.gyp +55 -3
- package/csrc/arena.c +7 -5
- package/csrc/batch.c +818 -71
- package/csrc/buffer.c +84 -8
- package/csrc/cJSON.c +262 -19
- package/csrc/cJSON.h +17 -1
- package/csrc/codec_csv.c +1074 -181
- package/csrc/codec_jsonl.c +830 -118
- package/csrc/codec_table.c +108 -78
- package/csrc/codec_text.c +286 -68
- package/csrc/compiler.c +31 -3
- package/csrc/config.h +21 -0
- package/csrc/dsl.c +4722 -485
- package/csrc/expr.c +363 -55
- package/csrc/expr.h +2 -0
- package/csrc/internal.h +316 -27
- package/csrc/ir.c +65 -18
- package/csrc/ir.h +41 -0
- package/csrc/ir_schema.c +20 -5
- package/csrc/ir_serialize.c +68 -6
- package/csrc/ir_sql.c +796 -185
- package/csrc/ir_validate.c +462 -6
- package/csrc/json_path.c +210 -0
- package/csrc/main.c +879 -30
- package/csrc/memory_estimate.c +477 -0
- package/csrc/op_acf.c +171 -21
- package/csrc/op_across.c +477 -0
- package/csrc/op_anomaly.c +167 -32
- package/csrc/op_assert.c +761 -0
- package/csrc/op_bin.c +168 -29
- package/csrc/op_cast.c +383 -55
- package/csrc/op_clip.c +30 -19
- package/csrc/op_date_trunc.c +208 -34
- package/csrc/op_datetime.c +259 -77
- package/csrc/op_derive.c +65 -97
- package/csrc/op_diff.c +146 -30
- package/csrc/op_ewma.c +149 -30
- package/csrc/op_explode.c +124 -26
- package/csrc/op_fill_down.c +125 -53
- package/csrc/op_fill_null.c +176 -31
- package/csrc/op_filter.c +89 -40
- package/csrc/op_frequency.c +571 -43
- package/csrc/op_grep.c +36 -18
- package/csrc/op_group_agg.c +1790 -119
- package/csrc/op_hash.c +48 -15
- package/csrc/op_head.c +21 -86
- package/csrc/op_interpolate.c +268 -62
- package/csrc/op_join.c +2700 -182
- package/csrc/op_json_extract.c +227 -0
- package/csrc/op_json_filter.c +384 -0
- package/csrc/op_json_flatten.c +293 -0
- package/csrc/op_json_schema.c +503 -0
- package/csrc/op_label_encode.c +328 -53
- package/csrc/op_lag.c +181 -0
- package/csrc/op_lead.c +141 -89
- package/csrc/op_normalize.c +363 -79
- package/csrc/op_onehot.c +345 -73
- package/csrc/op_pivot.c +1546 -162
- package/csrc/op_quarantine.c +189 -0
- package/csrc/op_registry.c +2062 -166
- package/csrc/op_rename.c +41 -50
- package/csrc/op_replace.c +270 -118
- package/csrc/op_rleid.c +297 -0
- package/csrc/op_rowid.c +559 -0
- package/csrc/op_sample.c +80 -23
- package/csrc/op_schema.c +1341 -0
- package/csrc/op_schema_infer.c +252 -0
- package/csrc/op_select.c +265 -65
- package/csrc/op_set.c +3449 -0
- package/csrc/op_skip.c +30 -87
- package/csrc/op_sort.c +670 -124
- package/csrc/op_source_name.c +120 -0
- package/csrc/op_split.c +65 -28
- package/csrc/op_split_data.c +41 -9
- package/csrc/op_stack.c +178 -222
- package/csrc/op_stats.c +206 -110
- package/csrc/op_step.c +217 -55
- package/csrc/op_tail.c +21 -12
- package/csrc/op_tee.c +338 -0
- package/csrc/op_top.c +260 -53
- package/csrc/op_trim.c +48 -19
- package/csrc/op_unique.c +1193 -150
- package/csrc/op_unpivot.c +100 -66
- package/csrc/op_validate.c +601 -24
- package/csrc/op_window.c +492 -51
- package/csrc/path_policy.c +85 -0
- package/csrc/pipeline.c +872 -99
- package/csrc/recipes.c +3 -1
- package/csrc/report.c +73 -30
- package/csrc/selector.c +1097 -0
- package/csrc/size_utils.c +348 -0
- package/csrc/spill.c +317 -0
- package/csrc/spill.h +21 -0
- package/csrc/tranfi.h +169 -1
- package/csrc/transform.h +209 -0
- package/csrc/transform_api.c +2237 -0
- package/csrc/transform_categorical.c +923 -0
- package/csrc/transform_internal.h +472 -0
- package/csrc/transform_json.c +3812 -0
- package/csrc/transform_numeric.c +1966 -0
- package/csrc/transform_sha256.c +154 -0
- package/csrc/transform_wasm.h +162 -0
- package/csrc/transform_wasm_api.c +1373 -0
- package/csrc/wasm_api.c +70 -9
- package/napi_api.c +219 -11
- package/napi_transform.c +1648 -0
- package/napi_transform.h +8 -0
- package/package.json +27 -11
- package/scripts/install-native.js +76 -0
- package/scripts/prepack.js +64 -0
- package/scripts/sync-csrc.js +23 -0
- package/src/cli.js +8 -11
- package/src/engines/duckdb.js +45 -12
- package/src/index.js +661 -42
- package/src/memory_policy.js +411 -0
- package/src/native.js +1 -5
- package/src/pipeline.js +454 -31
- package/src/recipe_json.js +80 -0
- package/src/server.js +10 -8
- package/src/transform.js +403 -0
- package/src/transform_error.js +10 -0
- package/src/wasm.js +6 -4
- package/wasm/index.js +498 -10
- package/wasm/tranfi_core.js +0 -0
- package/wasm/transform.js +1156 -0
- package/wasm/worker.js +786 -0
- package/csrc/plan.c +0 -206
package/csrc/op_registry.c
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
/*
|
|
2
2
|
* op_registry.c — Declarative registry of all built-in ops.
|
|
3
3
|
*
|
|
4
|
-
* Replaces the hard-coded strcmp chain in plan.c.
|
|
5
4
|
* Each entry describes an op's kind, capabilities, arguments,
|
|
6
5
|
* schema inference callback, and native constructor.
|
|
7
6
|
*/
|
|
@@ -10,18 +9,97 @@
|
|
|
10
9
|
#include "internal.h"
|
|
11
10
|
#include "cJSON.h"
|
|
12
11
|
#include <string.h>
|
|
12
|
+
#include <strings.h>
|
|
13
|
+
#include <stdio.h>
|
|
13
14
|
#include <stdlib.h>
|
|
14
15
|
|
|
15
16
|
/* ---- Schema inference callbacks ---- */
|
|
16
17
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
(void)node; (void)in;
|
|
18
|
+
static void registry_schema_unknown(tf_schema *out) {
|
|
19
|
+
if (!out) return;
|
|
20
|
+
tf_schema_free(out);
|
|
21
21
|
out->col_names = NULL;
|
|
22
22
|
out->col_types = NULL;
|
|
23
23
|
out->n_cols = 0;
|
|
24
24
|
out->known = false;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
static int registry_schema_alloc_known(tf_schema *out, size_t n_cols) {
|
|
28
|
+
if (!out) return TF_ERROR;
|
|
29
|
+
tf_schema_free(out);
|
|
30
|
+
size_t alloc_n = n_cols ? n_cols : 1;
|
|
31
|
+
out->col_names = calloc(alloc_n, sizeof(char *));
|
|
32
|
+
out->col_types = calloc(alloc_n, sizeof(tf_type));
|
|
33
|
+
if (!out->col_names || !out->col_types) {
|
|
34
|
+
tf_schema_free(out);
|
|
35
|
+
return TF_ERROR;
|
|
36
|
+
}
|
|
37
|
+
out->n_cols = n_cols;
|
|
38
|
+
out->known = true;
|
|
39
|
+
return TF_OK;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
static int registry_schema_set_col(tf_schema *out, size_t idx,
|
|
43
|
+
const char *name, tf_type type) {
|
|
44
|
+
if (!out || !out->known || idx >= out->n_cols || !name) return TF_ERROR;
|
|
45
|
+
char *copy = strdup(name);
|
|
46
|
+
if (!copy) return TF_ERROR;
|
|
47
|
+
free(out->col_names[idx]);
|
|
48
|
+
out->col_names[idx] = copy;
|
|
49
|
+
out->col_types[idx] = type;
|
|
50
|
+
return TF_OK;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
static int registry_schema_take_col(tf_schema *out, size_t idx,
|
|
54
|
+
char *name, tf_type type) {
|
|
55
|
+
if (!out || !out->known || idx >= out->n_cols || !name) {
|
|
56
|
+
free(name);
|
|
57
|
+
return TF_ERROR;
|
|
58
|
+
}
|
|
59
|
+
free(out->col_names[idx]);
|
|
60
|
+
out->col_names[idx] = name;
|
|
61
|
+
out->col_types[idx] = type;
|
|
62
|
+
return TF_OK;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
static void registry_set_schema_error(const char *op, const char *detail) {
|
|
66
|
+
char buf[512];
|
|
67
|
+
snprintf(buf, sizeof(buf), "%s schema: %s",
|
|
68
|
+
op ? op : "op", (detail && detail[0]) ? detail : "schema inference failed");
|
|
69
|
+
tf_set_last_error(buf);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
static int registry_schema_copy_input_columns(tf_schema *out, const tf_schema *in) {
|
|
73
|
+
if (!out || !in || !out->known || !in->known || out->n_cols < in->n_cols) return TF_ERROR;
|
|
74
|
+
for (size_t i = 0; i < in->n_cols; i++) {
|
|
75
|
+
if (registry_schema_set_col(out, i, in->col_names[i] ? in->col_names[i] : "",
|
|
76
|
+
in->col_types[i]) != TF_OK) {
|
|
77
|
+
tf_schema_free(out);
|
|
78
|
+
return TF_ERROR;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return TF_OK;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
static int registry_schema_copy_input_plus(tf_schema *out, const tf_schema *in,
|
|
85
|
+
size_t n_extra) {
|
|
86
|
+
if (!in || !in->known) {
|
|
87
|
+
registry_schema_unknown(out);
|
|
88
|
+
return TF_OK;
|
|
89
|
+
}
|
|
90
|
+
size_t total = 0;
|
|
91
|
+
if (tf_size_add(in->n_cols, n_extra, &total) != TF_OK ||
|
|
92
|
+
registry_schema_alloc_known(out, total) != TF_OK) {
|
|
93
|
+
return TF_ERROR;
|
|
94
|
+
}
|
|
95
|
+
return registry_schema_copy_input_columns(out, in);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/* Decoders: output schema unknown until runtime */
|
|
99
|
+
static int infer_schema_unknown(const tf_ir_node *node,
|
|
100
|
+
const tf_schema *in, tf_schema *out) {
|
|
101
|
+
(void)node; (void)in;
|
|
102
|
+
registry_schema_unknown(out);
|
|
25
103
|
return TF_OK;
|
|
26
104
|
}
|
|
27
105
|
|
|
@@ -29,10 +107,7 @@ static int infer_schema_unknown(const tf_ir_node *node,
|
|
|
29
107
|
static int infer_schema_sink(const tf_ir_node *node,
|
|
30
108
|
const tf_schema *in, tf_schema *out) {
|
|
31
109
|
(void)node; (void)in;
|
|
32
|
-
out
|
|
33
|
-
out->col_types = NULL;
|
|
34
|
-
out->n_cols = 0;
|
|
35
|
-
out->known = false;
|
|
110
|
+
registry_schema_unknown(out);
|
|
36
111
|
return TF_OK;
|
|
37
112
|
}
|
|
38
113
|
|
|
@@ -41,35 +116,53 @@ static int infer_schema_passthrough(const tf_ir_node *node,
|
|
|
41
116
|
const tf_schema *in, tf_schema *out) {
|
|
42
117
|
(void)node;
|
|
43
118
|
if (!in->known) {
|
|
44
|
-
out
|
|
45
|
-
out->col_names = NULL;
|
|
46
|
-
out->col_types = NULL;
|
|
47
|
-
out->n_cols = 0;
|
|
119
|
+
registry_schema_unknown(out);
|
|
48
120
|
return TF_OK;
|
|
49
121
|
}
|
|
50
|
-
|
|
51
|
-
return TF_OK;
|
|
122
|
+
return tf_schema_copy_checked(out, in);
|
|
52
123
|
}
|
|
53
124
|
|
|
54
125
|
/* select: output schema = subset of input columns */
|
|
55
126
|
static int infer_schema_select(const tf_ir_node *node,
|
|
56
127
|
const tf_schema *in, tf_schema *out) {
|
|
57
128
|
if (!in->known) {
|
|
58
|
-
out
|
|
59
|
-
out->col_names = NULL;
|
|
60
|
-
out->col_types = NULL;
|
|
61
|
-
out->n_cols = 0;
|
|
129
|
+
registry_schema_unknown(out);
|
|
62
130
|
return TF_OK;
|
|
63
131
|
}
|
|
64
132
|
|
|
65
133
|
cJSON *cols = cJSON_GetObjectItemCaseSensitive(node->args, "columns");
|
|
66
134
|
if (!cols || !cJSON_IsArray(cols)) return TF_ERROR;
|
|
67
135
|
|
|
136
|
+
if (tf_column_selectors_have_syntax_json(cols)) {
|
|
137
|
+
int *indices = NULL;
|
|
138
|
+
size_t n_indices = 0;
|
|
139
|
+
char *error = NULL;
|
|
140
|
+
int rc = tf_column_selectors_resolve_json(cols, in->col_names, in->col_types, in->n_cols,
|
|
141
|
+
&indices, &n_indices, &error);
|
|
142
|
+
if (rc != TF_OK) {
|
|
143
|
+
registry_set_schema_error("select", error);
|
|
144
|
+
free(error);
|
|
145
|
+
return TF_ERROR;
|
|
146
|
+
}
|
|
147
|
+
free(error);
|
|
148
|
+
if (registry_schema_alloc_known(out, n_indices) != TF_OK) {
|
|
149
|
+
free(indices);
|
|
150
|
+
return TF_ERROR;
|
|
151
|
+
}
|
|
152
|
+
for (size_t i = 0; i < n_indices; i++) {
|
|
153
|
+
int ci = indices[i];
|
|
154
|
+
if (registry_schema_set_col(out, i, in->col_names[ci], in->col_types[ci]) != TF_OK) {
|
|
155
|
+
free(indices);
|
|
156
|
+
tf_schema_free(out);
|
|
157
|
+
return TF_ERROR;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
free(indices);
|
|
161
|
+
return TF_OK;
|
|
162
|
+
}
|
|
163
|
+
|
|
68
164
|
int n = cJSON_GetArraySize(cols);
|
|
69
|
-
|
|
70
|
-
out->col_types = calloc(n, sizeof(tf_type));
|
|
71
|
-
out->n_cols = n;
|
|
72
|
-
out->known = true;
|
|
165
|
+
if (n < 0 || registry_schema_alloc_known(out, (size_t)n) != TF_OK) return TF_ERROR;
|
|
73
166
|
|
|
74
167
|
for (int i = 0; i < n; i++) {
|
|
75
168
|
cJSON *item = cJSON_GetArrayItem(cols, i);
|
|
@@ -82,18 +175,125 @@ static int infer_schema_select(const tf_ir_node *node,
|
|
|
82
175
|
bool found = false;
|
|
83
176
|
for (size_t j = 0; j < in->n_cols; j++) {
|
|
84
177
|
if (strcmp(in->col_names[j], name) == 0) {
|
|
85
|
-
out->
|
|
86
|
-
|
|
178
|
+
if (registry_schema_set_col(out, (size_t)i, name, in->col_types[j]) != TF_OK) {
|
|
179
|
+
tf_schema_free(out);
|
|
180
|
+
return TF_ERROR;
|
|
181
|
+
}
|
|
87
182
|
found = true;
|
|
88
183
|
break;
|
|
89
184
|
}
|
|
90
185
|
}
|
|
91
186
|
if (!found) {
|
|
92
187
|
/* Column not in input — still record it, validation can catch it */
|
|
93
|
-
out
|
|
94
|
-
|
|
188
|
+
if (registry_schema_set_col(out, (size_t)i, name, TF_TYPE_NULL) != TF_OK) {
|
|
189
|
+
tf_schema_free(out);
|
|
190
|
+
return TF_ERROR;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
return TF_OK;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
static int infer_schema_relocate(const tf_ir_node *node,
|
|
198
|
+
const tf_schema *in, tf_schema *out) {
|
|
199
|
+
if (!in->known) {
|
|
200
|
+
registry_schema_unknown(out);
|
|
201
|
+
return TF_OK;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
cJSON *cols = cJSON_GetObjectItemCaseSensitive(node->args, "columns");
|
|
205
|
+
if (!cols || !cJSON_IsArray(cols)) return TF_ERROR;
|
|
206
|
+
cJSON *before = cJSON_GetObjectItemCaseSensitive(node->args, "before");
|
|
207
|
+
cJSON *after = cJSON_GetObjectItemCaseSensitive(node->args, "after");
|
|
208
|
+
if (before && after) return TF_ERROR;
|
|
209
|
+
if (before && !cJSON_IsString(before)) return TF_ERROR;
|
|
210
|
+
if (after && !cJSON_IsString(after)) return TF_ERROR;
|
|
211
|
+
|
|
212
|
+
size_t n_in = in->n_cols;
|
|
213
|
+
int *move_idx = NULL;
|
|
214
|
+
size_t n_move = 0;
|
|
215
|
+
char *selector_error = NULL;
|
|
216
|
+
int rc = tf_column_selectors_resolve_json(cols, in->col_names, in->col_types, in->n_cols,
|
|
217
|
+
&move_idx, &n_move, &selector_error);
|
|
218
|
+
if (rc != TF_OK) {
|
|
219
|
+
registry_set_schema_error("relocate", selector_error);
|
|
220
|
+
free(selector_error);
|
|
221
|
+
return TF_ERROR;
|
|
222
|
+
}
|
|
223
|
+
free(selector_error);
|
|
224
|
+
|
|
225
|
+
int *is_moving = calloc(n_in ? n_in : 1, sizeof(int));
|
|
226
|
+
int *order = malloc(n_in ? n_in * sizeof(int) : sizeof(int));
|
|
227
|
+
if (!is_moving || !order) {
|
|
228
|
+
free(move_idx); free(is_moving); free(order);
|
|
229
|
+
return TF_ERROR;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
for (size_t i = 0; i < n_move; i++) {
|
|
233
|
+
int found = move_idx[i];
|
|
234
|
+
if (found < 0 || (size_t)found >= n_in || is_moving[found]) {
|
|
235
|
+
free(move_idx); free(is_moving); free(order);
|
|
236
|
+
return TF_ERROR;
|
|
237
|
+
}
|
|
238
|
+
is_moving[found] = 1;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
const char *anchor = before ? before->valuestring : (after ? after->valuestring : NULL);
|
|
242
|
+
int anchor_idx = -1;
|
|
243
|
+
if (anchor) {
|
|
244
|
+
for (size_t i = 0; i < n_in; i++) {
|
|
245
|
+
if (strcmp(in->col_names[i], anchor) == 0) {
|
|
246
|
+
anchor_idx = (int)i;
|
|
247
|
+
break;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
if (anchor_idx < 0 || is_moving[anchor_idx]) {
|
|
251
|
+
free(move_idx); free(is_moving); free(order);
|
|
252
|
+
return TF_ERROR;
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
size_t n_order = 0;
|
|
257
|
+
if (!anchor) {
|
|
258
|
+
for (size_t i = 0; i < n_move; i++) order[n_order++] = move_idx[i];
|
|
259
|
+
for (size_t i = 0; i < n_in; i++)
|
|
260
|
+
if (!is_moving[i]) order[n_order++] = (int)i;
|
|
261
|
+
} else {
|
|
262
|
+
for (size_t i = 0; i < n_in; i++) {
|
|
263
|
+
if (is_moving[i]) continue;
|
|
264
|
+
if (before && (int)i == anchor_idx) {
|
|
265
|
+
for (size_t j = 0; j < n_move; j++) order[n_order++] = move_idx[j];
|
|
266
|
+
}
|
|
267
|
+
order[n_order++] = (int)i;
|
|
268
|
+
if (after && (int)i == anchor_idx) {
|
|
269
|
+
for (size_t j = 0; j < n_move; j++) order[n_order++] = move_idx[j];
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
if (n_order != n_in) {
|
|
275
|
+
free(move_idx); free(is_moving); free(order);
|
|
276
|
+
return TF_ERROR;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
if (registry_schema_alloc_known(out, n_in) != TF_OK) {
|
|
280
|
+
free(move_idx); free(is_moving); free(order);
|
|
281
|
+
return TF_ERROR;
|
|
282
|
+
}
|
|
283
|
+
for (size_t i = 0; i < n_in; i++) {
|
|
284
|
+
int ci = order[i];
|
|
285
|
+
if (registry_schema_set_col(out, i, in->col_names[ci], in->col_types[ci]) != TF_OK) {
|
|
286
|
+
free(move_idx);
|
|
287
|
+
free(is_moving);
|
|
288
|
+
free(order);
|
|
289
|
+
tf_schema_free(out);
|
|
290
|
+
return TF_ERROR;
|
|
95
291
|
}
|
|
96
292
|
}
|
|
293
|
+
|
|
294
|
+
free(move_idx);
|
|
295
|
+
free(is_moving);
|
|
296
|
+
free(order);
|
|
97
297
|
return TF_OK;
|
|
98
298
|
}
|
|
99
299
|
|
|
@@ -101,14 +301,11 @@ static int infer_schema_select(const tf_ir_node *node,
|
|
|
101
301
|
static int infer_schema_rename(const tf_ir_node *node,
|
|
102
302
|
const tf_schema *in, tf_schema *out) {
|
|
103
303
|
if (!in->known) {
|
|
104
|
-
out
|
|
105
|
-
out->col_names = NULL;
|
|
106
|
-
out->col_types = NULL;
|
|
107
|
-
out->n_cols = 0;
|
|
304
|
+
registry_schema_unknown(out);
|
|
108
305
|
return TF_OK;
|
|
109
306
|
}
|
|
110
307
|
|
|
111
|
-
|
|
308
|
+
if (tf_schema_copy_checked(out, in) != TF_OK) return TF_ERROR;
|
|
112
309
|
|
|
113
310
|
cJSON *mapping = cJSON_GetObjectItemCaseSensitive(node->args, "mapping");
|
|
114
311
|
if (!mapping || !cJSON_IsObject(mapping)) return TF_OK; /* no renames */
|
|
@@ -120,8 +317,10 @@ static int infer_schema_rename(const tf_ir_node *node,
|
|
|
120
317
|
const char *new_name = entry->valuestring;
|
|
121
318
|
for (size_t i = 0; i < out->n_cols; i++) {
|
|
122
319
|
if (strcmp(out->col_names[i], old_name) == 0) {
|
|
123
|
-
|
|
124
|
-
|
|
320
|
+
if (registry_schema_set_col(out, i, new_name, out->col_types[i]) != TF_OK) {
|
|
321
|
+
tf_schema_free(out);
|
|
322
|
+
return TF_ERROR;
|
|
323
|
+
}
|
|
125
324
|
break;
|
|
126
325
|
}
|
|
127
326
|
}
|
|
@@ -133,35 +332,177 @@ static int infer_schema_rename(const tf_ir_node *node,
|
|
|
133
332
|
static int infer_schema_derive(const tf_ir_node *node,
|
|
134
333
|
const tf_schema *in, tf_schema *out) {
|
|
135
334
|
if (!in->known) {
|
|
136
|
-
out
|
|
137
|
-
out->col_names = NULL;
|
|
138
|
-
out->col_types = NULL;
|
|
139
|
-
out->n_cols = 0;
|
|
335
|
+
registry_schema_unknown(out);
|
|
140
336
|
return TF_OK;
|
|
141
337
|
}
|
|
142
338
|
|
|
143
339
|
cJSON *columns = cJSON_GetObjectItemCaseSensitive(node->args, "columns");
|
|
144
340
|
int n_derived = columns ? cJSON_GetArraySize(columns) : 0;
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
out->col_types = calloc(total, sizeof(tf_type));
|
|
149
|
-
out->n_cols = total;
|
|
150
|
-
out->known = true;
|
|
151
|
-
|
|
152
|
-
/* Copy input columns */
|
|
153
|
-
for (size_t i = 0; i < in->n_cols; i++) {
|
|
154
|
-
out->col_names[i] = strdup(in->col_names[i]);
|
|
155
|
-
out->col_types[i] = in->col_types[i];
|
|
341
|
+
if (n_derived < 0 ||
|
|
342
|
+
registry_schema_copy_input_plus(out, in, (size_t)n_derived) != TF_OK) {
|
|
343
|
+
return TF_ERROR;
|
|
156
344
|
}
|
|
157
345
|
|
|
158
346
|
/* Add derived columns (type unknown at compile time) */
|
|
159
347
|
for (int i = 0; i < n_derived; i++) {
|
|
160
348
|
cJSON *item = cJSON_GetArrayItem(columns, i);
|
|
161
349
|
cJSON *name_j = cJSON_GetObjectItemCaseSensitive(item, "name");
|
|
162
|
-
|
|
163
|
-
out
|
|
350
|
+
const char *name = cJSON_IsString(name_j) ? name_j->valuestring : "?";
|
|
351
|
+
if (registry_schema_set_col(out, in->n_cols + (size_t)i, name, TF_TYPE_NULL) != TF_OK) {
|
|
352
|
+
tf_schema_free(out);
|
|
353
|
+
return TF_ERROR;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
return TF_OK;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
static int registry_across_token_matches(const char *p, const char *tok) {
|
|
361
|
+
return strncmp(p, tok, strlen(tok)) == 0;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
static char *registry_across_format_name(const char *tmpl, const char *col, const char *fn) {
|
|
365
|
+
if (!tmpl || !tmpl[0]) tmpl = "{col}_{fn}";
|
|
366
|
+
if (!col) col = "";
|
|
367
|
+
if (!fn) fn = "";
|
|
368
|
+
size_t len = 0;
|
|
369
|
+
for (const char *p = tmpl; *p;) {
|
|
370
|
+
if (registry_across_token_matches(p, "{.col}")) { len += strlen(col); p += 6; }
|
|
371
|
+
else if (registry_across_token_matches(p, "{col}")) { len += strlen(col); p += 5; }
|
|
372
|
+
else if (registry_across_token_matches(p, "{.fn}")) { len += strlen(fn); p += 5; }
|
|
373
|
+
else if (registry_across_token_matches(p, "{fn}")) { len += strlen(fn); p += 4; }
|
|
374
|
+
else { len++; p++; }
|
|
375
|
+
}
|
|
376
|
+
char *out = malloc(len + 1);
|
|
377
|
+
if (!out) return NULL;
|
|
378
|
+
char *w = out;
|
|
379
|
+
for (const char *p = tmpl; *p;) {
|
|
380
|
+
const char *rep = NULL;
|
|
381
|
+
size_t tok_len = 0;
|
|
382
|
+
if (registry_across_token_matches(p, "{.col}")) { rep = col; tok_len = 6; }
|
|
383
|
+
else if (registry_across_token_matches(p, "{col}")) { rep = col; tok_len = 5; }
|
|
384
|
+
else if (registry_across_token_matches(p, "{.fn}")) { rep = fn; tok_len = 5; }
|
|
385
|
+
else if (registry_across_token_matches(p, "{fn}")) { rep = fn; tok_len = 4; }
|
|
386
|
+
if (rep) {
|
|
387
|
+
size_t n = strlen(rep);
|
|
388
|
+
memcpy(w, rep, n);
|
|
389
|
+
w += n;
|
|
390
|
+
p += tok_len;
|
|
391
|
+
} else {
|
|
392
|
+
*w++ = *p++;
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
*w = '\0';
|
|
396
|
+
return out;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
static tf_type registry_across_output_type(const char *fn, tf_type input) {
|
|
400
|
+
if (!fn) return TF_TYPE_NULL;
|
|
401
|
+
if (strcasecmp(fn, "trim") == 0 || strcasecmp(fn, "lower") == 0 ||
|
|
402
|
+
strcasecmp(fn, "tolower") == 0 || strcasecmp(fn, "upper") == 0 ||
|
|
403
|
+
strcasecmp(fn, "toupper") == 0) {
|
|
404
|
+
return input == TF_TYPE_STRING ? TF_TYPE_STRING : TF_TYPE_NULL;
|
|
164
405
|
}
|
|
406
|
+
if (!(input == TF_TYPE_INT64 || input == TF_TYPE_FLOAT64)) return TF_TYPE_NULL;
|
|
407
|
+
if (strcasecmp(fn, "abs") == 0) return input;
|
|
408
|
+
if (strcasecmp(fn, "round") == 0 || strcasecmp(fn, "floor") == 0 ||
|
|
409
|
+
strcasecmp(fn, "ceil") == 0 || strcasecmp(fn, "ceiling") == 0) return TF_TYPE_INT64;
|
|
410
|
+
if (strcasecmp(fn, "sqrt") == 0 || strcasecmp(fn, "log") == 0 ||
|
|
411
|
+
strcasecmp(fn, "exp") == 0) return TF_TYPE_FLOAT64;
|
|
412
|
+
return TF_TYPE_NULL;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
static int registry_across_functions(const tf_ir_node *node, cJSON **out_fns, int *out_n) {
|
|
416
|
+
cJSON *functions = cJSON_GetObjectItemCaseSensitive(node->args, "functions");
|
|
417
|
+
if (!functions) functions = cJSON_GetObjectItemCaseSensitive(node->args, "fns");
|
|
418
|
+
cJSON *fn = cJSON_GetObjectItemCaseSensitive(node->args, "fn");
|
|
419
|
+
if (cJSON_IsArray(functions)) {
|
|
420
|
+
int n = cJSON_GetArraySize(functions);
|
|
421
|
+
if (n <= 0) return TF_ERROR;
|
|
422
|
+
*out_fns = functions;
|
|
423
|
+
*out_n = n;
|
|
424
|
+
return TF_OK;
|
|
425
|
+
}
|
|
426
|
+
if (cJSON_IsString(functions) || cJSON_IsString(fn)) {
|
|
427
|
+
*out_fns = cJSON_IsString(functions) ? functions : fn;
|
|
428
|
+
*out_n = 1;
|
|
429
|
+
return TF_OK;
|
|
430
|
+
}
|
|
431
|
+
return TF_ERROR;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
/* across: input schema with selected columns replaced, or input + generated columns */
|
|
435
|
+
static int infer_schema_across(const tf_ir_node *node,
|
|
436
|
+
const tf_schema *in, tf_schema *out) {
|
|
437
|
+
if (!in->known) {
|
|
438
|
+
registry_schema_unknown(out);
|
|
439
|
+
return TF_OK;
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
cJSON *cols = cJSON_GetObjectItemCaseSensitive(node->args, "columns");
|
|
443
|
+
if (!cols || !cJSON_IsArray(cols)) return TF_ERROR;
|
|
444
|
+
int *indices = NULL;
|
|
445
|
+
size_t n_indices = 0;
|
|
446
|
+
char *error = NULL;
|
|
447
|
+
int rc = tf_column_selectors_resolve_json(cols, in->col_names, in->col_types, in->n_cols,
|
|
448
|
+
&indices, &n_indices, &error);
|
|
449
|
+
if (rc != TF_OK) {
|
|
450
|
+
registry_set_schema_error("across", error);
|
|
451
|
+
free(error);
|
|
452
|
+
return TF_ERROR;
|
|
453
|
+
}
|
|
454
|
+
free(error);
|
|
455
|
+
|
|
456
|
+
cJSON *fns = NULL;
|
|
457
|
+
int n_fns = 0;
|
|
458
|
+
if (registry_across_functions(node, &fns, &n_fns) != TF_OK) { free(indices); return TF_ERROR; }
|
|
459
|
+
cJSON *names = cJSON_GetObjectItemCaseSensitive(node->args, "names");
|
|
460
|
+
cJSON *replace_j = cJSON_GetObjectItemCaseSensitive(node->args, "replace");
|
|
461
|
+
int replace = cJSON_IsBool(replace_j) ? cJSON_IsTrue(replace_j) : (n_fns == 1 && !cJSON_IsString(names));
|
|
462
|
+
if (replace && n_fns != 1) { free(indices); return TF_ERROR; }
|
|
463
|
+
const char *tmpl = cJSON_IsString(names) ? names->valuestring : (replace ? "{col}" : "{col}_{fn}");
|
|
464
|
+
|
|
465
|
+
size_t extra = 0;
|
|
466
|
+
size_t total = 0;
|
|
467
|
+
if (!replace && tf_size_mul(n_indices, (size_t)n_fns, &extra) != TF_OK) {
|
|
468
|
+
free(indices);
|
|
469
|
+
return TF_ERROR;
|
|
470
|
+
}
|
|
471
|
+
if (tf_size_add(in->n_cols, extra, &total) != TF_OK ||
|
|
472
|
+
registry_schema_alloc_known(out, total) != TF_OK ||
|
|
473
|
+
registry_schema_copy_input_columns(out, in) != TF_OK) {
|
|
474
|
+
free(indices);
|
|
475
|
+
return TF_ERROR;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
if (replace) {
|
|
479
|
+
const char *fn_name = cJSON_IsArray(fns) ? cJSON_GetArrayItem(fns, 0)->valuestring : fns->valuestring;
|
|
480
|
+
for (size_t i = 0; i < n_indices; i++) {
|
|
481
|
+
size_t ci = (size_t)indices[i];
|
|
482
|
+
tf_type t = registry_across_output_type(fn_name, in->col_types[ci]);
|
|
483
|
+
if (t == TF_TYPE_NULL) { free(indices); tf_schema_free(out); return TF_ERROR; }
|
|
484
|
+
out->col_types[ci] = t;
|
|
485
|
+
}
|
|
486
|
+
} else {
|
|
487
|
+
size_t a = 0;
|
|
488
|
+
for (size_t i = 0; i < n_indices; i++) {
|
|
489
|
+
size_t ci = (size_t)indices[i];
|
|
490
|
+
for (int f = 0; f < n_fns; f++) {
|
|
491
|
+
cJSON *fn_j = cJSON_IsArray(fns) ? cJSON_GetArrayItem(fns, f) : fns;
|
|
492
|
+
if (!cJSON_IsString(fn_j)) { free(indices); tf_schema_free(out); return TF_ERROR; }
|
|
493
|
+
tf_type t = registry_across_output_type(fn_j->valuestring, in->col_types[ci]);
|
|
494
|
+
if (t == TF_TYPE_NULL) { free(indices); tf_schema_free(out); return TF_ERROR; }
|
|
495
|
+
char *name = registry_across_format_name(tmpl, in->col_names[ci], fn_j->valuestring);
|
|
496
|
+
if (registry_schema_take_col(out, in->n_cols + a, name, t) != TF_OK) {
|
|
497
|
+
free(indices);
|
|
498
|
+
tf_schema_free(out);
|
|
499
|
+
return TF_ERROR;
|
|
500
|
+
}
|
|
501
|
+
a++;
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
free(indices);
|
|
165
506
|
return TF_OK;
|
|
166
507
|
}
|
|
167
508
|
|
|
@@ -169,17 +510,80 @@ static int infer_schema_derive(const tf_ir_node *node,
|
|
|
169
510
|
static int infer_schema_validate(const tf_ir_node *node,
|
|
170
511
|
const tf_schema *in, tf_schema *out) {
|
|
171
512
|
(void)node;
|
|
172
|
-
if (!in->known) { out
|
|
173
|
-
out
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
513
|
+
if (!in->known) { registry_schema_unknown(out); return TF_OK; }
|
|
514
|
+
if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
|
|
515
|
+
if (registry_schema_set_col(out, in->n_cols, "_valid", TF_TYPE_BOOL) != TF_OK) {
|
|
516
|
+
tf_schema_free(out);
|
|
517
|
+
return TF_ERROR;
|
|
518
|
+
}
|
|
519
|
+
return TF_OK;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
static int infer_schema_assert(const tf_ir_node *node,
|
|
523
|
+
const tf_schema *in, tf_schema *out) {
|
|
524
|
+
cJSON *action_j = cJSON_GetObjectItemCaseSensitive(node->args, "action");
|
|
525
|
+
const char *action = cJSON_IsString(action_j) ? action_j->valuestring : "fail";
|
|
526
|
+
if (strcmp(action, "annotate") != 0) return infer_schema_passthrough(node, in, out);
|
|
527
|
+
if (!in->known) { registry_schema_unknown(out); return TF_OK; }
|
|
528
|
+
cJSON *result_j = cJSON_GetObjectItemCaseSensitive(node->args, "result");
|
|
529
|
+
const char *result = cJSON_IsString(result_j) && result_j->valuestring[0]
|
|
530
|
+
? result_j->valuestring : "_assert";
|
|
531
|
+
if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
|
|
532
|
+
if (registry_schema_set_col(out, in->n_cols, result, TF_TYPE_BOOL) != TF_OK) {
|
|
533
|
+
tf_schema_free(out);
|
|
534
|
+
return TF_ERROR;
|
|
535
|
+
}
|
|
536
|
+
return TF_OK;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
static int infer_schema_schema(const tf_ir_node *node,
|
|
540
|
+
const tf_schema *in, tf_schema *out) {
|
|
541
|
+
cJSON *action_j = cJSON_GetObjectItemCaseSensitive(node->args, "action");
|
|
542
|
+
cJSON *mode_j = cJSON_GetObjectItemCaseSensitive(node->args, "mode");
|
|
543
|
+
const char *action = cJSON_IsString(action_j) ? action_j->valuestring :
|
|
544
|
+
(cJSON_IsString(mode_j) ? mode_j->valuestring : "fail");
|
|
545
|
+
if (strcmp(action, "annotate") != 0) return infer_schema_passthrough(node, in, out);
|
|
546
|
+
if (!in->known) { registry_schema_unknown(out); return TF_OK; }
|
|
547
|
+
cJSON *result_j = cJSON_GetObjectItemCaseSensitive(node->args, "result");
|
|
548
|
+
const char *result = cJSON_IsString(result_j) && result_j->valuestring[0]
|
|
549
|
+
? result_j->valuestring : "_schema";
|
|
550
|
+
if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
|
|
551
|
+
if (registry_schema_set_col(out, in->n_cols, result, TF_TYPE_BOOL) != TF_OK) {
|
|
552
|
+
tf_schema_free(out);
|
|
553
|
+
return TF_ERROR;
|
|
554
|
+
}
|
|
555
|
+
return TF_OK;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
static int infer_schema_schema_infer(const tf_ir_node *node,
|
|
559
|
+
const tf_schema *in, tf_schema *out) {
|
|
560
|
+
(void)node; (void)in;
|
|
561
|
+
static const char *names[] = {
|
|
562
|
+
"column", "type", "nullable", "non_null", "rows_seen",
|
|
563
|
+
"rows_sampled", "missing", "non_missing", "observed_types", "warning"
|
|
564
|
+
};
|
|
565
|
+
static const tf_type types[] = {
|
|
566
|
+
TF_TYPE_STRING, TF_TYPE_STRING, TF_TYPE_BOOL, TF_TYPE_BOOL, TF_TYPE_INT64,
|
|
567
|
+
TF_TYPE_INT64, TF_TYPE_INT64, TF_TYPE_INT64, TF_TYPE_STRING, TF_TYPE_STRING
|
|
568
|
+
};
|
|
569
|
+
size_t n = sizeof(names) / sizeof(names[0]);
|
|
570
|
+
if (registry_schema_alloc_known(out, n) != TF_OK) return TF_ERROR;
|
|
571
|
+
for (size_t i = 0; i < n; i++) {
|
|
572
|
+
if (registry_schema_set_col(out, i, names[i], types[i]) != TF_OK) {
|
|
573
|
+
tf_schema_free(out);
|
|
574
|
+
return TF_ERROR;
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
return TF_OK;
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
static int infer_schema_add_int64_column(const tf_schema *in, tf_schema *out, const char *name) {
|
|
581
|
+
if (!in->known) { registry_schema_unknown(out); return TF_OK; }
|
|
582
|
+
if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
|
|
583
|
+
if (registry_schema_set_col(out, in->n_cols, name, TF_TYPE_INT64) != TF_OK) {
|
|
584
|
+
tf_schema_free(out);
|
|
585
|
+
return TF_ERROR;
|
|
180
586
|
}
|
|
181
|
-
out->col_names[in->n_cols] = strdup("_valid");
|
|
182
|
-
out->col_types[in->n_cols] = TF_TYPE_BOOL;
|
|
183
587
|
return TF_OK;
|
|
184
588
|
}
|
|
185
589
|
|
|
@@ -187,17 +591,42 @@ static int infer_schema_validate(const tf_ir_node *node,
|
|
|
187
591
|
static int infer_schema_add_hash(const tf_ir_node *node,
|
|
188
592
|
const tf_schema *in, tf_schema *out) {
|
|
189
593
|
(void)node;
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
594
|
+
return infer_schema_add_int64_column(in, out, "_hash");
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
static int infer_schema_rleid(const tf_ir_node *node,
|
|
598
|
+
const tf_schema *in, tf_schema *out) {
|
|
599
|
+
const char *name = "_rleid";
|
|
600
|
+
cJSON *res = cJSON_GetObjectItemCaseSensitive(node->args, "result");
|
|
601
|
+
if (cJSON_IsString(res) && res->valuestring[0] != '\0') name = res->valuestring;
|
|
602
|
+
return infer_schema_add_int64_column(in, out, name);
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
static int infer_schema_rowid(const tf_ir_node *node,
|
|
606
|
+
const tf_schema *in, tf_schema *out) {
|
|
607
|
+
const char *name = "_rowid";
|
|
608
|
+
cJSON *res = cJSON_GetObjectItemCaseSensitive(node->args, "result");
|
|
609
|
+
if (cJSON_IsString(res) && res->valuestring[0] != '\0') name = res->valuestring;
|
|
610
|
+
return infer_schema_add_int64_column(in, out, name);
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
static int infer_schema_source_name(const tf_ir_node *node,
|
|
614
|
+
const tf_schema *in, tf_schema *out) {
|
|
615
|
+
if (!in->known) {
|
|
616
|
+
registry_schema_unknown(out);
|
|
617
|
+
return TF_OK;
|
|
618
|
+
}
|
|
619
|
+
const char *name = "_source";
|
|
620
|
+
cJSON *res = cJSON_GetObjectItemCaseSensitive(node->args, "result");
|
|
621
|
+
if (!cJSON_IsString(res)) res = cJSON_GetObjectItemCaseSensitive(node->args, "column");
|
|
622
|
+
if (!cJSON_IsString(res)) res = cJSON_GetObjectItemCaseSensitive(node->args, "as");
|
|
623
|
+
if (cJSON_IsString(res) && res->valuestring[0] != '\0') name = res->valuestring;
|
|
624
|
+
|
|
625
|
+
if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
|
|
626
|
+
if (registry_schema_set_col(out, in->n_cols, name, TF_TYPE_STRING) != TF_OK) {
|
|
627
|
+
tf_schema_free(out);
|
|
628
|
+
return TF_ERROR;
|
|
198
629
|
}
|
|
199
|
-
out->col_names[in->n_cols] = strdup("_hash");
|
|
200
|
-
out->col_types[in->n_cols] = TF_TYPE_INT64;
|
|
201
630
|
return TF_OK;
|
|
202
631
|
}
|
|
203
632
|
|
|
@@ -205,74 +634,382 @@ static int infer_schema_add_hash(const tf_ir_node *node,
|
|
|
205
634
|
static int infer_schema_frequency(const tf_ir_node *node,
|
|
206
635
|
const tf_schema *in, tf_schema *out) {
|
|
207
636
|
(void)node; (void)in;
|
|
208
|
-
out
|
|
209
|
-
out
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
out->col_names[1] = strdup("count");
|
|
215
|
-
out->col_types[1] = TF_TYPE_INT64;
|
|
637
|
+
if (registry_schema_alloc_known(out, 2) != TF_OK) return TF_ERROR;
|
|
638
|
+
if (registry_schema_set_col(out, 0, "value", TF_TYPE_STRING) != TF_OK ||
|
|
639
|
+
registry_schema_set_col(out, 1, "count", TF_TYPE_INT64) != TF_OK) {
|
|
640
|
+
tf_schema_free(out);
|
|
641
|
+
return TF_ERROR;
|
|
642
|
+
}
|
|
216
643
|
return TF_OK;
|
|
217
644
|
}
|
|
218
645
|
|
|
219
|
-
|
|
646
|
+
static int schema_col_index(const tf_schema *schema, const char *name);
|
|
647
|
+
|
|
648
|
+
static int infer_schema_pivot(const tf_ir_node *node,
|
|
649
|
+
const tf_schema *in, tf_schema *out) {
|
|
650
|
+
if (!node || !node->args || !in || !in->known) {
|
|
651
|
+
registry_schema_unknown(out);
|
|
652
|
+
return TF_OK;
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
cJSON *cats = cJSON_GetObjectItemCaseSensitive(node->args, "categories");
|
|
656
|
+
if (!cats) {
|
|
657
|
+
registry_schema_unknown(out);
|
|
658
|
+
return TF_OK;
|
|
659
|
+
}
|
|
660
|
+
if (!cJSON_IsArray(cats)) {
|
|
661
|
+
registry_set_schema_error("pivot", "categories must be an array");
|
|
662
|
+
return TF_ERROR;
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
cJSON *name_j = cJSON_GetObjectItemCaseSensitive(node->args, "name_column");
|
|
666
|
+
cJSON *value_j = cJSON_GetObjectItemCaseSensitive(node->args, "value_column");
|
|
667
|
+
const char *name_col = cJSON_IsString(name_j) ? name_j->valuestring : NULL;
|
|
668
|
+
const char *value_col = cJSON_IsString(value_j) ? value_j->valuestring : NULL;
|
|
669
|
+
if (!name_col || !name_col[0] || !value_col || !value_col[0]) {
|
|
670
|
+
registry_set_schema_error("pivot", "missing name_column or value_column");
|
|
671
|
+
return TF_ERROR;
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
int name_idx = schema_col_index(in, name_col);
|
|
675
|
+
int value_idx = schema_col_index(in, value_col);
|
|
676
|
+
if (name_idx < 0 || value_idx < 0) {
|
|
677
|
+
registry_set_schema_error("pivot", "name_column or value_column not found");
|
|
678
|
+
return TF_ERROR;
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
int n_cats = cJSON_GetArraySize(cats);
|
|
682
|
+
if (n_cats <= 0) {
|
|
683
|
+
registry_set_schema_error("pivot", "categories cannot be empty");
|
|
684
|
+
return TF_ERROR;
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
size_t unique_cats = 0;
|
|
688
|
+
for (int i = 0; i < n_cats; i++) {
|
|
689
|
+
cJSON *cat = cJSON_GetArrayItem(cats, i);
|
|
690
|
+
if (!cJSON_IsString(cat) || !cat->valuestring || !cat->valuestring[0]) {
|
|
691
|
+
registry_set_schema_error("pivot", "categories must be non-empty strings");
|
|
692
|
+
return TF_ERROR;
|
|
693
|
+
}
|
|
694
|
+
int seen = 0;
|
|
695
|
+
for (int j = 0; j < i; j++) {
|
|
696
|
+
cJSON *prev = cJSON_GetArrayItem(cats, j);
|
|
697
|
+
if (cJSON_IsString(prev) && prev->valuestring &&
|
|
698
|
+
strcmp(prev->valuestring, cat->valuestring) == 0) {
|
|
699
|
+
seen = 1;
|
|
700
|
+
break;
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
if (!seen) {
|
|
704
|
+
size_t next_unique = 0;
|
|
705
|
+
if (tf_size_add(unique_cats, 1, &next_unique) != TF_OK) return TF_ERROR;
|
|
706
|
+
unique_cats = next_unique;
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
size_t n_pass = 0;
|
|
711
|
+
for (size_t c = 0; c < in->n_cols; c++) {
|
|
712
|
+
if ((int)c != name_idx && (int)c != value_idx) n_pass++;
|
|
713
|
+
}
|
|
714
|
+
size_t total = 0;
|
|
715
|
+
if (tf_size_add(n_pass, unique_cats, &total) != TF_OK ||
|
|
716
|
+
registry_schema_alloc_known(out, total) != TF_OK) {
|
|
717
|
+
return TF_ERROR;
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
size_t out_idx = 0;
|
|
721
|
+
for (size_t c = 0; c < in->n_cols; c++) {
|
|
722
|
+
if ((int)c == name_idx || (int)c == value_idx) continue;
|
|
723
|
+
if (registry_schema_set_col(out, out_idx++, in->col_names[c] ? in->col_names[c] : "",
|
|
724
|
+
in->col_types[c]) != TF_OK) {
|
|
725
|
+
tf_schema_free(out);
|
|
726
|
+
return TF_ERROR;
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
cJSON *agg_j = cJSON_GetObjectItemCaseSensitive(node->args, "agg");
|
|
731
|
+
const char *agg = cJSON_IsString(agg_j) ? agg_j->valuestring : NULL;
|
|
732
|
+
tf_type pivot_type = (agg && strcmp(agg, "count") == 0) ? TF_TYPE_INT64 : TF_TYPE_FLOAT64;
|
|
733
|
+
for (int i = 0; i < n_cats; i++) {
|
|
734
|
+
cJSON *cat = cJSON_GetArrayItem(cats, i);
|
|
735
|
+
int seen = 0;
|
|
736
|
+
for (int j = 0; j < i; j++) {
|
|
737
|
+
cJSON *prev = cJSON_GetArrayItem(cats, j);
|
|
738
|
+
if (cJSON_IsString(prev) && prev->valuestring &&
|
|
739
|
+
strcmp(prev->valuestring, cat->valuestring) == 0) {
|
|
740
|
+
seen = 1;
|
|
741
|
+
break;
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
if (seen) continue;
|
|
745
|
+
if (registry_schema_set_col(out, out_idx++, cat->valuestring, pivot_type) != TF_OK) {
|
|
746
|
+
tf_schema_free(out);
|
|
747
|
+
return TF_ERROR;
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
return TF_OK;
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
static int schema_col_index(const tf_schema *schema, const char *name) {
|
|
754
|
+
if (!schema || !schema->known || !name) return -1;
|
|
755
|
+
for (size_t i = 0; i < schema->n_cols; i++) {
|
|
756
|
+
if (schema->col_names[i] && strcmp(schema->col_names[i], name) == 0) return (int)i;
|
|
757
|
+
}
|
|
758
|
+
return -1;
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
static int infer_schema_append_source_column(const tf_ir_node *node,
|
|
763
|
+
const tf_schema *in,
|
|
764
|
+
tf_schema *out,
|
|
765
|
+
const char *default_suffix) {
|
|
766
|
+
cJSON *col_j = cJSON_GetObjectItemCaseSensitive(node->args, "column");
|
|
767
|
+
const char *column = cJSON_IsString(col_j) ? col_j->valuestring : NULL;
|
|
768
|
+
if (!in->known || !column) {
|
|
769
|
+
registry_schema_unknown(out);
|
|
770
|
+
return TF_OK;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
int ci = schema_col_index(in, column);
|
|
774
|
+
if (ci < 0) {
|
|
775
|
+
registry_schema_unknown(out);
|
|
776
|
+
return TF_OK;
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
const char *name = NULL;
|
|
780
|
+
cJSON *res = cJSON_GetObjectItemCaseSensitive(node->args, "result");
|
|
781
|
+
if (cJSON_IsString(res) && res->valuestring[0] != '\0') name = res->valuestring;
|
|
782
|
+
|
|
783
|
+
char fallback[256];
|
|
784
|
+
if (!name) {
|
|
785
|
+
snprintf(fallback, sizeof(fallback), "%s_%s", column, default_suffix);
|
|
786
|
+
name = fallback;
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
|
|
790
|
+
if (registry_schema_set_col(out, in->n_cols, name, in->col_types[ci]) != TF_OK) {
|
|
791
|
+
tf_schema_free(out);
|
|
792
|
+
return TF_ERROR;
|
|
793
|
+
}
|
|
794
|
+
return TF_OK;
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
static int infer_schema_lead(const tf_ir_node *node,
|
|
798
|
+
const tf_schema *in, tf_schema *out) {
|
|
799
|
+
return infer_schema_append_source_column(node, in, out, "lead");
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
static int infer_schema_lag(const tf_ir_node *node,
|
|
803
|
+
const tf_schema *in, tf_schema *out) {
|
|
804
|
+
return infer_schema_append_source_column(node, in, out, "lag");
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
static int infer_schema_shift(const tf_ir_node *node,
|
|
808
|
+
const tf_schema *in, tf_schema *out) {
|
|
809
|
+
const char *suffix = "shift";
|
|
810
|
+
cJSON *type = cJSON_GetObjectItemCaseSensitive(node->args, "type");
|
|
811
|
+
if (cJSON_IsString(type) && strcmp(type->valuestring, "lead") == 0) suffix = "lead";
|
|
812
|
+
return infer_schema_append_source_column(node, in, out, suffix);
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
static tf_type json_extract_type_from_arg(const cJSON *args) {
|
|
816
|
+
cJSON *type = cJSON_GetObjectItemCaseSensitive(args, "type");
|
|
817
|
+
const char *s = cJSON_IsString(type) ? type->valuestring : "string";
|
|
818
|
+
if (strcmp(s, "int") == 0 || strcmp(s, "int64") == 0) return TF_TYPE_INT64;
|
|
819
|
+
if (strcmp(s, "float") == 0 || strcmp(s, "float64") == 0 || strcmp(s, "number") == 0) return TF_TYPE_FLOAT64;
|
|
820
|
+
if (strcmp(s, "bool") == 0 || strcmp(s, "boolean") == 0) return TF_TYPE_BOOL;
|
|
821
|
+
return TF_TYPE_STRING;
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
static int infer_schema_json_extract(const tf_ir_node *node,
|
|
825
|
+
const tf_schema *in, tf_schema *out) {
|
|
826
|
+
if (!in->known) {
|
|
827
|
+
registry_schema_unknown(out);
|
|
828
|
+
return TF_OK;
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
cJSON *res = cJSON_GetObjectItemCaseSensitive(node->args, "result");
|
|
832
|
+
if (!cJSON_IsString(res) || res->valuestring[0] == '\0') return TF_ERROR;
|
|
833
|
+
|
|
834
|
+
if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
|
|
835
|
+
if (registry_schema_set_col(out, in->n_cols, res->valuestring,
|
|
836
|
+
json_extract_type_from_arg(node->args)) != TF_OK) {
|
|
837
|
+
tf_schema_free(out);
|
|
838
|
+
return TF_ERROR;
|
|
839
|
+
}
|
|
840
|
+
return TF_OK;
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
static int infer_schema_json_schema(const tf_ir_node *node,
|
|
844
|
+
const tf_schema *in, tf_schema *out) {
|
|
845
|
+
cJSON *mode_j = cJSON_GetObjectItemCaseSensitive(node->args, "mode");
|
|
846
|
+
const char *mode = cJSON_IsString(mode_j) ? mode_j->valuestring : "annotate";
|
|
847
|
+
if (strcmp(mode, "filter") == 0 || strcmp(mode, "keep") == 0) {
|
|
848
|
+
return infer_schema_passthrough(node, in, out);
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
if (!in->known) {
|
|
852
|
+
registry_schema_unknown(out);
|
|
853
|
+
return TF_OK;
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
const char *result = "_valid";
|
|
857
|
+
cJSON *res = cJSON_GetObjectItemCaseSensitive(node->args, "result");
|
|
858
|
+
if (cJSON_IsString(res) && res->valuestring[0] != '\0') result = res->valuestring;
|
|
859
|
+
|
|
860
|
+
if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
|
|
861
|
+
if (registry_schema_set_col(out, in->n_cols, result, TF_TYPE_BOOL) != TF_OK) {
|
|
862
|
+
tf_schema_free(out);
|
|
863
|
+
return TF_ERROR;
|
|
864
|
+
}
|
|
865
|
+
return TF_OK;
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
static tf_type json_flatten_type_from_string(const char *type_s) {
|
|
869
|
+
if (!type_s || strcmp(type_s, "string") == 0 || strcmp(type_s, "str") == 0 || strcmp(type_s, "json") == 0) return TF_TYPE_STRING;
|
|
870
|
+
if (strcmp(type_s, "int") == 0 || strcmp(type_s, "int64") == 0) return TF_TYPE_INT64;
|
|
871
|
+
if (strcmp(type_s, "float") == 0 || strcmp(type_s, "float64") == 0 || strcmp(type_s, "number") == 0) return TF_TYPE_FLOAT64;
|
|
872
|
+
if (strcmp(type_s, "bool") == 0 || strcmp(type_s, "boolean") == 0) return TF_TYPE_BOOL;
|
|
873
|
+
return TF_TYPE_NULL;
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
static int json_flatten_parse_string_field(const char *spec, char **name, tf_type *type) {
|
|
877
|
+
if (!spec) return TF_ERROR;
|
|
878
|
+
char *copy = strdup(spec);
|
|
879
|
+
if (!copy) return TF_ERROR;
|
|
880
|
+
char *first = strchr(copy, ':');
|
|
881
|
+
if (!first) { free(copy); return TF_ERROR; }
|
|
882
|
+
*first = '\0';
|
|
883
|
+
char *second = strchr(first + 1, ':');
|
|
884
|
+
if (second) *second = '\0';
|
|
885
|
+
tf_type parsed = json_flatten_type_from_string(second ? second + 1 : "string");
|
|
886
|
+
if (first[1] == '\0' || parsed == TF_TYPE_NULL) {
|
|
887
|
+
free(copy);
|
|
888
|
+
return TF_ERROR;
|
|
889
|
+
}
|
|
890
|
+
*name = strdup(first + 1);
|
|
891
|
+
*type = parsed;
|
|
892
|
+
free(copy);
|
|
893
|
+
return *name ? TF_OK : TF_ERROR;
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
static int json_flatten_field_schema(const cJSON *item, char **name, tf_type *type) {
|
|
897
|
+
*name = NULL;
|
|
898
|
+
*type = TF_TYPE_STRING;
|
|
899
|
+
if (cJSON_IsString(item)) return json_flatten_parse_string_field(item->valuestring, name, type);
|
|
900
|
+
if (!cJSON_IsObject(item)) return TF_ERROR;
|
|
901
|
+
cJSON *name_j = cJSON_GetObjectItemCaseSensitive(item, "name");
|
|
902
|
+
if (!cJSON_IsString(name_j)) name_j = cJSON_GetObjectItemCaseSensitive(item, "result");
|
|
903
|
+
cJSON *type_j = cJSON_GetObjectItemCaseSensitive(item, "type");
|
|
904
|
+
tf_type parsed = json_flatten_type_from_string(cJSON_IsString(type_j) ? type_j->valuestring : "string");
|
|
905
|
+
if (!cJSON_IsString(name_j) || name_j->valuestring[0] == '\0' || parsed == TF_TYPE_NULL) return TF_ERROR;
|
|
906
|
+
*name = strdup(name_j->valuestring);
|
|
907
|
+
*type = parsed;
|
|
908
|
+
return *name ? TF_OK : TF_ERROR;
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
static int infer_schema_json_flatten(const tf_ir_node *node,
|
|
912
|
+
const tf_schema *in, tf_schema *out) {
|
|
913
|
+
if (!in->known) {
|
|
914
|
+
registry_schema_unknown(out);
|
|
915
|
+
return TF_OK;
|
|
916
|
+
}
|
|
917
|
+
cJSON *fields = cJSON_GetObjectItemCaseSensitive(node->args, "fields");
|
|
918
|
+
if (!cJSON_IsArray(fields) || cJSON_GetArraySize(fields) <= 0) return TF_ERROR;
|
|
919
|
+
int n_fields = cJSON_GetArraySize(fields);
|
|
920
|
+
if (registry_schema_copy_input_plus(out, in, (size_t)n_fields) != TF_OK) return TF_ERROR;
|
|
921
|
+
for (int i = 0; i < n_fields; i++) {
|
|
922
|
+
char *name = NULL;
|
|
923
|
+
tf_type type = TF_TYPE_STRING;
|
|
924
|
+
if (json_flatten_field_schema(cJSON_GetArrayItem(fields, i), &name, &type) != TF_OK) {
|
|
925
|
+
free(name);
|
|
926
|
+
tf_schema_free(out);
|
|
927
|
+
return TF_ERROR;
|
|
928
|
+
}
|
|
929
|
+
if (registry_schema_take_col(out, in->n_cols + (size_t)i, name, type) != TF_OK) {
|
|
930
|
+
tf_schema_free(out);
|
|
931
|
+
return TF_ERROR;
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
return TF_OK;
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
/* group-agg: typed group columns + aggregate columns (float64) */
|
|
220
939
|
static int infer_schema_group_agg(const tf_ir_node *node,
|
|
221
940
|
const tf_schema *in, tf_schema *out) {
|
|
222
|
-
(void)in;
|
|
223
941
|
cJSON *group_by = cJSON_GetObjectItemCaseSensitive(node->args, "group_by");
|
|
224
942
|
cJSON *aggs = cJSON_GetObjectItemCaseSensitive(node->args, "aggs");
|
|
225
943
|
int ng = group_by ? cJSON_GetArraySize(group_by) : 0;
|
|
226
944
|
int na = aggs ? cJSON_GetArraySize(aggs) : 0;
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
945
|
+
if (ng < 0 || na < 0) return TF_ERROR;
|
|
946
|
+
size_t total = 0;
|
|
947
|
+
if (tf_size_add((size_t)ng, (size_t)na, &total) != TF_OK ||
|
|
948
|
+
registry_schema_alloc_known(out, total) != TF_OK) return TF_ERROR;
|
|
231
949
|
for (int i = 0; i < ng; i++) {
|
|
232
950
|
cJSON *item = cJSON_GetArrayItem(group_by, i);
|
|
233
|
-
|
|
234
|
-
|
|
951
|
+
const char *name = cJSON_IsString(item) ? item->valuestring : "?";
|
|
952
|
+
int input_idx = schema_col_index(in, name);
|
|
953
|
+
tf_type type = input_idx >= 0 ? in->col_types[input_idx] : TF_TYPE_STRING;
|
|
954
|
+
if (registry_schema_set_col(out, (size_t)i, name, type) != TF_OK) {
|
|
955
|
+
tf_schema_free(out);
|
|
956
|
+
return TF_ERROR;
|
|
957
|
+
}
|
|
235
958
|
}
|
|
236
959
|
for (int i = 0; i < na; i++) {
|
|
237
960
|
cJSON *item = cJSON_GetArrayItem(aggs, i);
|
|
238
961
|
cJSON *name_j = cJSON_GetObjectItemCaseSensitive(item, "name");
|
|
239
|
-
|
|
240
|
-
out
|
|
962
|
+
const char *name = name_j && cJSON_IsString(name_j) ? name_j->valuestring : "?";
|
|
963
|
+
if (registry_schema_set_col(out, (size_t)ng + (size_t)i, name, TF_TYPE_FLOAT64) != TF_OK) {
|
|
964
|
+
tf_schema_free(out);
|
|
965
|
+
return TF_ERROR;
|
|
966
|
+
}
|
|
241
967
|
}
|
|
242
968
|
return TF_OK;
|
|
243
969
|
}
|
|
244
970
|
|
|
245
|
-
/* stats: output schema is known
|
|
971
|
+
/* stats/scan: output schema is known from requested or profile-default stats */
|
|
246
972
|
static int infer_schema_stats(const tf_ir_node *node,
|
|
247
973
|
const tf_schema *in, tf_schema *out) {
|
|
248
974
|
(void)in;
|
|
249
|
-
|
|
250
|
-
int
|
|
251
|
-
int w_var=0, w_stddev
|
|
252
|
-
int
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
975
|
+
int w_count = 0, w_missing = 0, w_complete_rate = 0;
|
|
976
|
+
int w_sum = 0, w_avg = 0, w_min = 0, w_max = 0;
|
|
977
|
+
int w_var = 0, w_stddev = 0;
|
|
978
|
+
int w_median = 0, w_p25 = 0, w_p75 = 0;
|
|
979
|
+
int w_skewness = 0, w_kurtosis = 0;
|
|
980
|
+
int w_distinct = 0, w_hist = 0, w_sample = 0;
|
|
981
|
+
|
|
982
|
+
cJSON *arr = node ? cJSON_GetObjectItemCaseSensitive(node->args, "stats") : NULL;
|
|
983
|
+
if (arr && cJSON_IsArray(arr)) {
|
|
984
|
+
int n = cJSON_GetArraySize(arr);
|
|
256
985
|
for (int i = 0; i < n; i++) {
|
|
257
|
-
cJSON *item = cJSON_GetArrayItem(
|
|
986
|
+
cJSON *item = cJSON_GetArrayItem(arr, i);
|
|
258
987
|
if (!cJSON_IsString(item)) continue;
|
|
259
988
|
const char *s = item->valuestring;
|
|
260
989
|
if (strcmp(s, "count") == 0) w_count = 1;
|
|
990
|
+
else if (strcmp(s, "missing") == 0) w_missing = 1;
|
|
991
|
+
else if (strcmp(s, "complete_rate") == 0) w_complete_rate = 1;
|
|
261
992
|
else if (strcmp(s, "sum") == 0) w_sum = 1;
|
|
262
|
-
else if (strcmp(s, "avg") == 0) w_avg = 1;
|
|
993
|
+
else if (strcmp(s, "avg") == 0 || strcmp(s, "mean") == 0) w_avg = 1;
|
|
263
994
|
else if (strcmp(s, "min") == 0) w_min = 1;
|
|
264
995
|
else if (strcmp(s, "max") == 0) w_max = 1;
|
|
265
|
-
else if (strcmp(s, "var") == 0) w_var = 1;
|
|
266
|
-
else if (strcmp(s, "stddev") == 0) w_stddev = 1;
|
|
996
|
+
else if (strcmp(s, "var") == 0 || strcmp(s, "variance") == 0) w_var = 1;
|
|
997
|
+
else if (strcmp(s, "stddev") == 0 || strcmp(s, "sd") == 0) w_stddev = 1;
|
|
267
998
|
else if (strcmp(s, "median") == 0) w_median = 1;
|
|
268
|
-
else if (strcmp(s, "p25") == 0) w_p25 = 1;
|
|
269
|
-
else if (strcmp(s, "p75") == 0) w_p75 = 1;
|
|
270
|
-
else if (strcmp(s, "skewness") == 0) w_skewness = 1;
|
|
271
|
-
else if (strcmp(s, "kurtosis") == 0) w_kurtosis = 1;
|
|
999
|
+
else if (strcmp(s, "p25") == 0 || strcmp(s, "q1") == 0) w_p25 = 1;
|
|
1000
|
+
else if (strcmp(s, "p75") == 0 || strcmp(s, "q3") == 0) w_p75 = 1;
|
|
1001
|
+
else if (strcmp(s, "skewness") == 0 || strcmp(s, "skew") == 0) w_skewness = 1;
|
|
1002
|
+
else if (strcmp(s, "kurtosis") == 0 || strcmp(s, "kurt") == 0) w_kurtosis = 1;
|
|
272
1003
|
else if (strcmp(s, "distinct") == 0) w_distinct = 1;
|
|
273
1004
|
else if (strcmp(s, "hist") == 0) w_hist = 1;
|
|
274
1005
|
else if (strcmp(s, "sample") == 0) w_sample = 1;
|
|
275
1006
|
}
|
|
1007
|
+
} else if (node && node->op && strcmp(node->op, "scan") == 0) {
|
|
1008
|
+
w_count = w_missing = w_complete_rate = 1;
|
|
1009
|
+
w_distinct = 1;
|
|
1010
|
+
w_min = w_max = w_avg = w_stddev = 1;
|
|
1011
|
+
w_p25 = w_median = w_p75 = 1;
|
|
1012
|
+
w_hist = w_sample = 1;
|
|
276
1013
|
} else {
|
|
277
1014
|
/* Default: basic + variance + median */
|
|
278
1015
|
w_count = w_sum = w_avg = w_min = w_max = 1;
|
|
@@ -281,6 +1018,8 @@ static int infer_schema_stats(const tf_ir_node *node,
|
|
|
281
1018
|
|
|
282
1019
|
size_t n_cols = 1; /* column */
|
|
283
1020
|
if (w_count) n_cols++;
|
|
1021
|
+
if (w_missing) n_cols++;
|
|
1022
|
+
if (w_complete_rate) n_cols++;
|
|
284
1023
|
if (w_sum) n_cols++;
|
|
285
1024
|
if (w_avg) n_cols++;
|
|
286
1025
|
if (w_min) n_cols++;
|
|
@@ -296,28 +1035,37 @@ static int infer_schema_stats(const tf_ir_node *node,
|
|
|
296
1035
|
if (w_hist) n_cols++;
|
|
297
1036
|
if (w_sample) n_cols++;
|
|
298
1037
|
|
|
299
|
-
|
|
300
|
-
out->col_types = calloc(n_cols, sizeof(tf_type));
|
|
301
|
-
out->n_cols = n_cols;
|
|
302
|
-
out->known = true;
|
|
1038
|
+
if (registry_schema_alloc_known(out, n_cols) != TF_OK) return TF_ERROR;
|
|
303
1039
|
|
|
304
1040
|
size_t ci = 0;
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
if (
|
|
315
|
-
if (
|
|
316
|
-
if (
|
|
317
|
-
if (
|
|
318
|
-
if (
|
|
319
|
-
if (
|
|
320
|
-
if (
|
|
1041
|
+
#define ADD_STAT_COLUMN(name, type) do { \
|
|
1042
|
+
if (registry_schema_set_col(out, ci, (name), (type)) != TF_OK) { \
|
|
1043
|
+
tf_schema_free(out); \
|
|
1044
|
+
return TF_ERROR; \
|
|
1045
|
+
} \
|
|
1046
|
+
ci++; \
|
|
1047
|
+
} while (0)
|
|
1048
|
+
|
|
1049
|
+
ADD_STAT_COLUMN("column", TF_TYPE_STRING);
|
|
1050
|
+
if (w_count) ADD_STAT_COLUMN("count", TF_TYPE_INT64);
|
|
1051
|
+
if (w_missing) ADD_STAT_COLUMN("missing", TF_TYPE_INT64);
|
|
1052
|
+
if (w_complete_rate) ADD_STAT_COLUMN("complete_rate", TF_TYPE_FLOAT64);
|
|
1053
|
+
if (w_sum) ADD_STAT_COLUMN("sum", TF_TYPE_FLOAT64);
|
|
1054
|
+
if (w_avg) ADD_STAT_COLUMN("avg", TF_TYPE_FLOAT64);
|
|
1055
|
+
if (w_min) ADD_STAT_COLUMN("min", TF_TYPE_FLOAT64);
|
|
1056
|
+
if (w_max) ADD_STAT_COLUMN("max", TF_TYPE_FLOAT64);
|
|
1057
|
+
if (w_var) ADD_STAT_COLUMN("var", TF_TYPE_FLOAT64);
|
|
1058
|
+
if (w_stddev) ADD_STAT_COLUMN("stddev", TF_TYPE_FLOAT64);
|
|
1059
|
+
if (w_median) ADD_STAT_COLUMN("median", TF_TYPE_FLOAT64);
|
|
1060
|
+
if (w_p25) ADD_STAT_COLUMN("p25", TF_TYPE_FLOAT64);
|
|
1061
|
+
if (w_p75) ADD_STAT_COLUMN("p75", TF_TYPE_FLOAT64);
|
|
1062
|
+
if (w_skewness) ADD_STAT_COLUMN("skewness", TF_TYPE_FLOAT64);
|
|
1063
|
+
if (w_kurtosis) ADD_STAT_COLUMN("kurtosis", TF_TYPE_FLOAT64);
|
|
1064
|
+
if (w_distinct) ADD_STAT_COLUMN("distinct", TF_TYPE_INT64);
|
|
1065
|
+
if (w_hist) ADD_STAT_COLUMN("hist", TF_TYPE_STRING);
|
|
1066
|
+
if (w_sample) ADD_STAT_COLUMN("sample", TF_TYPE_STRING);
|
|
1067
|
+
#undef ADD_STAT_COLUMN
|
|
1068
|
+
|
|
321
1069
|
return TF_OK;
|
|
322
1070
|
}
|
|
323
1071
|
|
|
@@ -328,6 +1076,22 @@ static tf_arg_desc csv_decode_args[] = {
|
|
|
328
1076
|
{"header", "bool", false, "true"},
|
|
329
1077
|
{"batch_size","int", false, "1024"},
|
|
330
1078
|
{"repair", "bool", false, "false"},
|
|
1079
|
+
{"mode", "string", false, "permissive"},
|
|
1080
|
+
{"strict", "bool", false, "false"},
|
|
1081
|
+
{"max_error_bytes", "int", false, "4096"},
|
|
1082
|
+
{"max_record_bytes", "int", false, "67108864"},
|
|
1083
|
+
{"max_columns", "int", false, "8192"},
|
|
1084
|
+
{"nulls", "string|array", false, """"},
|
|
1085
|
+
{"na", "string|array", false, """"},
|
|
1086
|
+
{"quoted_nulls", "bool", false, "true"},
|
|
1087
|
+
{"audit", "bool", false, "false"},
|
|
1088
|
+
{"audit_limit", "int", false, "1000"},
|
|
1089
|
+
{"audit_include_row", "bool", false, "true"},
|
|
1090
|
+
{"audit_columns", "string[]", false, NULL},
|
|
1091
|
+
{"audit_redact", "string[]", false, NULL},
|
|
1092
|
+
{"audit_hash_columns", "string[]", false, NULL},
|
|
1093
|
+
{"audit_max_bytes", "size", false, "0"},
|
|
1094
|
+
{"audit_max_cell_bytes", "size", false, "0"},
|
|
331
1095
|
};
|
|
332
1096
|
|
|
333
1097
|
static tf_arg_desc csv_encode_args[] = {
|
|
@@ -337,6 +1101,10 @@ static tf_arg_desc csv_encode_args[] = {
|
|
|
337
1101
|
|
|
338
1102
|
static tf_arg_desc jsonl_decode_args[] = {
|
|
339
1103
|
{"batch_size", "int", false, "1024"},
|
|
1104
|
+
{"on_error", "string", false, "skip"},
|
|
1105
|
+
{"max_error_bytes", "int", false, "4096"},
|
|
1106
|
+
{"max_record_bytes", "int", false, "67108864"},
|
|
1107
|
+
{"max_columns", "int", false, "8192"},
|
|
340
1108
|
};
|
|
341
1109
|
|
|
342
1110
|
static tf_arg_desc jsonl_encode_args[] = {
|
|
@@ -345,6 +1113,9 @@ static tf_arg_desc jsonl_encode_args[] = {
|
|
|
345
1113
|
|
|
346
1114
|
static tf_arg_desc text_decode_args[] = {
|
|
347
1115
|
{"batch_size", "int", false, "1024"},
|
|
1116
|
+
{"max_error_bytes", "int", false, "4096"},
|
|
1117
|
+
{"max_record_bytes", "int", false, "67108864"},
|
|
1118
|
+
{"max_columns", "int", false, "8192"},
|
|
348
1119
|
};
|
|
349
1120
|
|
|
350
1121
|
static tf_arg_desc text_encode_args[] = {
|
|
@@ -360,12 +1131,26 @@ static tf_arg_desc grep_args[] = {
|
|
|
360
1131
|
|
|
361
1132
|
static tf_arg_desc filter_args[] = {
|
|
362
1133
|
{"expr", "string", true, NULL},
|
|
1134
|
+
{"audit", "bool", false, "false"},
|
|
1135
|
+
{"audit_limit", "int", false, "1000"},
|
|
1136
|
+
{"audit_include_row", "bool", false, "true"},
|
|
1137
|
+
{"audit_columns", "string[]", false, NULL},
|
|
1138
|
+
{"audit_redact", "string[]", false, NULL},
|
|
1139
|
+
{"audit_hash_columns", "string[]", false, NULL},
|
|
1140
|
+
{"audit_max_bytes", "int", false, NULL},
|
|
1141
|
+
{"audit_max_cell_bytes", "int", false, NULL},
|
|
363
1142
|
};
|
|
364
1143
|
|
|
365
1144
|
static tf_arg_desc select_args[] = {
|
|
366
1145
|
{"columns", "string[]", true, NULL},
|
|
367
1146
|
};
|
|
368
1147
|
|
|
1148
|
+
static tf_arg_desc relocate_args[] = {
|
|
1149
|
+
{"columns", "string[]", true, NULL},
|
|
1150
|
+
{"before", "string", false, NULL},
|
|
1151
|
+
{"after", "string", false, NULL},
|
|
1152
|
+
};
|
|
1153
|
+
|
|
369
1154
|
static tf_arg_desc rename_args[] = {
|
|
370
1155
|
{"mapping", "map", true, NULL},
|
|
371
1156
|
};
|
|
@@ -382,20 +1167,141 @@ static tf_arg_desc derive_args[] = {
|
|
|
382
1167
|
{"columns", "map[]", true, NULL},
|
|
383
1168
|
};
|
|
384
1169
|
|
|
1170
|
+
|
|
1171
|
+
static tf_arg_desc across_args[] = {
|
|
1172
|
+
{"columns", "string[]", true, NULL},
|
|
1173
|
+
{"fn", "string", false, NULL},
|
|
1174
|
+
{"functions", "string[]", false, NULL},
|
|
1175
|
+
{"names", "string", false, NULL},
|
|
1176
|
+
{"replace", "bool", false, NULL},
|
|
1177
|
+
};
|
|
1178
|
+
|
|
385
1179
|
static tf_arg_desc stats_args[] = {
|
|
386
1180
|
{"stats", "string[]", false, NULL},
|
|
387
1181
|
};
|
|
388
1182
|
|
|
1183
|
+
static tf_arg_desc scan_args[] = {
|
|
1184
|
+
{"stats", "string[]", false, NULL},
|
|
1185
|
+
};
|
|
1186
|
+
|
|
389
1187
|
static tf_arg_desc unique_args[] = {
|
|
390
1188
|
{"columns", "string[]", false, NULL},
|
|
1189
|
+
{"max_keys", "int", false, NULL},
|
|
1190
|
+
{"max_state_bytes", "int", false, NULL},
|
|
1191
|
+
{"spill_dir", "string", false, NULL},
|
|
1192
|
+
{"spill_memory_bytes", "int", false, NULL},
|
|
1193
|
+
{"spill_run_rows", "int", false, NULL},
|
|
1194
|
+
{"spill_output_rows", "int", false, NULL},
|
|
1195
|
+
{"sorted", "bool", false, "false"},
|
|
1196
|
+
{"mode", "string", false, "\"exact\""},
|
|
1197
|
+
{"approx", "bool", false, "false"},
|
|
1198
|
+
{"bloom_bytes", "int", false, NULL},
|
|
1199
|
+
{"bloom_hashes", "int", false, "7"},
|
|
391
1200
|
};
|
|
392
1201
|
|
|
393
1202
|
static tf_arg_desc sort_args[] = {
|
|
394
1203
|
{"columns", "map[]", true, NULL},
|
|
1204
|
+
{"spill_dir", "string", false, NULL},
|
|
1205
|
+
{"spill_memory_bytes", "int", false, NULL},
|
|
1206
|
+
{"spill_run_rows", "int", false, NULL},
|
|
1207
|
+
{"spill_output_rows", "int", false, NULL},
|
|
395
1208
|
};
|
|
396
1209
|
|
|
397
1210
|
static tf_arg_desc validate_args[] = {
|
|
1211
|
+
{"expr", "string", false, NULL},
|
|
1212
|
+
{"rules", "array|object", false, NULL},
|
|
1213
|
+
{"rules_file", "string", false, NULL},
|
|
1214
|
+
{"audit", "bool", false, "false"},
|
|
1215
|
+
{"audit_limit", "int", false, "1000"},
|
|
1216
|
+
{"audit_include_row", "bool", false, "true"},
|
|
1217
|
+
{"audit_columns", "string[]", false, NULL},
|
|
1218
|
+
{"audit_redact", "string[]", false, NULL},
|
|
1219
|
+
{"audit_hash_columns", "string[]", false, NULL},
|
|
1220
|
+
{"audit_max_bytes", "int", false, NULL},
|
|
1221
|
+
{"audit_max_cell_bytes", "int", false, NULL},
|
|
1222
|
+
{"max_failures", "int", false, NULL},
|
|
1223
|
+
{"max_failure_rate", "number", false, NULL},
|
|
1224
|
+
{"warn_failure_rate", "number", false, NULL},
|
|
1225
|
+
{"name", "string", false, "\"validate\""},
|
|
1226
|
+
{"message", "string", false, "\"\""},
|
|
1227
|
+
};
|
|
1228
|
+
|
|
1229
|
+
static tf_arg_desc assert_args[] = {
|
|
1230
|
+
{"expr", "string", false, NULL},
|
|
1231
|
+
{"aggregate", "string", false, NULL},
|
|
1232
|
+
{"column", "string", false, NULL},
|
|
1233
|
+
{"op", "string", false, NULL},
|
|
1234
|
+
{"value", "number", false, NULL},
|
|
1235
|
+
{"tolerance", "number", false, "1e-12"},
|
|
1236
|
+
{"rel", "bool", false, "true"},
|
|
1237
|
+
{"action", "string", false, "\"fail\""},
|
|
1238
|
+
{"name", "string", false, "\"assert\""},
|
|
1239
|
+
{"message", "string", false, "\"\""},
|
|
1240
|
+
{"result", "string", false, "\"_assert\""},
|
|
1241
|
+
{"audit", "bool", false, "false"},
|
|
1242
|
+
{"audit_limit", "int", false, "1000"},
|
|
1243
|
+
{"audit_include_row", "bool", false, "true"},
|
|
1244
|
+
{"audit_columns", "string[]", false, NULL},
|
|
1245
|
+
{"audit_redact", "string[]", false, NULL},
|
|
1246
|
+
{"audit_hash_columns", "string[]", false, NULL},
|
|
1247
|
+
{"audit_max_bytes", "int", false, NULL},
|
|
1248
|
+
{"audit_max_cell_bytes", "int", false, NULL},
|
|
1249
|
+
};
|
|
1250
|
+
|
|
1251
|
+
static tf_arg_desc quarantine_args[] = {
|
|
398
1252
|
{"expr", "string", true, NULL},
|
|
1253
|
+
{"name", "string", false, "\"quarantine\""},
|
|
1254
|
+
{"message", "string", false, "\"\""},
|
|
1255
|
+
{"audit_include_row", "bool", false, "true"},
|
|
1256
|
+
{"audit_columns", "string[]", false, NULL},
|
|
1257
|
+
{"audit_redact", "string[]", false, NULL},
|
|
1258
|
+
{"audit_hash_columns", "string[]", false, NULL},
|
|
1259
|
+
{"audit_max_bytes", "size", false, "0"},
|
|
1260
|
+
{"audit_max_cell_bytes", "size", false, "0"},
|
|
1261
|
+
};
|
|
1262
|
+
|
|
1263
|
+
static tf_arg_desc tee_args[] = {
|
|
1264
|
+
{"expr", "string", false, NULL},
|
|
1265
|
+
{"channel", "string", false, "\"samples\""},
|
|
1266
|
+
{"columns", "string[]", false, NULL},
|
|
1267
|
+
{"limit", "int", false, "1000"},
|
|
1268
|
+
{"every", "int", false, "1"},
|
|
1269
|
+
{"name", "string", false, "\"tee\""},
|
|
1270
|
+
{"include_row", "bool", false, "true"},
|
|
1271
|
+
{"audit_include_row", "bool", false, "true"},
|
|
1272
|
+
{"audit_columns", "string[]", false, NULL},
|
|
1273
|
+
{"audit_redact", "string[]", false, NULL},
|
|
1274
|
+
{"audit_hash_columns", "string[]", false, NULL},
|
|
1275
|
+
{"audit_max_bytes", "size", false, "0"},
|
|
1276
|
+
{"audit_max_cell_bytes", "size", false, "0"},
|
|
1277
|
+
};
|
|
1278
|
+
|
|
1279
|
+
static tf_arg_desc schema_args[] = {
|
|
1280
|
+
{"columns", "map", false, NULL},
|
|
1281
|
+
{"required", "string[]", false, NULL},
|
|
1282
|
+
{"non_null", "string[]", false, NULL},
|
|
1283
|
+
{"nullable", "string[]", false, NULL},
|
|
1284
|
+
{"values", "map", false, NULL},
|
|
1285
|
+
{"min", "map", false, NULL},
|
|
1286
|
+
{"max", "map", false, NULL},
|
|
1287
|
+
{"regex", "map", false, NULL},
|
|
1288
|
+
{"max_regex_pattern_bytes", "int", false, "4096"},
|
|
1289
|
+
{"max_regex_cell_bytes", "int", false, "65536"},
|
|
1290
|
+
{"audit_include_row", "bool", false, "true"},
|
|
1291
|
+
{"audit_columns", "string[]", false, NULL},
|
|
1292
|
+
{"audit_redact", "string[]", false, NULL},
|
|
1293
|
+
{"audit_hash_columns", "string[]", false, NULL},
|
|
1294
|
+
{"audit_max_bytes", "int", false, NULL},
|
|
1295
|
+
{"audit_max_cell_bytes", "int", false, NULL},
|
|
1296
|
+
{"mode", "string", false, "\"fail\""},
|
|
1297
|
+
{"action", "string", false, "\"fail\""},
|
|
1298
|
+
{"name", "string", false, "\"schema\""},
|
|
1299
|
+
{"message", "string", false, "\"\""},
|
|
1300
|
+
{"result", "string", false, "\"_schema\""},
|
|
1301
|
+
};
|
|
1302
|
+
|
|
1303
|
+
static tf_arg_desc schema_infer_args[] = {
|
|
1304
|
+
{"rows", "int", false, "10000"},
|
|
399
1305
|
};
|
|
400
1306
|
|
|
401
1307
|
static tf_arg_desc trim_args[] = {
|
|
@@ -404,10 +1310,26 @@ static tf_arg_desc trim_args[] = {
|
|
|
404
1310
|
|
|
405
1311
|
static tf_arg_desc fill_null_args[] = {
|
|
406
1312
|
{"mapping", "map", true, NULL},
|
|
1313
|
+
{"audit", "bool", false, "false"},
|
|
1314
|
+
{"audit_limit", "int", false, "1000"},
|
|
1315
|
+
{"audit_include_row", "bool", false, "true"},
|
|
1316
|
+
{"audit_columns", "string[]", false, NULL},
|
|
1317
|
+
{"audit_redact", "string[]", false, NULL},
|
|
1318
|
+
{"audit_hash_columns", "string[]", false, NULL},
|
|
1319
|
+
{"audit_max_bytes", "size", false, "0"},
|
|
1320
|
+
{"audit_max_cell_bytes", "size", false, "0"},
|
|
407
1321
|
};
|
|
408
1322
|
|
|
409
1323
|
static tf_arg_desc cast_args[] = {
|
|
410
1324
|
{"mapping", "map", true, NULL},
|
|
1325
|
+
{"audit", "bool", false, "false"},
|
|
1326
|
+
{"audit_limit", "int", false, "1000"},
|
|
1327
|
+
{"audit_include_row", "bool", false, "true"},
|
|
1328
|
+
{"audit_columns", "string[]", false, NULL},
|
|
1329
|
+
{"audit_redact", "string[]", false, NULL},
|
|
1330
|
+
{"audit_hash_columns", "string[]", false, NULL},
|
|
1331
|
+
{"audit_max_bytes", "size", false, "0"},
|
|
1332
|
+
{"audit_max_cell_bytes", "size", false, "0"},
|
|
411
1333
|
};
|
|
412
1334
|
|
|
413
1335
|
static tf_arg_desc clip_args[] = {
|
|
@@ -421,6 +1343,14 @@ static tf_arg_desc replace_args[] = {
|
|
|
421
1343
|
{"pattern", "string", true, NULL},
|
|
422
1344
|
{"replacement", "string", true, NULL},
|
|
423
1345
|
{"regex", "bool", false, "false"},
|
|
1346
|
+
{"audit", "bool", false, "false"},
|
|
1347
|
+
{"audit_limit", "int", false, "1000"},
|
|
1348
|
+
{"audit_include_row", "bool", false, "true"},
|
|
1349
|
+
{"audit_columns", "string[]", false, NULL},
|
|
1350
|
+
{"audit_redact", "string[]", false, NULL},
|
|
1351
|
+
{"audit_hash_columns", "string[]", false, NULL},
|
|
1352
|
+
{"audit_max_bytes", "size", false, "0"},
|
|
1353
|
+
{"audit_max_cell_bytes", "size", false, "0"},
|
|
424
1354
|
};
|
|
425
1355
|
|
|
426
1356
|
static tf_arg_desc hash_args[] = {
|
|
@@ -430,6 +1360,8 @@ static tf_arg_desc hash_args[] = {
|
|
|
430
1360
|
static tf_arg_desc bin_args[] = {
|
|
431
1361
|
{"column", "string", true, NULL},
|
|
432
1362
|
{"boundaries", "float[]", true, NULL},
|
|
1363
|
+
{"missing", "string", false, "\"error\""},
|
|
1364
|
+
{"on_type_error", "string", false, "\"fail\""},
|
|
433
1365
|
};
|
|
434
1366
|
|
|
435
1367
|
static tf_arg_desc fill_down_args[] = {
|
|
@@ -440,6 +1372,8 @@ static tf_arg_desc step_args[] = {
|
|
|
440
1372
|
{"column", "string", true, NULL},
|
|
441
1373
|
{"func", "string", true, NULL},
|
|
442
1374
|
{"result", "string", false, NULL},
|
|
1375
|
+
{"missing", "string", false, "\"error\""},
|
|
1376
|
+
{"on_type_error", "string", false, "\"fail\""},
|
|
443
1377
|
};
|
|
444
1378
|
|
|
445
1379
|
static tf_arg_desc window_args[] = {
|
|
@@ -447,11 +1381,32 @@ static tf_arg_desc window_args[] = {
|
|
|
447
1381
|
{"size", "int", true, NULL},
|
|
448
1382
|
{"func", "string", true, NULL},
|
|
449
1383
|
{"result", "string", false, NULL},
|
|
1384
|
+
{"missing", "string", false, "\"error\""},
|
|
1385
|
+
{"on_type_error", "string", false, "\"fail\""},
|
|
1386
|
+
};
|
|
1387
|
+
|
|
1388
|
+
static tf_arg_desc rolling_args[] = {
|
|
1389
|
+
{"column", "string", true, NULL},
|
|
1390
|
+
{"size", "int", true, NULL},
|
|
1391
|
+
{"result", "string", false, NULL},
|
|
1392
|
+
{"missing", "string", false, "\"error\""},
|
|
1393
|
+
{"on_type_error", "string", false, "\"fail\""},
|
|
1394
|
+
};
|
|
1395
|
+
|
|
1396
|
+
static tf_arg_desc rolling_bool_args[] = {
|
|
1397
|
+
{"column", "string", true, NULL},
|
|
1398
|
+
{"size", "int", true, NULL},
|
|
1399
|
+
{"result", "string", false, NULL},
|
|
1400
|
+
{"nulls", "string", false, "\"ignore\""},
|
|
450
1401
|
};
|
|
451
1402
|
|
|
452
1403
|
static tf_arg_desc explode_args[] = {
|
|
453
1404
|
{"column", "string", true, NULL},
|
|
454
1405
|
{"delimiter", "string", false, "\",\""},
|
|
1406
|
+
{"max_tokens_per_row", "int", false, NULL},
|
|
1407
|
+
{"max_output_rows_per_input_row", "int", false, NULL},
|
|
1408
|
+
{"max_output_rows_per_batch", "int", false, NULL},
|
|
1409
|
+
{"max_token_bytes", "int", false, NULL},
|
|
455
1410
|
};
|
|
456
1411
|
|
|
457
1412
|
static tf_arg_desc split_args[] = {
|
|
@@ -462,6 +1417,8 @@ static tf_arg_desc split_args[] = {
|
|
|
462
1417
|
|
|
463
1418
|
static tf_arg_desc unpivot_args[] = {
|
|
464
1419
|
{"columns", "string[]", true, NULL},
|
|
1420
|
+
{"max_output_rows_per_input_row", "int", false, NULL},
|
|
1421
|
+
{"max_output_rows_per_batch", "int", false, NULL},
|
|
465
1422
|
};
|
|
466
1423
|
|
|
467
1424
|
static tf_arg_desc tail_args[] = {
|
|
@@ -472,36 +1429,112 @@ static tf_arg_desc top_args[] = {
|
|
|
472
1429
|
{"n", "int", true, NULL},
|
|
473
1430
|
{"column", "string", true, NULL},
|
|
474
1431
|
{"desc", "bool", false, "true"},
|
|
1432
|
+
{"with_ties", "bool", false, "false"},
|
|
475
1433
|
};
|
|
476
1434
|
|
|
477
1435
|
static tf_arg_desc sample_args[] = {
|
|
478
1436
|
{"n", "int", true, NULL},
|
|
1437
|
+
{"seed", "int|string", false, "0"},
|
|
479
1438
|
};
|
|
480
1439
|
|
|
481
1440
|
static tf_arg_desc group_agg_args[] = {
|
|
482
1441
|
{"group_by", "string[]", true, NULL},
|
|
483
1442
|
{"aggs", "map[]", true, NULL},
|
|
1443
|
+
{"max_groups", "int", false, NULL},
|
|
1444
|
+
{"max_state_bytes", "int", false, NULL},
|
|
1445
|
+
{"spill_dir", "string", false, NULL},
|
|
1446
|
+
{"spill_memory_bytes", "int", false, NULL},
|
|
1447
|
+
{"spill_run_rows", "int", false, NULL},
|
|
1448
|
+
{"spill_output_rows", "int", false, NULL},
|
|
1449
|
+
{"sorted", "bool", false, "false"},
|
|
484
1450
|
};
|
|
485
1451
|
|
|
486
1452
|
static tf_arg_desc frequency_args[] = {
|
|
487
1453
|
{"columns", "string[]", false, NULL},
|
|
1454
|
+
{"max_values", "int", false, NULL},
|
|
1455
|
+
{"max_state_bytes", "int", false, NULL},
|
|
1456
|
+
{"overflow", "string", false, "\"error\""},
|
|
1457
|
+
{"other", "string", false, "\"__other__\""},
|
|
1458
|
+
{"audit", "bool", false, "false"},
|
|
1459
|
+
{"audit_limit", "int", false, "1000"},
|
|
1460
|
+
{"audit_include_row", "bool", false, "true"},
|
|
1461
|
+
{"audit_columns", "string[]", false, NULL},
|
|
1462
|
+
{"audit_redact", "string[]", false, NULL},
|
|
1463
|
+
{"audit_hash_columns", "string[]", false, NULL},
|
|
1464
|
+
{"audit_max_bytes", "size", false, "0"},
|
|
1465
|
+
{"audit_max_cell_bytes", "size", false, "0"},
|
|
488
1466
|
};
|
|
489
1467
|
|
|
490
1468
|
static tf_arg_desc datetime_args[] = {
|
|
491
1469
|
{"column", "string", true, NULL},
|
|
492
1470
|
{"extract", "string[]", false, NULL},
|
|
1471
|
+
{"missing", "string", false, "\"error\""},
|
|
1472
|
+
{"on_type_error", "string", false, "\"fail\""},
|
|
493
1473
|
};
|
|
494
1474
|
|
|
495
1475
|
static tf_arg_desc pivot_args[] = {
|
|
496
1476
|
{"name_column", "string", true, NULL},
|
|
497
1477
|
{"value_column", "string", true, NULL},
|
|
498
1478
|
{"agg", "string", false, "\"first\""},
|
|
1479
|
+
{"categories", "array", false, NULL},
|
|
1480
|
+
{"max_categories", "int", false, NULL},
|
|
1481
|
+
{"sorted", "bool", false, "false"},
|
|
1482
|
+
{"spill_dir", "string", false, NULL},
|
|
1483
|
+
{"spill_memory_bytes", "int", false, NULL},
|
|
1484
|
+
{"spill_run_rows", "int", false, NULL},
|
|
1485
|
+
{"spill_output_rows", "int", false, NULL},
|
|
499
1486
|
};
|
|
500
1487
|
|
|
501
1488
|
static tf_arg_desc join_args[] = {
|
|
502
1489
|
{"file", "string", true, NULL},
|
|
503
1490
|
{"on", "string", true, NULL},
|
|
504
1491
|
{"how", "string", false, "\"inner\""},
|
|
1492
|
+
{"max_lookup_rows", "int", false, NULL},
|
|
1493
|
+
{"max_lookup_keys", "int", false, NULL},
|
|
1494
|
+
{"max_lookup_bytes", "int", false, NULL},
|
|
1495
|
+
{"max_state_bytes", "int", false, NULL},
|
|
1496
|
+
{"max_matches_per_row", "int", false, NULL},
|
|
1497
|
+
{"max_output_rows", "int", false, NULL},
|
|
1498
|
+
{"spill_dir", "string", false, NULL},
|
|
1499
|
+
{"spill_memory_bytes", "int", false, NULL},
|
|
1500
|
+
{"spill_run_rows", "int", false, NULL},
|
|
1501
|
+
{"spill_output_rows", "int", false, NULL},
|
|
1502
|
+
{"sorted", "bool", false, "false"},
|
|
1503
|
+
};
|
|
1504
|
+
|
|
1505
|
+
static tf_arg_desc set_file_args[] = {
|
|
1506
|
+
{"file", "string", true, NULL},
|
|
1507
|
+
{"columns", "string[]", false, NULL},
|
|
1508
|
+
{"max_lookup_rows", "int", false, NULL},
|
|
1509
|
+
{"max_lookup_keys", "int", false, NULL},
|
|
1510
|
+
{"max_lookup_bytes", "int", false, NULL},
|
|
1511
|
+
{"max_output_keys", "int", false, NULL},
|
|
1512
|
+
{"max_state_bytes", "int", false, NULL},
|
|
1513
|
+
{"spill_dir", "string", false, NULL},
|
|
1514
|
+
{"spill_memory_bytes", "int", false, NULL},
|
|
1515
|
+
{"spill_run_rows", "int", false, NULL},
|
|
1516
|
+
{"spill_output_rows", "int", false, NULL},
|
|
1517
|
+
{"sorted", "bool", false, "false"},
|
|
1518
|
+
};
|
|
1519
|
+
|
|
1520
|
+
static tf_arg_desc union_args[] = {
|
|
1521
|
+
{"file", "string", true, NULL},
|
|
1522
|
+
{"columns", "string[]", false, NULL},
|
|
1523
|
+
{"max_lookup_rows", "int", false, NULL},
|
|
1524
|
+
{"max_lookup_bytes", "int", false, NULL},
|
|
1525
|
+
{"max_output_keys", "int", false, NULL},
|
|
1526
|
+
{"max_state_bytes", "int", false, NULL},
|
|
1527
|
+
{"spill_dir", "string", false, NULL},
|
|
1528
|
+
{"spill_memory_bytes", "int", false, NULL},
|
|
1529
|
+
{"spill_run_rows", "int", false, NULL},
|
|
1530
|
+
{"spill_output_rows", "int", false, NULL},
|
|
1531
|
+
{"sorted", "bool", false, "false"},
|
|
1532
|
+
};
|
|
1533
|
+
|
|
1534
|
+
static tf_arg_desc union_all_args[] = {
|
|
1535
|
+
{"file", "string", true, NULL},
|
|
1536
|
+
{"max_lookup_rows", "int", false, NULL},
|
|
1537
|
+
{"max_lookup_bytes", "int", false, NULL},
|
|
505
1538
|
};
|
|
506
1539
|
|
|
507
1540
|
static tf_arg_desc stack_args[] = {
|
|
@@ -516,38 +1549,80 @@ static tf_arg_desc lead_args[] = {
|
|
|
516
1549
|
{"result", "string", false, NULL},
|
|
517
1550
|
};
|
|
518
1551
|
|
|
1552
|
+
static tf_arg_desc lag_args[] = {
|
|
1553
|
+
{"column", "string", true, NULL},
|
|
1554
|
+
{"offset", "int", false, "1"},
|
|
1555
|
+
{"result", "string", false, NULL},
|
|
1556
|
+
};
|
|
1557
|
+
|
|
1558
|
+
static tf_arg_desc shift_args[] = {
|
|
1559
|
+
{"column", "string", true, NULL},
|
|
1560
|
+
{"offset", "int", false, "1"},
|
|
1561
|
+
{"result", "string", false, NULL},
|
|
1562
|
+
{"type", "string", false, "\"lag\""},
|
|
1563
|
+
};
|
|
1564
|
+
|
|
1565
|
+
static tf_arg_desc rowid_args[] = {
|
|
1566
|
+
{"columns", "string[]", false, NULL},
|
|
1567
|
+
{"result", "string", false, "\"_rowid\""},
|
|
1568
|
+
{"sorted", "bool", false, "false"},
|
|
1569
|
+
{"max_keys", "int", false, NULL},
|
|
1570
|
+
{"max_state_bytes", "int", false, NULL},
|
|
1571
|
+
};
|
|
1572
|
+
|
|
1573
|
+
static tf_arg_desc rleid_args[] = {
|
|
1574
|
+
{"columns", "string[]", true, NULL},
|
|
1575
|
+
{"result", "string", false, "\"_rleid\""},
|
|
1576
|
+
};
|
|
1577
|
+
|
|
519
1578
|
static tf_arg_desc date_trunc_args[] = {
|
|
520
1579
|
{"column", "string", true, NULL},
|
|
521
1580
|
{"trunc", "string", true, NULL},
|
|
522
1581
|
{"result", "string", false, NULL},
|
|
1582
|
+
{"missing", "string", false, "\"error\""},
|
|
1583
|
+
{"on_type_error", "string", false, "\"fail\""},
|
|
523
1584
|
};
|
|
524
1585
|
|
|
525
1586
|
static tf_arg_desc onehot_args[] = {
|
|
526
1587
|
{"column", "string", true, NULL},
|
|
527
1588
|
{"drop", "bool", false, "false"},
|
|
1589
|
+
{"categories", "array", false, NULL},
|
|
1590
|
+
{"max_categories", "int", false, NULL},
|
|
1591
|
+
{"max_state_bytes", "int", false, NULL},
|
|
1592
|
+
{"unknown", "string", false, NULL},
|
|
528
1593
|
};
|
|
529
1594
|
|
|
530
1595
|
static tf_arg_desc label_encode_args[] = {
|
|
531
1596
|
{"column", "string", true, NULL},
|
|
532
1597
|
{"result", "string", false, NULL},
|
|
1598
|
+
{"categories", "array", false, NULL},
|
|
1599
|
+
{"max_categories", "int", false, NULL},
|
|
1600
|
+
{"max_state_bytes", "int", false, NULL},
|
|
1601
|
+
{"unknown", "string", false, NULL},
|
|
533
1602
|
};
|
|
534
1603
|
|
|
535
1604
|
static tf_arg_desc ewma_args[] = {
|
|
536
1605
|
{"column", "string", true, NULL},
|
|
537
1606
|
{"alpha", "float", true, NULL},
|
|
538
1607
|
{"result", "string", false, NULL},
|
|
1608
|
+
{"missing", "string", false, "\"error\""},
|
|
1609
|
+
{"on_type_error", "string", false, "\"fail\""},
|
|
539
1610
|
};
|
|
540
1611
|
|
|
541
1612
|
static tf_arg_desc diff_args[] = {
|
|
542
1613
|
{"column", "string", true, NULL},
|
|
543
1614
|
{"order", "int", false, "1"},
|
|
544
1615
|
{"result", "string", false, NULL},
|
|
1616
|
+
{"missing", "string", false, "\"error\""},
|
|
1617
|
+
{"on_type_error", "string", false, "\"fail\""},
|
|
545
1618
|
};
|
|
546
1619
|
|
|
547
1620
|
static tf_arg_desc anomaly_args[] = {
|
|
548
1621
|
{"column", "string", true, NULL},
|
|
549
1622
|
{"threshold", "float", false, "3.0"},
|
|
550
1623
|
{"result", "string", false, NULL},
|
|
1624
|
+
{"missing", "string", false, "\"error\""},
|
|
1625
|
+
{"on_type_error", "string", false, "\"fail\""},
|
|
551
1626
|
};
|
|
552
1627
|
|
|
553
1628
|
static tf_arg_desc split_data_args[] = {
|
|
@@ -556,19 +1631,73 @@ static tf_arg_desc split_data_args[] = {
|
|
|
556
1631
|
{"seed", "int", false, "42"},
|
|
557
1632
|
};
|
|
558
1633
|
|
|
1634
|
+
static tf_arg_desc source_name_args[] = {
|
|
1635
|
+
{"result", "string", false, "\"_source\""},
|
|
1636
|
+
{"default", "string", false, "\"\""},
|
|
1637
|
+
};
|
|
1638
|
+
|
|
559
1639
|
static tf_arg_desc interpolate_args[] = {
|
|
560
1640
|
{"column", "string", true, NULL},
|
|
561
1641
|
{"method", "string", false, "\"linear\""},
|
|
1642
|
+
{"missing", "string", false, "\"error\""},
|
|
1643
|
+
{"on_type_error", "string", false, "\"fail\""},
|
|
562
1644
|
};
|
|
563
1645
|
|
|
564
1646
|
static tf_arg_desc normalize_args[] = {
|
|
565
1647
|
{"columns", "string[]", true, NULL},
|
|
566
1648
|
{"method", "string", false, "\"minmax\""},
|
|
1649
|
+
{"missing", "string", false, "\"error\""},
|
|
1650
|
+
{"on_type_error", "string", false, "\"fail\""},
|
|
1651
|
+
{"audit", "bool", false, "false"},
|
|
1652
|
+
{"audit_limit", "int", false, "1000"},
|
|
1653
|
+
{"audit_include_row", "bool", false, "true"},
|
|
1654
|
+
{"audit_columns", "string[]", false, NULL},
|
|
1655
|
+
{"audit_redact", "string[]", false, NULL},
|
|
1656
|
+
{"audit_hash_columns", "string[]", false, NULL},
|
|
1657
|
+
{"audit_max_bytes", "size", false, "0"},
|
|
1658
|
+
{"audit_max_cell_bytes", "size", false, "0"},
|
|
567
1659
|
};
|
|
568
1660
|
|
|
569
1661
|
static tf_arg_desc acf_args[] = {
|
|
570
1662
|
{"column", "string", true, NULL},
|
|
571
1663
|
{"lags", "int", false, "20"},
|
|
1664
|
+
{"missing", "string", false, "\"error\""},
|
|
1665
|
+
{"on_type_error", "string", false, "\"fail\""},
|
|
1666
|
+
};
|
|
1667
|
+
|
|
1668
|
+
static tf_arg_desc json_extract_args[] = {
|
|
1669
|
+
{"column", "string", false, "\"_line\""},
|
|
1670
|
+
{"path", "string", true, NULL},
|
|
1671
|
+
{"result", "string", true, NULL},
|
|
1672
|
+
{"type", "string", false, "\"string\""},
|
|
1673
|
+
};
|
|
1674
|
+
|
|
1675
|
+
static tf_arg_desc json_filter_args[] = {
|
|
1676
|
+
{"column", "string", false, "\"_line\""},
|
|
1677
|
+
{"path", "string", true, NULL},
|
|
1678
|
+
{"op", "string", false, "\"exists\""},
|
|
1679
|
+
{"value", "string", false, NULL},
|
|
1680
|
+
{"type", "string", false, "\"auto\""},
|
|
1681
|
+
};
|
|
1682
|
+
|
|
1683
|
+
static tf_arg_desc json_schema_args[] = {
|
|
1684
|
+
{"column", "string", false, "\"_line\""},
|
|
1685
|
+
{"schema", "object|string|bool", true, NULL},
|
|
1686
|
+
{"mode", "string", false, "\"annotate\""},
|
|
1687
|
+
{"result", "string", false, "\"_valid\""},
|
|
1688
|
+
{"audit", "bool", false, "false"},
|
|
1689
|
+
{"audit_limit", "int", false, "1000"},
|
|
1690
|
+
{"audit_include_row", "bool", false, "true"},
|
|
1691
|
+
{"audit_columns", "string[]", false, NULL},
|
|
1692
|
+
{"audit_redact", "string[]", false, NULL},
|
|
1693
|
+
{"audit_hash_columns", "string[]", false, NULL},
|
|
1694
|
+
{"audit_max_bytes", "int", false, NULL},
|
|
1695
|
+
{"audit_max_cell_bytes", "int", false, NULL},
|
|
1696
|
+
};
|
|
1697
|
+
|
|
1698
|
+
static tf_arg_desc json_flatten_args[] = {
|
|
1699
|
+
{"column", "string", false, "\"_line\""},
|
|
1700
|
+
{"fields", "object[]|string[]", true, NULL},
|
|
572
1701
|
};
|
|
573
1702
|
|
|
574
1703
|
static tf_arg_desc table_encode_args[] = {
|
|
@@ -576,14 +1705,49 @@ static tf_arg_desc table_encode_args[] = {
|
|
|
576
1705
|
{"max_rows", "int", false, "0"},
|
|
577
1706
|
};
|
|
578
1707
|
|
|
1708
|
+
|
|
1709
|
+
const char *tf_memory_class_name(tf_memory_class cls) {
|
|
1710
|
+
switch (cls) {
|
|
1711
|
+
case TF_MEM_ROW_LOCAL: return "row_local";
|
|
1712
|
+
case TF_MEM_BOUNDED_STATE: return "bounded_state";
|
|
1713
|
+
case TF_MEM_KEY_STATE: return "key_state";
|
|
1714
|
+
case TF_MEM_BLOCKING: return "blocking";
|
|
1715
|
+
case TF_MEM_EXTERNAL: return "external";
|
|
1716
|
+
default: return "unknown";
|
|
1717
|
+
}
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
const char *tf_emit_class_name(tf_emit_class cls) {
|
|
1721
|
+
switch (cls) {
|
|
1722
|
+
case TF_EMIT_PER_BATCH: return "per_batch";
|
|
1723
|
+
case TF_EMIT_ON_FLUSH: return "on_flush";
|
|
1724
|
+
case TF_EMIT_SIDE_ONLY: return "side_only";
|
|
1725
|
+
case TF_EMIT_MIXED: return "mixed";
|
|
1726
|
+
default: return "unknown";
|
|
1727
|
+
}
|
|
1728
|
+
}
|
|
1729
|
+
|
|
1730
|
+
const char *tf_schema_class_name(tf_schema_class cls) {
|
|
1731
|
+
switch (cls) {
|
|
1732
|
+
case TF_SCHEMA_STABLE: return "stable";
|
|
1733
|
+
case TF_SCHEMA_PARAMETRIC: return "parametric";
|
|
1734
|
+
case TF_SCHEMA_DATA_DEPENDENT: return "data_dependent";
|
|
1735
|
+
default: return "unknown";
|
|
1736
|
+
}
|
|
1737
|
+
}
|
|
1738
|
+
|
|
579
1739
|
static tf_op_entry builtin_ops[] = {
|
|
580
1740
|
{
|
|
581
1741
|
.name = "codec.csv.decode",
|
|
582
1742
|
.kind = TF_OP_DECODER,
|
|
583
1743
|
.tier = TF_TIER_CORE,
|
|
584
1744
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1745
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
1746
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1747
|
+
.schema_class = TF_SCHEMA_DATA_DEPENDENT,
|
|
1748
|
+
.state_estimate = "O(batch_size * columns)",
|
|
585
1749
|
.args = csv_decode_args,
|
|
586
|
-
.n_args =
|
|
1750
|
+
.n_args = 20,
|
|
587
1751
|
.infer_schema = infer_schema_unknown,
|
|
588
1752
|
.create_native = (void *(*)(const cJSON *))tf_csv_decoder_create,
|
|
589
1753
|
},
|
|
@@ -592,6 +1756,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
592
1756
|
.kind = TF_OP_ENCODER,
|
|
593
1757
|
.tier = TF_TIER_CORE,
|
|
594
1758
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1759
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
1760
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1761
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
1762
|
+
.state_estimate = "O(output_chunk)",
|
|
595
1763
|
.args = csv_encode_args,
|
|
596
1764
|
.n_args = 2,
|
|
597
1765
|
.infer_schema = infer_schema_sink,
|
|
@@ -602,8 +1770,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
602
1770
|
.kind = TF_OP_DECODER,
|
|
603
1771
|
.tier = TF_TIER_CORE,
|
|
604
1772
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1773
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
1774
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1775
|
+
.schema_class = TF_SCHEMA_DATA_DEPENDENT,
|
|
1776
|
+
.state_estimate = "O(batch_size)",
|
|
605
1777
|
.args = jsonl_decode_args,
|
|
606
|
-
.n_args =
|
|
1778
|
+
.n_args = 3,
|
|
607
1779
|
.infer_schema = infer_schema_unknown,
|
|
608
1780
|
.create_native = (void *(*)(const cJSON *))tf_jsonl_decoder_create,
|
|
609
1781
|
},
|
|
@@ -612,6 +1784,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
612
1784
|
.kind = TF_OP_ENCODER,
|
|
613
1785
|
.tier = TF_TIER_CORE,
|
|
614
1786
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1787
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
1788
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1789
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
1790
|
+
.state_estimate = "O(output_chunk)",
|
|
615
1791
|
.args = jsonl_encode_args,
|
|
616
1792
|
.n_args = 0,
|
|
617
1793
|
.infer_schema = infer_schema_sink,
|
|
@@ -622,6 +1798,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
622
1798
|
.kind = TF_OP_DECODER,
|
|
623
1799
|
.tier = TF_TIER_CORE,
|
|
624
1800
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1801
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
1802
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1803
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
1804
|
+
.state_estimate = "O(batch_size)",
|
|
625
1805
|
.args = text_decode_args,
|
|
626
1806
|
.n_args = 1,
|
|
627
1807
|
.infer_schema = infer_schema_unknown,
|
|
@@ -632,6 +1812,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
632
1812
|
.kind = TF_OP_ENCODER,
|
|
633
1813
|
.tier = TF_TIER_CORE,
|
|
634
1814
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1815
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
1816
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1817
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
1818
|
+
.state_estimate = "O(output_chunk)",
|
|
635
1819
|
.args = text_encode_args,
|
|
636
1820
|
.n_args = 0,
|
|
637
1821
|
.infer_schema = infer_schema_sink,
|
|
@@ -642,6 +1826,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
642
1826
|
.kind = TF_OP_TRANSFORM,
|
|
643
1827
|
.tier = TF_TIER_CORE,
|
|
644
1828
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1829
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
1830
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1831
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
1832
|
+
.state_estimate = "O(batch_rows)",
|
|
645
1833
|
.args = grep_args,
|
|
646
1834
|
.n_args = 4,
|
|
647
1835
|
.infer_schema = infer_schema_passthrough,
|
|
@@ -652,8 +1840,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
652
1840
|
.kind = TF_OP_TRANSFORM,
|
|
653
1841
|
.tier = TF_TIER_CORE,
|
|
654
1842
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1843
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
1844
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1845
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
1846
|
+
.state_estimate = "O(batch_rows * columns)",
|
|
655
1847
|
.args = filter_args,
|
|
656
|
-
.n_args =
|
|
1848
|
+
.n_args = 9,
|
|
657
1849
|
.infer_schema = infer_schema_passthrough,
|
|
658
1850
|
.create_native = (void *(*)(const cJSON *))tf_filter_create,
|
|
659
1851
|
},
|
|
@@ -662,6 +1854,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
662
1854
|
.kind = TF_OP_TRANSFORM,
|
|
663
1855
|
.tier = TF_TIER_CORE,
|
|
664
1856
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1857
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
1858
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1859
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
1860
|
+
.state_estimate = "O(batch_rows * selected_columns)",
|
|
665
1861
|
.args = select_args,
|
|
666
1862
|
.n_args = 1,
|
|
667
1863
|
.infer_schema = infer_schema_select,
|
|
@@ -672,6 +1868,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
672
1868
|
.kind = TF_OP_TRANSFORM,
|
|
673
1869
|
.tier = TF_TIER_CORE,
|
|
674
1870
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1871
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
1872
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1873
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
1874
|
+
.state_estimate = "O(columns)",
|
|
675
1875
|
.args = rename_args,
|
|
676
1876
|
.n_args = 1,
|
|
677
1877
|
.infer_schema = infer_schema_rename,
|
|
@@ -682,48 +1882,126 @@ static tf_op_entry builtin_ops[] = {
|
|
|
682
1882
|
.kind = TF_OP_TRANSFORM,
|
|
683
1883
|
.tier = TF_TIER_CORE,
|
|
684
1884
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1885
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
1886
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1887
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
1888
|
+
.state_estimate = "O(1)",
|
|
685
1889
|
.args = head_args,
|
|
686
1890
|
.n_args = 1,
|
|
687
1891
|
.infer_schema = infer_schema_passthrough,
|
|
688
1892
|
.create_native = (void *(*)(const cJSON *))tf_head_create,
|
|
689
1893
|
},
|
|
690
1894
|
{
|
|
691
|
-
.name = "
|
|
1895
|
+
.name = "slice-head",
|
|
1896
|
+
.kind = TF_OP_TRANSFORM,
|
|
1897
|
+
.tier = TF_TIER_CORE,
|
|
1898
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1899
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
1900
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1901
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
1902
|
+
.state_estimate = "O(1)",
|
|
1903
|
+
.args = head_args,
|
|
1904
|
+
.n_args = 1,
|
|
1905
|
+
.infer_schema = infer_schema_passthrough,
|
|
1906
|
+
.create_native = (void *(*)(const cJSON *))tf_head_create,
|
|
1907
|
+
},
|
|
1908
|
+
{
|
|
1909
|
+
.name = "skip",
|
|
1910
|
+
.kind = TF_OP_TRANSFORM,
|
|
1911
|
+
.tier = TF_TIER_CORE,
|
|
1912
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1913
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
1914
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1915
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
1916
|
+
.state_estimate = "O(1)",
|
|
1917
|
+
.args = skip_args,
|
|
1918
|
+
.n_args = 1,
|
|
1919
|
+
.infer_schema = infer_schema_passthrough,
|
|
1920
|
+
.create_native = (void *(*)(const cJSON *))tf_skip_create,
|
|
1921
|
+
},
|
|
1922
|
+
{
|
|
1923
|
+
.name = "derive",
|
|
1924
|
+
.kind = TF_OP_TRANSFORM,
|
|
1925
|
+
.tier = TF_TIER_CORE,
|
|
1926
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1927
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
1928
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1929
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
1930
|
+
.state_estimate = "O(batch_rows * derived_columns)",
|
|
1931
|
+
.args = derive_args,
|
|
1932
|
+
.n_args = 1,
|
|
1933
|
+
.infer_schema = infer_schema_derive,
|
|
1934
|
+
.create_native = (void *(*)(const cJSON *))tf_derive_create,
|
|
1935
|
+
},
|
|
1936
|
+
|
|
1937
|
+
{
|
|
1938
|
+
.name = "source-name",
|
|
1939
|
+
.kind = TF_OP_TRANSFORM,
|
|
1940
|
+
.tier = TF_TIER_CORE,
|
|
1941
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1942
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
1943
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1944
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
1945
|
+
.state_estimate = "O(batch_rows)",
|
|
1946
|
+
.args = source_name_args,
|
|
1947
|
+
.n_args = 2,
|
|
1948
|
+
.infer_schema = infer_schema_source_name,
|
|
1949
|
+
.create_native = (void *(*)(const cJSON *))tf_source_name_create,
|
|
1950
|
+
},
|
|
1951
|
+
|
|
1952
|
+
{
|
|
1953
|
+
.name = "across",
|
|
692
1954
|
.kind = TF_OP_TRANSFORM,
|
|
693
1955
|
.tier = TF_TIER_CORE,
|
|
694
1956
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
695
|
-
.
|
|
696
|
-
.
|
|
697
|
-
.
|
|
698
|
-
.
|
|
1957
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
1958
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
1959
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
1960
|
+
.state_estimate = "O(batch_rows * selected_columns * functions)",
|
|
1961
|
+
.args = across_args,
|
|
1962
|
+
.n_args = 5,
|
|
1963
|
+
.infer_schema = infer_schema_across,
|
|
1964
|
+
.create_native = (void *(*)(const cJSON *))tf_across_create,
|
|
699
1965
|
},
|
|
700
1966
|
{
|
|
701
|
-
.name = "
|
|
1967
|
+
.name = "stats",
|
|
702
1968
|
.kind = TF_OP_TRANSFORM,
|
|
703
1969
|
.tier = TF_TIER_CORE,
|
|
704
1970
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
705
|
-
.
|
|
1971
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
1972
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
1973
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
1974
|
+
.state_estimate = "O(columns * requested_stats)",
|
|
1975
|
+
.args = stats_args,
|
|
706
1976
|
.n_args = 1,
|
|
707
|
-
.infer_schema =
|
|
708
|
-
.create_native = (void *(*)(const cJSON *))
|
|
1977
|
+
.infer_schema = infer_schema_stats,
|
|
1978
|
+
.create_native = (void *(*)(const cJSON *))tf_stats_create,
|
|
709
1979
|
},
|
|
710
1980
|
{
|
|
711
|
-
.name = "
|
|
1981
|
+
.name = "scan",
|
|
712
1982
|
.kind = TF_OP_TRANSFORM,
|
|
713
1983
|
.tier = TF_TIER_CORE,
|
|
714
1984
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
715
|
-
.
|
|
1985
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
1986
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
1987
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
1988
|
+
.state_estimate = "O(columns * profile_stats)",
|
|
1989
|
+
.args = scan_args,
|
|
716
1990
|
.n_args = 1,
|
|
717
1991
|
.infer_schema = infer_schema_stats,
|
|
718
|
-
.create_native = (void *(*)(const cJSON *))
|
|
1992
|
+
.create_native = (void *(*)(const cJSON *))tf_scan_create,
|
|
719
1993
|
},
|
|
720
1994
|
{
|
|
721
1995
|
.name = "unique",
|
|
722
1996
|
.kind = TF_OP_TRANSFORM,
|
|
723
1997
|
.tier = TF_TIER_CORE,
|
|
724
1998
|
.caps = TF_CAP_STREAMING | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
1999
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2000
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2001
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2002
|
+
.state_estimate = "O(1) sorted, O(distinct_keys) exact unsorted or capped by max_keys/max_state_bytes, O(bloom_bytes) approximate, or O(spill_run_rows * columns) RAM with spill_dir",
|
|
725
2003
|
.args = unique_args,
|
|
726
|
-
.n_args =
|
|
2004
|
+
.n_args = 12,
|
|
727
2005
|
.infer_schema = infer_schema_passthrough,
|
|
728
2006
|
.create_native = (void *(*)(const cJSON *))tf_unique_create,
|
|
729
2007
|
},
|
|
@@ -732,8 +2010,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
732
2010
|
.kind = TF_OP_TRANSFORM,
|
|
733
2011
|
.tier = TF_TIER_CORE,
|
|
734
2012
|
.caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2013
|
+
.memory_class = TF_MEM_BLOCKING,
|
|
2014
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
2015
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2016
|
+
.state_estimate = "O(input_rows)",
|
|
735
2017
|
.args = sort_args,
|
|
736
|
-
.n_args =
|
|
2018
|
+
.n_args = 5,
|
|
737
2019
|
.infer_schema = infer_schema_passthrough,
|
|
738
2020
|
.create_native = (void *(*)(const cJSON *))tf_sort_create,
|
|
739
2021
|
},
|
|
@@ -743,18 +2025,40 @@ static tf_op_entry builtin_ops[] = {
|
|
|
743
2025
|
.kind = TF_OP_TRANSFORM,
|
|
744
2026
|
.tier = TF_TIER_CORE,
|
|
745
2027
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2028
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2029
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2030
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2031
|
+
.state_estimate = "O(batch_rows * selected_columns)",
|
|
746
2032
|
.args = select_args,
|
|
747
2033
|
.n_args = 1,
|
|
748
2034
|
.infer_schema = infer_schema_select,
|
|
749
2035
|
.create_native = (void *(*)(const cJSON *))tf_select_create,
|
|
750
2036
|
},
|
|
2037
|
+
{
|
|
2038
|
+
.name = "relocate",
|
|
2039
|
+
.kind = TF_OP_TRANSFORM,
|
|
2040
|
+
.tier = TF_TIER_CORE,
|
|
2041
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2042
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2043
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2044
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2045
|
+
.state_estimate = "O(batch_rows * columns)",
|
|
2046
|
+
.args = relocate_args,
|
|
2047
|
+
.n_args = 3,
|
|
2048
|
+
.infer_schema = infer_schema_relocate,
|
|
2049
|
+
.create_native = (void *(*)(const cJSON *))tf_relocate_create,
|
|
2050
|
+
},
|
|
751
2051
|
{
|
|
752
2052
|
.name = "dedup",
|
|
753
2053
|
.kind = TF_OP_TRANSFORM,
|
|
754
2054
|
.tier = TF_TIER_CORE,
|
|
755
2055
|
.caps = TF_CAP_STREAMING | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2056
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2057
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2058
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2059
|
+
.state_estimate = "O(1) sorted, O(distinct_keys) exact unsorted or capped by max_keys/max_state_bytes, O(bloom_bytes) approximate, or O(spill_run_rows * columns) RAM with spill_dir",
|
|
756
2060
|
.args = unique_args,
|
|
757
|
-
.n_args =
|
|
2061
|
+
.n_args = 12,
|
|
758
2062
|
.infer_schema = infer_schema_passthrough,
|
|
759
2063
|
.create_native = (void *(*)(const cJSON *))tf_unique_create,
|
|
760
2064
|
},
|
|
@@ -764,16 +2068,94 @@ static tf_op_entry builtin_ops[] = {
|
|
|
764
2068
|
.kind = TF_OP_TRANSFORM,
|
|
765
2069
|
.tier = TF_TIER_CORE,
|
|
766
2070
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2071
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2072
|
+
.emit_class = TF_EMIT_MIXED,
|
|
2073
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2074
|
+
.state_estimate = "O(batch_rows * columns)",
|
|
767
2075
|
.args = validate_args,
|
|
768
|
-
.n_args =
|
|
2076
|
+
.n_args = 16,
|
|
769
2077
|
.infer_schema = infer_schema_validate,
|
|
770
2078
|
.create_native = (void *(*)(const cJSON *))tf_validate_create,
|
|
771
2079
|
},
|
|
2080
|
+
{
|
|
2081
|
+
.name = "assert",
|
|
2082
|
+
.kind = TF_OP_TRANSFORM,
|
|
2083
|
+
.tier = TF_TIER_CORE,
|
|
2084
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2085
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2086
|
+
.emit_class = TF_EMIT_MIXED,
|
|
2087
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2088
|
+
.state_estimate = "O(batch_rows * columns) row mode, O(1) aggregate mode",
|
|
2089
|
+
.args = assert_args,
|
|
2090
|
+
.n_args = 19,
|
|
2091
|
+
.infer_schema = infer_schema_assert,
|
|
2092
|
+
.create_native = (void *(*)(const cJSON *))tf_assert_create,
|
|
2093
|
+
},
|
|
2094
|
+
{
|
|
2095
|
+
.name = "quarantine",
|
|
2096
|
+
.kind = TF_OP_TRANSFORM,
|
|
2097
|
+
.tier = TF_TIER_CORE,
|
|
2098
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2099
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2100
|
+
.emit_class = TF_EMIT_MIXED,
|
|
2101
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2102
|
+
.state_estimate = "O(batch_rows * columns)",
|
|
2103
|
+
.args = quarantine_args,
|
|
2104
|
+
.n_args = 9,
|
|
2105
|
+
.infer_schema = infer_schema_passthrough,
|
|
2106
|
+
.create_native = (void *(*)(const cJSON *))tf_quarantine_create,
|
|
2107
|
+
},
|
|
2108
|
+
{
|
|
2109
|
+
.name = "tee",
|
|
2110
|
+
.kind = TF_OP_TRANSFORM,
|
|
2111
|
+
.tier = TF_TIER_CORE,
|
|
2112
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2113
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2114
|
+
.emit_class = TF_EMIT_MIXED,
|
|
2115
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2116
|
+
.state_estimate = "O(batch_rows * selected_columns)",
|
|
2117
|
+
.args = tee_args,
|
|
2118
|
+
.n_args = 13,
|
|
2119
|
+
.infer_schema = infer_schema_passthrough,
|
|
2120
|
+
.create_native = (void *(*)(const cJSON *))tf_tee_create,
|
|
2121
|
+
},
|
|
2122
|
+
{
|
|
2123
|
+
.name = "schema",
|
|
2124
|
+
.kind = TF_OP_TRANSFORM,
|
|
2125
|
+
.tier = TF_TIER_CORE,
|
|
2126
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2127
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2128
|
+
.emit_class = TF_EMIT_MIXED,
|
|
2129
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2130
|
+
.state_estimate = "O(batch_rows * declared_columns)",
|
|
2131
|
+
.args = schema_args,
|
|
2132
|
+
.n_args = 13,
|
|
2133
|
+
.infer_schema = infer_schema_schema,
|
|
2134
|
+
.create_native = (void *(*)(const cJSON *))tf_schema_create,
|
|
2135
|
+
},
|
|
2136
|
+
{
|
|
2137
|
+
.name = "schema-infer",
|
|
2138
|
+
.kind = TF_OP_TRANSFORM,
|
|
2139
|
+
.tier = TF_TIER_CORE,
|
|
2140
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2141
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2142
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
2143
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2144
|
+
.state_estimate = "O(columns)",
|
|
2145
|
+
.args = schema_infer_args,
|
|
2146
|
+
.n_args = 1,
|
|
2147
|
+
.infer_schema = infer_schema_schema_infer,
|
|
2148
|
+
.create_native = (void *(*)(const cJSON *))tf_schema_infer_create,
|
|
2149
|
+
},
|
|
772
2150
|
{
|
|
773
2151
|
.name = "trim",
|
|
774
2152
|
.kind = TF_OP_TRANSFORM,
|
|
775
2153
|
.tier = TF_TIER_CORE,
|
|
776
2154
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2155
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2156
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2157
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2158
|
+
.state_estimate = "O(batch_rows * columns)",
|
|
777
2159
|
.args = trim_args,
|
|
778
2160
|
.n_args = 1,
|
|
779
2161
|
.infer_schema = infer_schema_passthrough,
|
|
@@ -784,8 +2166,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
784
2166
|
.kind = TF_OP_TRANSFORM,
|
|
785
2167
|
.tier = TF_TIER_CORE,
|
|
786
2168
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2169
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2170
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2171
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2172
|
+
.state_estimate = "O(batch_rows * columns)",
|
|
787
2173
|
.args = fill_null_args,
|
|
788
|
-
.n_args =
|
|
2174
|
+
.n_args = 9,
|
|
789
2175
|
.infer_schema = infer_schema_passthrough,
|
|
790
2176
|
.create_native = (void *(*)(const cJSON *))tf_fill_null_create,
|
|
791
2177
|
},
|
|
@@ -794,8 +2180,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
794
2180
|
.kind = TF_OP_TRANSFORM,
|
|
795
2181
|
.tier = TF_TIER_CORE,
|
|
796
2182
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2183
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2184
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2185
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2186
|
+
.state_estimate = "O(batch_rows * columns)",
|
|
797
2187
|
.args = cast_args,
|
|
798
|
-
.n_args =
|
|
2188
|
+
.n_args = 9,
|
|
799
2189
|
.infer_schema = infer_schema_passthrough, /* type changes at runtime */
|
|
800
2190
|
.create_native = (void *(*)(const cJSON *))tf_cast_create,
|
|
801
2191
|
},
|
|
@@ -804,6 +2194,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
804
2194
|
.kind = TF_OP_TRANSFORM,
|
|
805
2195
|
.tier = TF_TIER_CORE,
|
|
806
2196
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2197
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2198
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2199
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2200
|
+
.state_estimate = "O(batch_rows)",
|
|
807
2201
|
.args = clip_args,
|
|
808
2202
|
.n_args = 3,
|
|
809
2203
|
.infer_schema = infer_schema_passthrough,
|
|
@@ -814,8 +2208,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
814
2208
|
.kind = TF_OP_TRANSFORM,
|
|
815
2209
|
.tier = TF_TIER_CORE,
|
|
816
2210
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2211
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2212
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2213
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2214
|
+
.state_estimate = "O(batch_rows)",
|
|
817
2215
|
.args = replace_args,
|
|
818
|
-
.n_args =
|
|
2216
|
+
.n_args = 12,
|
|
819
2217
|
.infer_schema = infer_schema_passthrough,
|
|
820
2218
|
.create_native = (void *(*)(const cJSON *))tf_replace_create,
|
|
821
2219
|
},
|
|
@@ -824,6 +2222,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
824
2222
|
.kind = TF_OP_TRANSFORM,
|
|
825
2223
|
.tier = TF_TIER_CORE,
|
|
826
2224
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2225
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2226
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2227
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2228
|
+
.state_estimate = "O(batch_rows * selected_columns)",
|
|
827
2229
|
.args = hash_args,
|
|
828
2230
|
.n_args = 1,
|
|
829
2231
|
.infer_schema = infer_schema_add_hash,
|
|
@@ -834,8 +2236,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
834
2236
|
.kind = TF_OP_TRANSFORM,
|
|
835
2237
|
.tier = TF_TIER_CORE,
|
|
836
2238
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2239
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2240
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2241
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2242
|
+
.state_estimate = "O(boundaries)",
|
|
837
2243
|
.args = bin_args,
|
|
838
|
-
.n_args =
|
|
2244
|
+
.n_args = 4,
|
|
839
2245
|
.infer_schema = infer_schema_passthrough, /* adds column at runtime */
|
|
840
2246
|
.create_native = (void *(*)(const cJSON *))tf_bin_create,
|
|
841
2247
|
},
|
|
@@ -845,6 +2251,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
845
2251
|
.kind = TF_OP_TRANSFORM,
|
|
846
2252
|
.tier = TF_TIER_CORE,
|
|
847
2253
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2254
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2255
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2256
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2257
|
+
.state_estimate = "O(columns)",
|
|
848
2258
|
.args = fill_down_args,
|
|
849
2259
|
.n_args = 1,
|
|
850
2260
|
.infer_schema = infer_schema_passthrough,
|
|
@@ -855,6 +2265,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
855
2265
|
.kind = TF_OP_TRANSFORM,
|
|
856
2266
|
.tier = TF_TIER_CORE,
|
|
857
2267
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2268
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2269
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2270
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2271
|
+
.state_estimate = "O(1)",
|
|
858
2272
|
.args = step_args,
|
|
859
2273
|
.n_args = 3,
|
|
860
2274
|
.infer_schema = infer_schema_passthrough, /* adds column at runtime */
|
|
@@ -865,17 +2279,109 @@ static tf_op_entry builtin_ops[] = {
|
|
|
865
2279
|
.kind = TF_OP_TRANSFORM,
|
|
866
2280
|
.tier = TF_TIER_CORE,
|
|
867
2281
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2282
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2283
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2284
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2285
|
+
.state_estimate = "O(window_size)",
|
|
868
2286
|
.args = window_args,
|
|
869
|
-
.n_args =
|
|
2287
|
+
.n_args = 6,
|
|
870
2288
|
.infer_schema = infer_schema_passthrough, /* adds column at runtime */
|
|
871
2289
|
.create_native = (void *(*)(const cJSON *))tf_window_create,
|
|
872
2290
|
},
|
|
2291
|
+
{
|
|
2292
|
+
.name = "rolling-sum",
|
|
2293
|
+
.kind = TF_OP_TRANSFORM,
|
|
2294
|
+
.tier = TF_TIER_CORE,
|
|
2295
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2296
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2297
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2298
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2299
|
+
.state_estimate = "O(window_size)",
|
|
2300
|
+
.args = rolling_args,
|
|
2301
|
+
.n_args = 5,
|
|
2302
|
+
.infer_schema = infer_schema_passthrough,
|
|
2303
|
+
.create_native = (void *(*)(const cJSON *))tf_rolling_sum_create,
|
|
2304
|
+
},
|
|
2305
|
+
{
|
|
2306
|
+
.name = "rolling-mean",
|
|
2307
|
+
.kind = TF_OP_TRANSFORM,
|
|
2308
|
+
.tier = TF_TIER_CORE,
|
|
2309
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2310
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2311
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2312
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2313
|
+
.state_estimate = "O(window_size)",
|
|
2314
|
+
.args = rolling_args,
|
|
2315
|
+
.n_args = 5,
|
|
2316
|
+
.infer_schema = infer_schema_passthrough,
|
|
2317
|
+
.create_native = (void *(*)(const cJSON *))tf_rolling_mean_create,
|
|
2318
|
+
},
|
|
2319
|
+
{
|
|
2320
|
+
.name = "rolling-min",
|
|
2321
|
+
.kind = TF_OP_TRANSFORM,
|
|
2322
|
+
.tier = TF_TIER_CORE,
|
|
2323
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2324
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2325
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2326
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2327
|
+
.state_estimate = "O(window_size)",
|
|
2328
|
+
.args = rolling_args,
|
|
2329
|
+
.n_args = 5,
|
|
2330
|
+
.infer_schema = infer_schema_passthrough,
|
|
2331
|
+
.create_native = (void *(*)(const cJSON *))tf_rolling_min_create,
|
|
2332
|
+
},
|
|
2333
|
+
{
|
|
2334
|
+
.name = "rolling-max",
|
|
2335
|
+
.kind = TF_OP_TRANSFORM,
|
|
2336
|
+
.tier = TF_TIER_CORE,
|
|
2337
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2338
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2339
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2340
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2341
|
+
.state_estimate = "O(window_size)",
|
|
2342
|
+
.args = rolling_args,
|
|
2343
|
+
.n_args = 5,
|
|
2344
|
+
.infer_schema = infer_schema_passthrough,
|
|
2345
|
+
.create_native = (void *(*)(const cJSON *))tf_rolling_max_create,
|
|
2346
|
+
},
|
|
2347
|
+
{
|
|
2348
|
+
.name = "rolling-any",
|
|
2349
|
+
.kind = TF_OP_TRANSFORM,
|
|
2350
|
+
.tier = TF_TIER_CORE,
|
|
2351
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2352
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2353
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2354
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2355
|
+
.state_estimate = "O(window_size)",
|
|
2356
|
+
.args = rolling_bool_args,
|
|
2357
|
+
.n_args = 4,
|
|
2358
|
+
.infer_schema = infer_schema_passthrough,
|
|
2359
|
+
.create_native = (void *(*)(const cJSON *))tf_rolling_any_create,
|
|
2360
|
+
},
|
|
2361
|
+
{
|
|
2362
|
+
.name = "rolling-all",
|
|
2363
|
+
.kind = TF_OP_TRANSFORM,
|
|
2364
|
+
.tier = TF_TIER_CORE,
|
|
2365
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2366
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2367
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2368
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2369
|
+
.state_estimate = "O(window_size)",
|
|
2370
|
+
.args = rolling_bool_args,
|
|
2371
|
+
.n_args = 4,
|
|
2372
|
+
.infer_schema = infer_schema_passthrough,
|
|
2373
|
+
.create_native = (void *(*)(const cJSON *))tf_rolling_all_create,
|
|
2374
|
+
},
|
|
873
2375
|
/* ---- Row-multiplying streaming ---- */
|
|
874
2376
|
{
|
|
875
2377
|
.name = "explode",
|
|
876
2378
|
.kind = TF_OP_TRANSFORM,
|
|
877
2379
|
.tier = TF_TIER_CORE,
|
|
878
2380
|
.caps = TF_CAP_STREAMING | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2381
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2382
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2383
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2384
|
+
.state_estimate = "O(batch_rows)",
|
|
879
2385
|
.args = explode_args,
|
|
880
2386
|
.n_args = 2,
|
|
881
2387
|
.infer_schema = infer_schema_passthrough,
|
|
@@ -886,6 +2392,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
886
2392
|
.kind = TF_OP_TRANSFORM,
|
|
887
2393
|
.tier = TF_TIER_CORE,
|
|
888
2394
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2395
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2396
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2397
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2398
|
+
.state_estimate = "O(batch_rows * output_columns)",
|
|
889
2399
|
.args = split_args,
|
|
890
2400
|
.n_args = 3,
|
|
891
2401
|
.infer_schema = infer_schema_passthrough, /* adds columns at runtime */
|
|
@@ -896,6 +2406,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
896
2406
|
.kind = TF_OP_TRANSFORM,
|
|
897
2407
|
.tier = TF_TIER_CORE,
|
|
898
2408
|
.caps = TF_CAP_STREAMING | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2409
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2410
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2411
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2412
|
+
.state_estimate = "O(batch_rows * unpivot_columns)",
|
|
899
2413
|
.args = unpivot_args,
|
|
900
2414
|
.n_args = 1,
|
|
901
2415
|
.infer_schema = infer_schema_passthrough, /* schema changes at runtime */
|
|
@@ -907,6 +2421,24 @@ static tf_op_entry builtin_ops[] = {
|
|
|
907
2421
|
.kind = TF_OP_TRANSFORM,
|
|
908
2422
|
.tier = TF_TIER_CORE,
|
|
909
2423
|
.caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2424
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2425
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
2426
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2427
|
+
.state_estimate = "O(n)",
|
|
2428
|
+
.args = tail_args,
|
|
2429
|
+
.n_args = 1,
|
|
2430
|
+
.infer_schema = infer_schema_passthrough,
|
|
2431
|
+
.create_native = (void *(*)(const cJSON *))tf_tail_create,
|
|
2432
|
+
},
|
|
2433
|
+
{
|
|
2434
|
+
.name = "slice-tail",
|
|
2435
|
+
.kind = TF_OP_TRANSFORM,
|
|
2436
|
+
.tier = TF_TIER_CORE,
|
|
2437
|
+
.caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2438
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2439
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
2440
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2441
|
+
.state_estimate = "O(n)",
|
|
910
2442
|
.args = tail_args,
|
|
911
2443
|
.n_args = 1,
|
|
912
2444
|
.infer_schema = infer_schema_passthrough,
|
|
@@ -917,18 +2449,82 @@ static tf_op_entry builtin_ops[] = {
|
|
|
917
2449
|
.kind = TF_OP_TRANSFORM,
|
|
918
2450
|
.tier = TF_TIER_CORE,
|
|
919
2451
|
.caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2452
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2453
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
2454
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2455
|
+
.state_estimate = "O(n)",
|
|
920
2456
|
.args = top_args,
|
|
921
2457
|
.n_args = 3,
|
|
922
2458
|
.infer_schema = infer_schema_passthrough,
|
|
923
2459
|
.create_native = (void *(*)(const cJSON *))tf_top_create,
|
|
924
2460
|
},
|
|
2461
|
+
{
|
|
2462
|
+
.name = "top-k",
|
|
2463
|
+
.kind = TF_OP_TRANSFORM,
|
|
2464
|
+
.tier = TF_TIER_CORE,
|
|
2465
|
+
.caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2466
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2467
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
2468
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2469
|
+
.state_estimate = "O(n)",
|
|
2470
|
+
.args = top_args,
|
|
2471
|
+
.n_args = 3,
|
|
2472
|
+
.infer_schema = infer_schema_passthrough,
|
|
2473
|
+
.create_native = (void *(*)(const cJSON *))tf_top_k_create,
|
|
2474
|
+
},
|
|
2475
|
+
{
|
|
2476
|
+
.name = "bottom-k",
|
|
2477
|
+
.kind = TF_OP_TRANSFORM,
|
|
2478
|
+
.tier = TF_TIER_CORE,
|
|
2479
|
+
.caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2480
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2481
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
2482
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2483
|
+
.state_estimate = "O(n)",
|
|
2484
|
+
.args = top_args,
|
|
2485
|
+
.n_args = 3,
|
|
2486
|
+
.infer_schema = infer_schema_passthrough,
|
|
2487
|
+
.create_native = (void *(*)(const cJSON *))tf_bottom_k_create,
|
|
2488
|
+
},
|
|
2489
|
+
{
|
|
2490
|
+
.name = "slice-min",
|
|
2491
|
+
.kind = TF_OP_TRANSFORM,
|
|
2492
|
+
.tier = TF_TIER_CORE,
|
|
2493
|
+
.caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2494
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2495
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
2496
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2497
|
+
.state_estimate = "O(n)",
|
|
2498
|
+
.args = top_args,
|
|
2499
|
+
.n_args = 4,
|
|
2500
|
+
.infer_schema = infer_schema_passthrough,
|
|
2501
|
+
.create_native = (void *(*)(const cJSON *))tf_slice_min_create,
|
|
2502
|
+
},
|
|
2503
|
+
{
|
|
2504
|
+
.name = "slice-max",
|
|
2505
|
+
.kind = TF_OP_TRANSFORM,
|
|
2506
|
+
.tier = TF_TIER_CORE,
|
|
2507
|
+
.caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2508
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2509
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
2510
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2511
|
+
.state_estimate = "O(n)",
|
|
2512
|
+
.args = top_args,
|
|
2513
|
+
.n_args = 4,
|
|
2514
|
+
.infer_schema = infer_schema_passthrough,
|
|
2515
|
+
.create_native = (void *(*)(const cJSON *))tf_slice_max_create,
|
|
2516
|
+
},
|
|
925
2517
|
{
|
|
926
2518
|
.name = "sample",
|
|
927
2519
|
.kind = TF_OP_TRANSFORM,
|
|
928
2520
|
.tier = TF_TIER_CORE,
|
|
929
2521
|
.caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE,
|
|
2522
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2523
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
2524
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2525
|
+
.state_estimate = "O(n)",
|
|
930
2526
|
.args = sample_args,
|
|
931
|
-
.n_args =
|
|
2527
|
+
.n_args = 2,
|
|
932
2528
|
.infer_schema = infer_schema_passthrough,
|
|
933
2529
|
.create_native = (void *(*)(const cJSON *))tf_sample_create,
|
|
934
2530
|
},
|
|
@@ -937,8 +2533,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
937
2533
|
.kind = TF_OP_TRANSFORM,
|
|
938
2534
|
.tier = TF_TIER_CORE,
|
|
939
2535
|
.caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2536
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2537
|
+
.emit_class = TF_EMIT_MIXED,
|
|
2538
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2539
|
+
.state_estimate = "O(1) sorted, O(distinct_groups) unsorted or capped by max_groups/max_state_bytes, or O(spill_run_rows * columns) RAM with spill_dir",
|
|
940
2540
|
.args = group_agg_args,
|
|
941
|
-
.n_args =
|
|
2541
|
+
.n_args = 9,
|
|
942
2542
|
.infer_schema = infer_schema_group_agg,
|
|
943
2543
|
.create_native = (void *(*)(const cJSON *))tf_group_agg_create,
|
|
944
2544
|
},
|
|
@@ -947,8 +2547,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
947
2547
|
.kind = TF_OP_TRANSFORM,
|
|
948
2548
|
.tier = TF_TIER_CORE,
|
|
949
2549
|
.caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2550
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2551
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
2552
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2553
|
+
.state_estimate = "O(distinct_values) or capped by max_values/max_state_bytes",
|
|
950
2554
|
.args = frequency_args,
|
|
951
|
-
.n_args =
|
|
2555
|
+
.n_args = 13,
|
|
952
2556
|
.infer_schema = infer_schema_frequency,
|
|
953
2557
|
.create_native = (void *(*)(const cJSON *))tf_frequency_create,
|
|
954
2558
|
},
|
|
@@ -958,16 +2562,80 @@ static tf_op_entry builtin_ops[] = {
|
|
|
958
2562
|
.kind = TF_OP_TRANSFORM,
|
|
959
2563
|
.tier = TF_TIER_CORE,
|
|
960
2564
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2565
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2566
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2567
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2568
|
+
.state_estimate = "O(batch_rows)",
|
|
961
2569
|
.args = datetime_args,
|
|
962
|
-
.n_args =
|
|
2570
|
+
.n_args = 4,
|
|
963
2571
|
.infer_schema = infer_schema_passthrough, /* adds columns at runtime */
|
|
964
2572
|
.create_native = (void *(*)(const cJSON *))tf_datetime_create,
|
|
965
2573
|
},
|
|
2574
|
+
{
|
|
2575
|
+
.name = "json-extract",
|
|
2576
|
+
.kind = TF_OP_TRANSFORM,
|
|
2577
|
+
.tier = TF_TIER_CORE,
|
|
2578
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2579
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2580
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2581
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2582
|
+
.state_estimate = "O(batch_rows + path_segments)",
|
|
2583
|
+
.args = json_extract_args,
|
|
2584
|
+
.n_args = 4,
|
|
2585
|
+
.infer_schema = infer_schema_json_extract,
|
|
2586
|
+
.create_native = (void *(*)(const cJSON *))tf_json_extract_create,
|
|
2587
|
+
},
|
|
2588
|
+
{
|
|
2589
|
+
.name = "json-filter",
|
|
2590
|
+
.kind = TF_OP_TRANSFORM,
|
|
2591
|
+
.tier = TF_TIER_CORE,
|
|
2592
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2593
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2594
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2595
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2596
|
+
.state_estimate = "O(batch_rows + path_segments)",
|
|
2597
|
+
.args = json_filter_args,
|
|
2598
|
+
.n_args = 5,
|
|
2599
|
+
.infer_schema = infer_schema_passthrough,
|
|
2600
|
+
.create_native = (void *(*)(const cJSON *))tf_json_filter_create,
|
|
2601
|
+
},
|
|
2602
|
+
{
|
|
2603
|
+
.name = "json-schema",
|
|
2604
|
+
.kind = TF_OP_TRANSFORM,
|
|
2605
|
+
.tier = TF_TIER_CORE,
|
|
2606
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2607
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2608
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2609
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2610
|
+
.state_estimate = "O(batch_rows + schema_nodes)",
|
|
2611
|
+
.args = json_schema_args,
|
|
2612
|
+
.n_args = 12,
|
|
2613
|
+
.infer_schema = infer_schema_json_schema,
|
|
2614
|
+
.create_native = (void *(*)(const cJSON *))tf_json_schema_create,
|
|
2615
|
+
},
|
|
2616
|
+
{
|
|
2617
|
+
.name = "json-flatten",
|
|
2618
|
+
.kind = TF_OP_TRANSFORM,
|
|
2619
|
+
.tier = TF_TIER_CORE,
|
|
2620
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2621
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2622
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2623
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2624
|
+
.state_estimate = "O(batch_rows * fields + path_segments)",
|
|
2625
|
+
.args = json_flatten_args,
|
|
2626
|
+
.n_args = 2,
|
|
2627
|
+
.infer_schema = infer_schema_json_flatten,
|
|
2628
|
+
.create_native = (void *(*)(const cJSON *))tf_json_flatten_create,
|
|
2629
|
+
},
|
|
966
2630
|
{
|
|
967
2631
|
.name = "flatten",
|
|
968
2632
|
.kind = TF_OP_TRANSFORM,
|
|
969
2633
|
.tier = TF_TIER_CORE,
|
|
970
2634
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2635
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2636
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2637
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2638
|
+
.state_estimate = "O(batch_rows * fields)",
|
|
971
2639
|
.args = NULL,
|
|
972
2640
|
.n_args = 0,
|
|
973
2641
|
.infer_schema = infer_schema_passthrough,
|
|
@@ -979,9 +2647,13 @@ static tf_op_entry builtin_ops[] = {
|
|
|
979
2647
|
.kind = TF_OP_TRANSFORM,
|
|
980
2648
|
.tier = TF_TIER_CORE,
|
|
981
2649
|
.caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2650
|
+
.memory_class = TF_MEM_BLOCKING,
|
|
2651
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
2652
|
+
.schema_class = TF_SCHEMA_DATA_DEPENDENT,
|
|
2653
|
+
.state_estimate = "O(input_rows + distinct_categories), O(current_group + categories) with categories+sorted=true, or external spill with categories/max_categories",
|
|
982
2654
|
.args = pivot_args,
|
|
983
|
-
.n_args =
|
|
984
|
-
.infer_schema =
|
|
2655
|
+
.n_args = 10,
|
|
2656
|
+
.infer_schema = infer_schema_pivot,
|
|
985
2657
|
.create_native = (void *(*)(const cJSON *))tf_pivot_create,
|
|
986
2658
|
},
|
|
987
2659
|
{
|
|
@@ -989,16 +2661,136 @@ static tf_op_entry builtin_ops[] = {
|
|
|
989
2661
|
.kind = TF_OP_TRANSFORM,
|
|
990
2662
|
.tier = TF_TIER_CORE,
|
|
991
2663
|
.caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
|
|
2664
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2665
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2666
|
+
.schema_class = TF_SCHEMA_DATA_DEPENDENT,
|
|
2667
|
+
.state_estimate = "O(current_lookup_run) sorted, O(lookup_rows + lookup_keys + output_batch) unsorted or capped by max_lookup_*/max_state_bytes/max_output_rows, or O(spill_run_rows * columns + max_matches_per_row) RAM with spill_dir",
|
|
992
2668
|
.args = join_args,
|
|
993
|
-
.n_args =
|
|
2669
|
+
.n_args = 14,
|
|
994
2670
|
.infer_schema = infer_schema_passthrough, /* schema depends on lookup file */
|
|
995
2671
|
.create_native = (void *(*)(const cJSON *))tf_join_create,
|
|
996
2672
|
},
|
|
2673
|
+
{
|
|
2674
|
+
.name = "semi-join",
|
|
2675
|
+
.kind = TF_OP_TRANSFORM,
|
|
2676
|
+
.tier = TF_TIER_CORE,
|
|
2677
|
+
.caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
|
|
2678
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2679
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2680
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2681
|
+
.state_estimate = "O(1) sorted, O(lookup_keys) unsorted or capped by max_lookup_*/max_state_bytes/max_output_rows, or O(spill_run_rows * columns) RAM with spill_dir",
|
|
2682
|
+
.args = join_args,
|
|
2683
|
+
.n_args = 14,
|
|
2684
|
+
.infer_schema = infer_schema_passthrough,
|
|
2685
|
+
.create_native = (void *(*)(const cJSON *))tf_semi_join_create,
|
|
2686
|
+
},
|
|
2687
|
+
{
|
|
2688
|
+
.name = "anti-join",
|
|
2689
|
+
.kind = TF_OP_TRANSFORM,
|
|
2690
|
+
.tier = TF_TIER_CORE,
|
|
2691
|
+
.caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
|
|
2692
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2693
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2694
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2695
|
+
.state_estimate = "O(1) sorted, O(lookup_keys) unsorted or capped by max_lookup_*/max_state_bytes/max_output_rows, or O(spill_run_rows * columns) RAM with spill_dir",
|
|
2696
|
+
.args = join_args,
|
|
2697
|
+
.n_args = 14,
|
|
2698
|
+
.infer_schema = infer_schema_passthrough,
|
|
2699
|
+
.create_native = (void *(*)(const cJSON *))tf_anti_join_create,
|
|
2700
|
+
},
|
|
2701
|
+
{
|
|
2702
|
+
.name = "intersect",
|
|
2703
|
+
.kind = TF_OP_TRANSFORM,
|
|
2704
|
+
.tier = TF_TIER_CORE,
|
|
2705
|
+
.caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
|
|
2706
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2707
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2708
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2709
|
+
.state_estimate = "O(1) sorted, O(lookup_keys + emitted_keys) unsorted or capped by max_lookup_*/max_output_keys/max_state_bytes",
|
|
2710
|
+
.args = set_file_args,
|
|
2711
|
+
.n_args = 12,
|
|
2712
|
+
.infer_schema = infer_schema_passthrough,
|
|
2713
|
+
.create_native = (void *(*)(const cJSON *))tf_intersect_create,
|
|
2714
|
+
},
|
|
2715
|
+
{
|
|
2716
|
+
.name = "setdiff",
|
|
2717
|
+
.kind = TF_OP_TRANSFORM,
|
|
2718
|
+
.tier = TF_TIER_CORE,
|
|
2719
|
+
.caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
|
|
2720
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2721
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2722
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2723
|
+
.state_estimate = "O(1) sorted, O(lookup_keys + emitted_keys) unsorted or capped by max_lookup_*/max_output_keys/max_state_bytes",
|
|
2724
|
+
.args = set_file_args,
|
|
2725
|
+
.n_args = 12,
|
|
2726
|
+
.infer_schema = infer_schema_passthrough,
|
|
2727
|
+
.create_native = (void *(*)(const cJSON *))tf_setdiff_create,
|
|
2728
|
+
},
|
|
2729
|
+
{
|
|
2730
|
+
.name = "intersect-all",
|
|
2731
|
+
.kind = TF_OP_TRANSFORM,
|
|
2732
|
+
.tier = TF_TIER_CORE,
|
|
2733
|
+
.caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
|
|
2734
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2735
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2736
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2737
|
+
.state_estimate = "O(lookup_keys) unsorted with max_lookup_keys/max_state_bytes; O(spill_run_rows * columns) RAM with spill_dir; O(1) current-run state with sorted=true; emits min(left_count, lookup_count) rows per key",
|
|
2738
|
+
.args = set_file_args,
|
|
2739
|
+
.n_args = 12,
|
|
2740
|
+
.infer_schema = infer_schema_passthrough,
|
|
2741
|
+
.create_native = (void *(*)(const cJSON *))tf_intersect_all_create,
|
|
2742
|
+
},
|
|
2743
|
+
{
|
|
2744
|
+
.name = "setdiff-all",
|
|
2745
|
+
.kind = TF_OP_TRANSFORM,
|
|
2746
|
+
.tier = TF_TIER_CORE,
|
|
2747
|
+
.caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
|
|
2748
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2749
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2750
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2751
|
+
.state_estimate = "O(lookup_keys) unsorted with max_lookup_keys/max_state_bytes; O(spill_run_rows * columns) RAM with spill_dir; O(1) current-run state with sorted=true; emits max(left_count - lookup_count, 0) rows per key",
|
|
2752
|
+
.args = set_file_args,
|
|
2753
|
+
.n_args = 12,
|
|
2754
|
+
.infer_schema = infer_schema_passthrough,
|
|
2755
|
+
.create_native = (void *(*)(const cJSON *))tf_setdiff_all_create,
|
|
2756
|
+
},
|
|
2757
|
+
{
|
|
2758
|
+
.name = "union",
|
|
2759
|
+
.kind = TF_OP_TRANSFORM,
|
|
2760
|
+
.tier = TF_TIER_CORE,
|
|
2761
|
+
.caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
|
|
2762
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2763
|
+
.emit_class = TF_EMIT_MIXED,
|
|
2764
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2765
|
+
.state_estimate = "O(emitted_keys), capped by max_output_keys/max_state_bytes; O(previous_left_key + current_file_key) with sorted=true; O(spill_run_rows * columns) RAM with spill_dir",
|
|
2766
|
+
.args = union_args,
|
|
2767
|
+
.n_args = 11,
|
|
2768
|
+
.infer_schema = infer_schema_passthrough,
|
|
2769
|
+
.create_native = (void *(*)(const cJSON *))tf_union_create,
|
|
2770
|
+
},
|
|
2771
|
+
{
|
|
2772
|
+
.name = "union-all",
|
|
2773
|
+
.kind = TF_OP_TRANSFORM,
|
|
2774
|
+
.tier = TF_TIER_CORE,
|
|
2775
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_FS | TF_CAP_DETERMINISTIC,
|
|
2776
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2777
|
+
.emit_class = TF_EMIT_MIXED,
|
|
2778
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2779
|
+
.state_estimate = "O(batch_rows)",
|
|
2780
|
+
.args = union_all_args,
|
|
2781
|
+
.n_args = 3,
|
|
2782
|
+
.infer_schema = infer_schema_passthrough,
|
|
2783
|
+
.create_native = (void *(*)(const cJSON *))tf_union_all_create,
|
|
2784
|
+
},
|
|
997
2785
|
{
|
|
998
2786
|
.name = "stack",
|
|
999
2787
|
.kind = TF_OP_TRANSFORM,
|
|
1000
2788
|
.tier = TF_TIER_CORE,
|
|
1001
|
-
.caps = TF_CAP_FS | TF_CAP_DETERMINISTIC,
|
|
2789
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_FS | TF_CAP_DETERMINISTIC,
|
|
2790
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2791
|
+
.emit_class = TF_EMIT_MIXED,
|
|
2792
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
2793
|
+
.state_estimate = "O(batch_rows)",
|
|
1002
2794
|
.args = stack_args,
|
|
1003
2795
|
.n_args = 3,
|
|
1004
2796
|
.infer_schema = infer_schema_passthrough,
|
|
@@ -1009,18 +2801,82 @@ static tf_op_entry builtin_ops[] = {
|
|
|
1009
2801
|
.kind = TF_OP_TRANSFORM,
|
|
1010
2802
|
.tier = TF_TIER_CORE,
|
|
1011
2803
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2804
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2805
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2806
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2807
|
+
.state_estimate = "O(offset)",
|
|
1012
2808
|
.args = lead_args,
|
|
1013
2809
|
.n_args = 3,
|
|
1014
|
-
.infer_schema =
|
|
2810
|
+
.infer_schema = infer_schema_lead,
|
|
1015
2811
|
.create_native = (void *(*)(const cJSON *))tf_lead_create,
|
|
1016
2812
|
},
|
|
2813
|
+
{
|
|
2814
|
+
.name = "lag",
|
|
2815
|
+
.kind = TF_OP_TRANSFORM,
|
|
2816
|
+
.tier = TF_TIER_CORE,
|
|
2817
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2818
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2819
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2820
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2821
|
+
.state_estimate = "O(offset * row_width)",
|
|
2822
|
+
.args = lag_args,
|
|
2823
|
+
.n_args = 3,
|
|
2824
|
+
.infer_schema = infer_schema_lag,
|
|
2825
|
+
.create_native = (void *(*)(const cJSON *))tf_lag_create,
|
|
2826
|
+
},
|
|
2827
|
+
{
|
|
2828
|
+
.name = "shift",
|
|
2829
|
+
.kind = TF_OP_TRANSFORM,
|
|
2830
|
+
.tier = TF_TIER_CORE,
|
|
2831
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2832
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2833
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2834
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2835
|
+
.state_estimate = "O(offset * row_width)",
|
|
2836
|
+
.args = shift_args,
|
|
2837
|
+
.n_args = 4,
|
|
2838
|
+
.infer_schema = infer_schema_shift,
|
|
2839
|
+
.create_native = (void *(*)(const cJSON *))tf_shift_create,
|
|
2840
|
+
},
|
|
2841
|
+
{
|
|
2842
|
+
.name = "rowid",
|
|
2843
|
+
.kind = TF_OP_TRANSFORM,
|
|
2844
|
+
.tier = TF_TIER_CORE,
|
|
2845
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2846
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2847
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2848
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2849
|
+
.state_estimate = "O(1) global/sorted, O(distinct_keys) unsorted or capped by max_keys/max_state_bytes",
|
|
2850
|
+
.args = rowid_args,
|
|
2851
|
+
.n_args = 5,
|
|
2852
|
+
.infer_schema = infer_schema_rowid,
|
|
2853
|
+
.create_native = (void *(*)(const cJSON *))tf_rowid_create,
|
|
2854
|
+
},
|
|
2855
|
+
{
|
|
2856
|
+
.name = "rleid",
|
|
2857
|
+
.kind = TF_OP_TRANSFORM,
|
|
2858
|
+
.tier = TF_TIER_CORE,
|
|
2859
|
+
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2860
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2861
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2862
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2863
|
+
.state_estimate = "O(key_width)",
|
|
2864
|
+
.args = rleid_args,
|
|
2865
|
+
.n_args = 2,
|
|
2866
|
+
.infer_schema = infer_schema_rleid,
|
|
2867
|
+
.create_native = (void *(*)(const cJSON *))tf_rleid_create,
|
|
2868
|
+
},
|
|
1017
2869
|
{
|
|
1018
2870
|
.name = "date-trunc",
|
|
1019
2871
|
.kind = TF_OP_TRANSFORM,
|
|
1020
2872
|
.tier = TF_TIER_CORE,
|
|
1021
2873
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2874
|
+
.memory_class = TF_MEM_ROW_LOCAL,
|
|
2875
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2876
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2877
|
+
.state_estimate = "O(batch_rows)",
|
|
1022
2878
|
.args = date_trunc_args,
|
|
1023
|
-
.n_args =
|
|
2879
|
+
.n_args = 5,
|
|
1024
2880
|
.infer_schema = infer_schema_passthrough,
|
|
1025
2881
|
.create_native = (void *(*)(const cJSON *))tf_date_trunc_create,
|
|
1026
2882
|
},
|
|
@@ -1029,8 +2885,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
1029
2885
|
.kind = TF_OP_TRANSFORM,
|
|
1030
2886
|
.tier = TF_TIER_CORE,
|
|
1031
2887
|
.caps = TF_CAP_STREAMING | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2888
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2889
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2890
|
+
.schema_class = TF_SCHEMA_DATA_DEPENDENT,
|
|
2891
|
+
.state_estimate = "O(distinct_categories) or capped by max_categories/max_state_bytes",
|
|
1032
2892
|
.args = onehot_args,
|
|
1033
|
-
.n_args =
|
|
2893
|
+
.n_args = 6,
|
|
1034
2894
|
.infer_schema = infer_schema_passthrough,
|
|
1035
2895
|
.create_native = (void *(*)(const cJSON *))tf_onehot_create,
|
|
1036
2896
|
},
|
|
@@ -1039,8 +2899,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
1039
2899
|
.kind = TF_OP_TRANSFORM,
|
|
1040
2900
|
.tier = TF_TIER_CORE,
|
|
1041
2901
|
.caps = TF_CAP_STREAMING | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2902
|
+
.memory_class = TF_MEM_KEY_STATE,
|
|
2903
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2904
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2905
|
+
.state_estimate = "O(distinct_categories) or capped by max_categories/max_state_bytes",
|
|
1042
2906
|
.args = label_encode_args,
|
|
1043
|
-
.n_args =
|
|
2907
|
+
.n_args = 6,
|
|
1044
2908
|
.infer_schema = infer_schema_passthrough,
|
|
1045
2909
|
.create_native = (void *(*)(const cJSON *))tf_label_encode_create,
|
|
1046
2910
|
},
|
|
@@ -1049,8 +2913,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
1049
2913
|
.kind = TF_OP_TRANSFORM,
|
|
1050
2914
|
.tier = TF_TIER_CORE,
|
|
1051
2915
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2916
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2917
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2918
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2919
|
+
.state_estimate = "O(1)",
|
|
1052
2920
|
.args = ewma_args,
|
|
1053
|
-
.n_args =
|
|
2921
|
+
.n_args = 5,
|
|
1054
2922
|
.infer_schema = infer_schema_passthrough,
|
|
1055
2923
|
.create_native = (void *(*)(const cJSON *))tf_ewma_create,
|
|
1056
2924
|
},
|
|
@@ -1059,6 +2927,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
1059
2927
|
.kind = TF_OP_TRANSFORM,
|
|
1060
2928
|
.tier = TF_TIER_CORE,
|
|
1061
2929
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2930
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2931
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2932
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2933
|
+
.state_estimate = "O(order)",
|
|
1062
2934
|
.args = diff_args,
|
|
1063
2935
|
.n_args = 3,
|
|
1064
2936
|
.infer_schema = infer_schema_passthrough,
|
|
@@ -1069,8 +2941,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
1069
2941
|
.kind = TF_OP_TRANSFORM,
|
|
1070
2942
|
.tier = TF_TIER_CORE,
|
|
1071
2943
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2944
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2945
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2946
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2947
|
+
.state_estimate = "O(1)",
|
|
1072
2948
|
.args = anomaly_args,
|
|
1073
|
-
.n_args =
|
|
2949
|
+
.n_args = 5,
|
|
1074
2950
|
.infer_schema = infer_schema_passthrough,
|
|
1075
2951
|
.create_native = (void *(*)(const cJSON *))tf_anomaly_create,
|
|
1076
2952
|
},
|
|
@@ -1079,6 +2955,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
1079
2955
|
.kind = TF_OP_TRANSFORM,
|
|
1080
2956
|
.tier = TF_TIER_CORE,
|
|
1081
2957
|
.caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE,
|
|
2958
|
+
.memory_class = TF_MEM_BOUNDED_STATE,
|
|
2959
|
+
.emit_class = TF_EMIT_PER_BATCH,
|
|
2960
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2961
|
+
.state_estimate = "O(1)",
|
|
1082
2962
|
.args = split_data_args,
|
|
1083
2963
|
.n_args = 3,
|
|
1084
2964
|
.infer_schema = infer_schema_passthrough,
|
|
@@ -1089,8 +2969,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
1089
2969
|
.kind = TF_OP_TRANSFORM,
|
|
1090
2970
|
.tier = TF_TIER_CORE,
|
|
1091
2971
|
.caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2972
|
+
.memory_class = TF_MEM_BLOCKING,
|
|
2973
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
2974
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2975
|
+
.state_estimate = "O(input_rows)",
|
|
1092
2976
|
.args = interpolate_args,
|
|
1093
|
-
.n_args =
|
|
2977
|
+
.n_args = 4,
|
|
1094
2978
|
.infer_schema = infer_schema_passthrough,
|
|
1095
2979
|
.create_native = (void *(*)(const cJSON *))tf_interpolate_create,
|
|
1096
2980
|
},
|
|
@@ -1099,8 +2983,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
1099
2983
|
.kind = TF_OP_TRANSFORM,
|
|
1100
2984
|
.tier = TF_TIER_CORE,
|
|
1101
2985
|
.caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
2986
|
+
.memory_class = TF_MEM_BLOCKING,
|
|
2987
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
2988
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
2989
|
+
.state_estimate = "O(input_rows)",
|
|
1102
2990
|
.args = normalize_args,
|
|
1103
|
-
.n_args =
|
|
2991
|
+
.n_args = 12,
|
|
1104
2992
|
.infer_schema = infer_schema_passthrough,
|
|
1105
2993
|
.create_native = (void *(*)(const cJSON *))tf_normalize_create,
|
|
1106
2994
|
},
|
|
@@ -1109,8 +2997,12 @@ static tf_op_entry builtin_ops[] = {
|
|
|
1109
2997
|
.kind = TF_OP_TRANSFORM,
|
|
1110
2998
|
.tier = TF_TIER_CORE,
|
|
1111
2999
|
.caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
3000
|
+
.memory_class = TF_MEM_BLOCKING,
|
|
3001
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
3002
|
+
.schema_class = TF_SCHEMA_PARAMETRIC,
|
|
3003
|
+
.state_estimate = "O(input_rows)",
|
|
1112
3004
|
.args = acf_args,
|
|
1113
|
-
.n_args =
|
|
3005
|
+
.n_args = 4,
|
|
1114
3006
|
.infer_schema = infer_schema_passthrough,
|
|
1115
3007
|
.create_native = (void *(*)(const cJSON *))tf_acf_create,
|
|
1116
3008
|
},
|
|
@@ -1119,6 +3011,10 @@ static tf_op_entry builtin_ops[] = {
|
|
|
1119
3011
|
.kind = TF_OP_ENCODER,
|
|
1120
3012
|
.tier = TF_TIER_CORE,
|
|
1121
3013
|
.caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
|
|
3014
|
+
.memory_class = TF_MEM_BLOCKING,
|
|
3015
|
+
.emit_class = TF_EMIT_ON_FLUSH,
|
|
3016
|
+
.schema_class = TF_SCHEMA_STABLE,
|
|
3017
|
+
.state_estimate = "O(input_rows)",
|
|
1122
3018
|
.args = table_encode_args,
|
|
1123
3019
|
.n_args = 2,
|
|
1124
3020
|
.infer_schema = infer_schema_sink,
|