tranfi 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/LICENSE +177 -0
  2. package/NOTICE +8 -0
  3. package/README.md +272 -40
  4. package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
  5. package/app/index.html +1 -1
  6. package/binding.gyp +55 -3
  7. package/csrc/arena.c +7 -5
  8. package/csrc/batch.c +818 -71
  9. package/csrc/buffer.c +84 -8
  10. package/csrc/cJSON.c +262 -19
  11. package/csrc/cJSON.h +17 -1
  12. package/csrc/codec_csv.c +1074 -181
  13. package/csrc/codec_jsonl.c +830 -118
  14. package/csrc/codec_table.c +108 -78
  15. package/csrc/codec_text.c +286 -68
  16. package/csrc/compiler.c +31 -3
  17. package/csrc/config.h +21 -0
  18. package/csrc/dsl.c +4722 -485
  19. package/csrc/expr.c +363 -55
  20. package/csrc/expr.h +2 -0
  21. package/csrc/internal.h +316 -27
  22. package/csrc/ir.c +65 -18
  23. package/csrc/ir.h +41 -0
  24. package/csrc/ir_schema.c +20 -5
  25. package/csrc/ir_serialize.c +68 -6
  26. package/csrc/ir_sql.c +796 -185
  27. package/csrc/ir_validate.c +462 -6
  28. package/csrc/json_path.c +210 -0
  29. package/csrc/main.c +879 -30
  30. package/csrc/memory_estimate.c +477 -0
  31. package/csrc/op_acf.c +171 -21
  32. package/csrc/op_across.c +477 -0
  33. package/csrc/op_anomaly.c +167 -32
  34. package/csrc/op_assert.c +761 -0
  35. package/csrc/op_bin.c +168 -29
  36. package/csrc/op_cast.c +383 -55
  37. package/csrc/op_clip.c +30 -19
  38. package/csrc/op_date_trunc.c +208 -34
  39. package/csrc/op_datetime.c +259 -77
  40. package/csrc/op_derive.c +65 -97
  41. package/csrc/op_diff.c +146 -30
  42. package/csrc/op_ewma.c +149 -30
  43. package/csrc/op_explode.c +124 -26
  44. package/csrc/op_fill_down.c +125 -53
  45. package/csrc/op_fill_null.c +176 -31
  46. package/csrc/op_filter.c +89 -40
  47. package/csrc/op_frequency.c +571 -43
  48. package/csrc/op_grep.c +36 -18
  49. package/csrc/op_group_agg.c +1790 -119
  50. package/csrc/op_hash.c +48 -15
  51. package/csrc/op_head.c +21 -86
  52. package/csrc/op_interpolate.c +268 -62
  53. package/csrc/op_join.c +2700 -182
  54. package/csrc/op_json_extract.c +227 -0
  55. package/csrc/op_json_filter.c +384 -0
  56. package/csrc/op_json_flatten.c +293 -0
  57. package/csrc/op_json_schema.c +503 -0
  58. package/csrc/op_label_encode.c +328 -53
  59. package/csrc/op_lag.c +181 -0
  60. package/csrc/op_lead.c +141 -89
  61. package/csrc/op_normalize.c +363 -79
  62. package/csrc/op_onehot.c +345 -73
  63. package/csrc/op_pivot.c +1546 -162
  64. package/csrc/op_quarantine.c +189 -0
  65. package/csrc/op_registry.c +2062 -166
  66. package/csrc/op_rename.c +41 -50
  67. package/csrc/op_replace.c +270 -118
  68. package/csrc/op_rleid.c +297 -0
  69. package/csrc/op_rowid.c +559 -0
  70. package/csrc/op_sample.c +80 -23
  71. package/csrc/op_schema.c +1341 -0
  72. package/csrc/op_schema_infer.c +252 -0
  73. package/csrc/op_select.c +265 -65
  74. package/csrc/op_set.c +3449 -0
  75. package/csrc/op_skip.c +30 -87
  76. package/csrc/op_sort.c +670 -124
  77. package/csrc/op_source_name.c +120 -0
  78. package/csrc/op_split.c +65 -28
  79. package/csrc/op_split_data.c +41 -9
  80. package/csrc/op_stack.c +178 -222
  81. package/csrc/op_stats.c +206 -110
  82. package/csrc/op_step.c +217 -55
  83. package/csrc/op_tail.c +21 -12
  84. package/csrc/op_tee.c +338 -0
  85. package/csrc/op_top.c +260 -53
  86. package/csrc/op_trim.c +48 -19
  87. package/csrc/op_unique.c +1193 -150
  88. package/csrc/op_unpivot.c +100 -66
  89. package/csrc/op_validate.c +601 -24
  90. package/csrc/op_window.c +492 -51
  91. package/csrc/path_policy.c +85 -0
  92. package/csrc/pipeline.c +872 -99
  93. package/csrc/recipes.c +3 -1
  94. package/csrc/report.c +73 -30
  95. package/csrc/selector.c +1097 -0
  96. package/csrc/size_utils.c +348 -0
  97. package/csrc/spill.c +317 -0
  98. package/csrc/spill.h +21 -0
  99. package/csrc/tranfi.h +169 -1
  100. package/csrc/transform.h +209 -0
  101. package/csrc/transform_api.c +2237 -0
  102. package/csrc/transform_categorical.c +923 -0
  103. package/csrc/transform_internal.h +472 -0
  104. package/csrc/transform_json.c +3812 -0
  105. package/csrc/transform_numeric.c +1966 -0
  106. package/csrc/transform_sha256.c +154 -0
  107. package/csrc/transform_wasm.h +162 -0
  108. package/csrc/transform_wasm_api.c +1373 -0
  109. package/csrc/wasm_api.c +70 -9
  110. package/napi_api.c +219 -11
  111. package/napi_transform.c +1648 -0
  112. package/napi_transform.h +8 -0
  113. package/package.json +27 -11
  114. package/scripts/install-native.js +76 -0
  115. package/scripts/prepack.js +64 -0
  116. package/scripts/sync-csrc.js +23 -0
  117. package/src/cli.js +8 -11
  118. package/src/engines/duckdb.js +45 -12
  119. package/src/index.js +661 -42
  120. package/src/memory_policy.js +411 -0
  121. package/src/native.js +1 -5
  122. package/src/pipeline.js +454 -31
  123. package/src/recipe_json.js +80 -0
  124. package/src/server.js +10 -8
  125. package/src/transform.js +403 -0
  126. package/src/transform_error.js +10 -0
  127. package/src/wasm.js +6 -4
  128. package/wasm/index.js +498 -10
  129. package/wasm/tranfi_core.js +0 -0
  130. package/wasm/transform.js +1156 -0
  131. package/wasm/worker.js +786 -0
  132. package/csrc/plan.c +0 -206
@@ -1,7 +1,6 @@
1
1
  /*
2
2
  * op_registry.c — Declarative registry of all built-in ops.
3
3
  *
4
- * Replaces the hard-coded strcmp chain in plan.c.
5
4
  * Each entry describes an op's kind, capabilities, arguments,
6
5
  * schema inference callback, and native constructor.
7
6
  */
@@ -10,18 +9,97 @@
10
9
  #include "internal.h"
11
10
  #include "cJSON.h"
12
11
  #include <string.h>
12
+ #include <strings.h>
13
+ #include <stdio.h>
13
14
  #include <stdlib.h>
14
15
 
15
16
  /* ---- Schema inference callbacks ---- */
16
17
 
17
- /* Decoders: output schema unknown until runtime */
18
- static int infer_schema_unknown(const tf_ir_node *node,
19
- const tf_schema *in, tf_schema *out) {
20
- (void)node; (void)in;
18
+ static void registry_schema_unknown(tf_schema *out) {
19
+ if (!out) return;
20
+ tf_schema_free(out);
21
21
  out->col_names = NULL;
22
22
  out->col_types = NULL;
23
23
  out->n_cols = 0;
24
24
  out->known = false;
25
+ }
26
+
27
+ static int registry_schema_alloc_known(tf_schema *out, size_t n_cols) {
28
+ if (!out) return TF_ERROR;
29
+ tf_schema_free(out);
30
+ size_t alloc_n = n_cols ? n_cols : 1;
31
+ out->col_names = calloc(alloc_n, sizeof(char *));
32
+ out->col_types = calloc(alloc_n, sizeof(tf_type));
33
+ if (!out->col_names || !out->col_types) {
34
+ tf_schema_free(out);
35
+ return TF_ERROR;
36
+ }
37
+ out->n_cols = n_cols;
38
+ out->known = true;
39
+ return TF_OK;
40
+ }
41
+
42
+ static int registry_schema_set_col(tf_schema *out, size_t idx,
43
+ const char *name, tf_type type) {
44
+ if (!out || !out->known || idx >= out->n_cols || !name) return TF_ERROR;
45
+ char *copy = strdup(name);
46
+ if (!copy) return TF_ERROR;
47
+ free(out->col_names[idx]);
48
+ out->col_names[idx] = copy;
49
+ out->col_types[idx] = type;
50
+ return TF_OK;
51
+ }
52
+
53
+ static int registry_schema_take_col(tf_schema *out, size_t idx,
54
+ char *name, tf_type type) {
55
+ if (!out || !out->known || idx >= out->n_cols || !name) {
56
+ free(name);
57
+ return TF_ERROR;
58
+ }
59
+ free(out->col_names[idx]);
60
+ out->col_names[idx] = name;
61
+ out->col_types[idx] = type;
62
+ return TF_OK;
63
+ }
64
+
65
+ static void registry_set_schema_error(const char *op, const char *detail) {
66
+ char buf[512];
67
+ snprintf(buf, sizeof(buf), "%s schema: %s",
68
+ op ? op : "op", (detail && detail[0]) ? detail : "schema inference failed");
69
+ tf_set_last_error(buf);
70
+ }
71
+
72
+ static int registry_schema_copy_input_columns(tf_schema *out, const tf_schema *in) {
73
+ if (!out || !in || !out->known || !in->known || out->n_cols < in->n_cols) return TF_ERROR;
74
+ for (size_t i = 0; i < in->n_cols; i++) {
75
+ if (registry_schema_set_col(out, i, in->col_names[i] ? in->col_names[i] : "",
76
+ in->col_types[i]) != TF_OK) {
77
+ tf_schema_free(out);
78
+ return TF_ERROR;
79
+ }
80
+ }
81
+ return TF_OK;
82
+ }
83
+
84
+ static int registry_schema_copy_input_plus(tf_schema *out, const tf_schema *in,
85
+ size_t n_extra) {
86
+ if (!in || !in->known) {
87
+ registry_schema_unknown(out);
88
+ return TF_OK;
89
+ }
90
+ size_t total = 0;
91
+ if (tf_size_add(in->n_cols, n_extra, &total) != TF_OK ||
92
+ registry_schema_alloc_known(out, total) != TF_OK) {
93
+ return TF_ERROR;
94
+ }
95
+ return registry_schema_copy_input_columns(out, in);
96
+ }
97
+
98
+ /* Decoders: output schema unknown until runtime */
99
+ static int infer_schema_unknown(const tf_ir_node *node,
100
+ const tf_schema *in, tf_schema *out) {
101
+ (void)node; (void)in;
102
+ registry_schema_unknown(out);
25
103
  return TF_OK;
26
104
  }
27
105
 
@@ -29,10 +107,7 @@ static int infer_schema_unknown(const tf_ir_node *node,
29
107
  static int infer_schema_sink(const tf_ir_node *node,
30
108
  const tf_schema *in, tf_schema *out) {
31
109
  (void)node; (void)in;
32
- out->col_names = NULL;
33
- out->col_types = NULL;
34
- out->n_cols = 0;
35
- out->known = false;
110
+ registry_schema_unknown(out);
36
111
  return TF_OK;
37
112
  }
38
113
 
@@ -41,35 +116,53 @@ static int infer_schema_passthrough(const tf_ir_node *node,
41
116
  const tf_schema *in, tf_schema *out) {
42
117
  (void)node;
43
118
  if (!in->known) {
44
- out->known = false;
45
- out->col_names = NULL;
46
- out->col_types = NULL;
47
- out->n_cols = 0;
119
+ registry_schema_unknown(out);
48
120
  return TF_OK;
49
121
  }
50
- tf_schema_copy(out, in);
51
- return TF_OK;
122
+ return tf_schema_copy_checked(out, in);
52
123
  }
53
124
 
54
125
  /* select: output schema = subset of input columns */
55
126
  static int infer_schema_select(const tf_ir_node *node,
56
127
  const tf_schema *in, tf_schema *out) {
57
128
  if (!in->known) {
58
- out->known = false;
59
- out->col_names = NULL;
60
- out->col_types = NULL;
61
- out->n_cols = 0;
129
+ registry_schema_unknown(out);
62
130
  return TF_OK;
63
131
  }
64
132
 
65
133
  cJSON *cols = cJSON_GetObjectItemCaseSensitive(node->args, "columns");
66
134
  if (!cols || !cJSON_IsArray(cols)) return TF_ERROR;
67
135
 
136
+ if (tf_column_selectors_have_syntax_json(cols)) {
137
+ int *indices = NULL;
138
+ size_t n_indices = 0;
139
+ char *error = NULL;
140
+ int rc = tf_column_selectors_resolve_json(cols, in->col_names, in->col_types, in->n_cols,
141
+ &indices, &n_indices, &error);
142
+ if (rc != TF_OK) {
143
+ registry_set_schema_error("select", error);
144
+ free(error);
145
+ return TF_ERROR;
146
+ }
147
+ free(error);
148
+ if (registry_schema_alloc_known(out, n_indices) != TF_OK) {
149
+ free(indices);
150
+ return TF_ERROR;
151
+ }
152
+ for (size_t i = 0; i < n_indices; i++) {
153
+ int ci = indices[i];
154
+ if (registry_schema_set_col(out, i, in->col_names[ci], in->col_types[ci]) != TF_OK) {
155
+ free(indices);
156
+ tf_schema_free(out);
157
+ return TF_ERROR;
158
+ }
159
+ }
160
+ free(indices);
161
+ return TF_OK;
162
+ }
163
+
68
164
  int n = cJSON_GetArraySize(cols);
69
- out->col_names = calloc(n, sizeof(char *));
70
- out->col_types = calloc(n, sizeof(tf_type));
71
- out->n_cols = n;
72
- out->known = true;
165
+ if (n < 0 || registry_schema_alloc_known(out, (size_t)n) != TF_OK) return TF_ERROR;
73
166
 
74
167
  for (int i = 0; i < n; i++) {
75
168
  cJSON *item = cJSON_GetArrayItem(cols, i);
@@ -82,18 +175,125 @@ static int infer_schema_select(const tf_ir_node *node,
82
175
  bool found = false;
83
176
  for (size_t j = 0; j < in->n_cols; j++) {
84
177
  if (strcmp(in->col_names[j], name) == 0) {
85
- out->col_names[i] = strdup(name);
86
- out->col_types[i] = in->col_types[j];
178
+ if (registry_schema_set_col(out, (size_t)i, name, in->col_types[j]) != TF_OK) {
179
+ tf_schema_free(out);
180
+ return TF_ERROR;
181
+ }
87
182
  found = true;
88
183
  break;
89
184
  }
90
185
  }
91
186
  if (!found) {
92
187
  /* Column not in input — still record it, validation can catch it */
93
- out->col_names[i] = strdup(name);
94
- out->col_types[i] = TF_TYPE_NULL;
188
+ if (registry_schema_set_col(out, (size_t)i, name, TF_TYPE_NULL) != TF_OK) {
189
+ tf_schema_free(out);
190
+ return TF_ERROR;
191
+ }
192
+ }
193
+ }
194
+ return TF_OK;
195
+ }
196
+
197
+ static int infer_schema_relocate(const tf_ir_node *node,
198
+ const tf_schema *in, tf_schema *out) {
199
+ if (!in->known) {
200
+ registry_schema_unknown(out);
201
+ return TF_OK;
202
+ }
203
+
204
+ cJSON *cols = cJSON_GetObjectItemCaseSensitive(node->args, "columns");
205
+ if (!cols || !cJSON_IsArray(cols)) return TF_ERROR;
206
+ cJSON *before = cJSON_GetObjectItemCaseSensitive(node->args, "before");
207
+ cJSON *after = cJSON_GetObjectItemCaseSensitive(node->args, "after");
208
+ if (before && after) return TF_ERROR;
209
+ if (before && !cJSON_IsString(before)) return TF_ERROR;
210
+ if (after && !cJSON_IsString(after)) return TF_ERROR;
211
+
212
+ size_t n_in = in->n_cols;
213
+ int *move_idx = NULL;
214
+ size_t n_move = 0;
215
+ char *selector_error = NULL;
216
+ int rc = tf_column_selectors_resolve_json(cols, in->col_names, in->col_types, in->n_cols,
217
+ &move_idx, &n_move, &selector_error);
218
+ if (rc != TF_OK) {
219
+ registry_set_schema_error("relocate", selector_error);
220
+ free(selector_error);
221
+ return TF_ERROR;
222
+ }
223
+ free(selector_error);
224
+
225
+ int *is_moving = calloc(n_in ? n_in : 1, sizeof(int));
226
+ int *order = malloc(n_in ? n_in * sizeof(int) : sizeof(int));
227
+ if (!is_moving || !order) {
228
+ free(move_idx); free(is_moving); free(order);
229
+ return TF_ERROR;
230
+ }
231
+
232
+ for (size_t i = 0; i < n_move; i++) {
233
+ int found = move_idx[i];
234
+ if (found < 0 || (size_t)found >= n_in || is_moving[found]) {
235
+ free(move_idx); free(is_moving); free(order);
236
+ return TF_ERROR;
237
+ }
238
+ is_moving[found] = 1;
239
+ }
240
+
241
+ const char *anchor = before ? before->valuestring : (after ? after->valuestring : NULL);
242
+ int anchor_idx = -1;
243
+ if (anchor) {
244
+ for (size_t i = 0; i < n_in; i++) {
245
+ if (strcmp(in->col_names[i], anchor) == 0) {
246
+ anchor_idx = (int)i;
247
+ break;
248
+ }
249
+ }
250
+ if (anchor_idx < 0 || is_moving[anchor_idx]) {
251
+ free(move_idx); free(is_moving); free(order);
252
+ return TF_ERROR;
253
+ }
254
+ }
255
+
256
+ size_t n_order = 0;
257
+ if (!anchor) {
258
+ for (size_t i = 0; i < n_move; i++) order[n_order++] = move_idx[i];
259
+ for (size_t i = 0; i < n_in; i++)
260
+ if (!is_moving[i]) order[n_order++] = (int)i;
261
+ } else {
262
+ for (size_t i = 0; i < n_in; i++) {
263
+ if (is_moving[i]) continue;
264
+ if (before && (int)i == anchor_idx) {
265
+ for (size_t j = 0; j < n_move; j++) order[n_order++] = move_idx[j];
266
+ }
267
+ order[n_order++] = (int)i;
268
+ if (after && (int)i == anchor_idx) {
269
+ for (size_t j = 0; j < n_move; j++) order[n_order++] = move_idx[j];
270
+ }
271
+ }
272
+ }
273
+
274
+ if (n_order != n_in) {
275
+ free(move_idx); free(is_moving); free(order);
276
+ return TF_ERROR;
277
+ }
278
+
279
+ if (registry_schema_alloc_known(out, n_in) != TF_OK) {
280
+ free(move_idx); free(is_moving); free(order);
281
+ return TF_ERROR;
282
+ }
283
+ for (size_t i = 0; i < n_in; i++) {
284
+ int ci = order[i];
285
+ if (registry_schema_set_col(out, i, in->col_names[ci], in->col_types[ci]) != TF_OK) {
286
+ free(move_idx);
287
+ free(is_moving);
288
+ free(order);
289
+ tf_schema_free(out);
290
+ return TF_ERROR;
95
291
  }
96
292
  }
293
+
294
+ free(move_idx);
295
+ free(is_moving);
296
+ free(order);
97
297
  return TF_OK;
98
298
  }
99
299
 
@@ -101,14 +301,11 @@ static int infer_schema_select(const tf_ir_node *node,
101
301
  static int infer_schema_rename(const tf_ir_node *node,
102
302
  const tf_schema *in, tf_schema *out) {
103
303
  if (!in->known) {
104
- out->known = false;
105
- out->col_names = NULL;
106
- out->col_types = NULL;
107
- out->n_cols = 0;
304
+ registry_schema_unknown(out);
108
305
  return TF_OK;
109
306
  }
110
307
 
111
- tf_schema_copy(out, in);
308
+ if (tf_schema_copy_checked(out, in) != TF_OK) return TF_ERROR;
112
309
 
113
310
  cJSON *mapping = cJSON_GetObjectItemCaseSensitive(node->args, "mapping");
114
311
  if (!mapping || !cJSON_IsObject(mapping)) return TF_OK; /* no renames */
@@ -120,8 +317,10 @@ static int infer_schema_rename(const tf_ir_node *node,
120
317
  const char *new_name = entry->valuestring;
121
318
  for (size_t i = 0; i < out->n_cols; i++) {
122
319
  if (strcmp(out->col_names[i], old_name) == 0) {
123
- free(out->col_names[i]);
124
- out->col_names[i] = strdup(new_name);
320
+ if (registry_schema_set_col(out, i, new_name, out->col_types[i]) != TF_OK) {
321
+ tf_schema_free(out);
322
+ return TF_ERROR;
323
+ }
125
324
  break;
126
325
  }
127
326
  }
@@ -133,35 +332,177 @@ static int infer_schema_rename(const tf_ir_node *node,
133
332
  static int infer_schema_derive(const tf_ir_node *node,
134
333
  const tf_schema *in, tf_schema *out) {
135
334
  if (!in->known) {
136
- out->known = false;
137
- out->col_names = NULL;
138
- out->col_types = NULL;
139
- out->n_cols = 0;
335
+ registry_schema_unknown(out);
140
336
  return TF_OK;
141
337
  }
142
338
 
143
339
  cJSON *columns = cJSON_GetObjectItemCaseSensitive(node->args, "columns");
144
340
  int n_derived = columns ? cJSON_GetArraySize(columns) : 0;
145
- size_t total = in->n_cols + n_derived;
146
-
147
- out->col_names = calloc(total, sizeof(char *));
148
- out->col_types = calloc(total, sizeof(tf_type));
149
- out->n_cols = total;
150
- out->known = true;
151
-
152
- /* Copy input columns */
153
- for (size_t i = 0; i < in->n_cols; i++) {
154
- out->col_names[i] = strdup(in->col_names[i]);
155
- out->col_types[i] = in->col_types[i];
341
+ if (n_derived < 0 ||
342
+ registry_schema_copy_input_plus(out, in, (size_t)n_derived) != TF_OK) {
343
+ return TF_ERROR;
156
344
  }
157
345
 
158
346
  /* Add derived columns (type unknown at compile time) */
159
347
  for (int i = 0; i < n_derived; i++) {
160
348
  cJSON *item = cJSON_GetArrayItem(columns, i);
161
349
  cJSON *name_j = cJSON_GetObjectItemCaseSensitive(item, "name");
162
- out->col_names[in->n_cols + i] = strdup(name_j ? name_j->valuestring : "?");
163
- out->col_types[in->n_cols + i] = TF_TYPE_NULL;
350
+ const char *name = cJSON_IsString(name_j) ? name_j->valuestring : "?";
351
+ if (registry_schema_set_col(out, in->n_cols + (size_t)i, name, TF_TYPE_NULL) != TF_OK) {
352
+ tf_schema_free(out);
353
+ return TF_ERROR;
354
+ }
355
+ }
356
+ return TF_OK;
357
+ }
358
+
359
+
360
+ static int registry_across_token_matches(const char *p, const char *tok) {
361
+ return strncmp(p, tok, strlen(tok)) == 0;
362
+ }
363
+
364
+ static char *registry_across_format_name(const char *tmpl, const char *col, const char *fn) {
365
+ if (!tmpl || !tmpl[0]) tmpl = "{col}_{fn}";
366
+ if (!col) col = "";
367
+ if (!fn) fn = "";
368
+ size_t len = 0;
369
+ for (const char *p = tmpl; *p;) {
370
+ if (registry_across_token_matches(p, "{.col}")) { len += strlen(col); p += 6; }
371
+ else if (registry_across_token_matches(p, "{col}")) { len += strlen(col); p += 5; }
372
+ else if (registry_across_token_matches(p, "{.fn}")) { len += strlen(fn); p += 5; }
373
+ else if (registry_across_token_matches(p, "{fn}")) { len += strlen(fn); p += 4; }
374
+ else { len++; p++; }
375
+ }
376
+ char *out = malloc(len + 1);
377
+ if (!out) return NULL;
378
+ char *w = out;
379
+ for (const char *p = tmpl; *p;) {
380
+ const char *rep = NULL;
381
+ size_t tok_len = 0;
382
+ if (registry_across_token_matches(p, "{.col}")) { rep = col; tok_len = 6; }
383
+ else if (registry_across_token_matches(p, "{col}")) { rep = col; tok_len = 5; }
384
+ else if (registry_across_token_matches(p, "{.fn}")) { rep = fn; tok_len = 5; }
385
+ else if (registry_across_token_matches(p, "{fn}")) { rep = fn; tok_len = 4; }
386
+ if (rep) {
387
+ size_t n = strlen(rep);
388
+ memcpy(w, rep, n);
389
+ w += n;
390
+ p += tok_len;
391
+ } else {
392
+ *w++ = *p++;
393
+ }
394
+ }
395
+ *w = '\0';
396
+ return out;
397
+ }
398
+
399
+ static tf_type registry_across_output_type(const char *fn, tf_type input) {
400
+ if (!fn) return TF_TYPE_NULL;
401
+ if (strcasecmp(fn, "trim") == 0 || strcasecmp(fn, "lower") == 0 ||
402
+ strcasecmp(fn, "tolower") == 0 || strcasecmp(fn, "upper") == 0 ||
403
+ strcasecmp(fn, "toupper") == 0) {
404
+ return input == TF_TYPE_STRING ? TF_TYPE_STRING : TF_TYPE_NULL;
164
405
  }
406
+ if (!(input == TF_TYPE_INT64 || input == TF_TYPE_FLOAT64)) return TF_TYPE_NULL;
407
+ if (strcasecmp(fn, "abs") == 0) return input;
408
+ if (strcasecmp(fn, "round") == 0 || strcasecmp(fn, "floor") == 0 ||
409
+ strcasecmp(fn, "ceil") == 0 || strcasecmp(fn, "ceiling") == 0) return TF_TYPE_INT64;
410
+ if (strcasecmp(fn, "sqrt") == 0 || strcasecmp(fn, "log") == 0 ||
411
+ strcasecmp(fn, "exp") == 0) return TF_TYPE_FLOAT64;
412
+ return TF_TYPE_NULL;
413
+ }
414
+
415
+ static int registry_across_functions(const tf_ir_node *node, cJSON **out_fns, int *out_n) {
416
+ cJSON *functions = cJSON_GetObjectItemCaseSensitive(node->args, "functions");
417
+ if (!functions) functions = cJSON_GetObjectItemCaseSensitive(node->args, "fns");
418
+ cJSON *fn = cJSON_GetObjectItemCaseSensitive(node->args, "fn");
419
+ if (cJSON_IsArray(functions)) {
420
+ int n = cJSON_GetArraySize(functions);
421
+ if (n <= 0) return TF_ERROR;
422
+ *out_fns = functions;
423
+ *out_n = n;
424
+ return TF_OK;
425
+ }
426
+ if (cJSON_IsString(functions) || cJSON_IsString(fn)) {
427
+ *out_fns = cJSON_IsString(functions) ? functions : fn;
428
+ *out_n = 1;
429
+ return TF_OK;
430
+ }
431
+ return TF_ERROR;
432
+ }
433
+
434
+ /* across: input schema with selected columns replaced, or input + generated columns */
435
+ static int infer_schema_across(const tf_ir_node *node,
436
+ const tf_schema *in, tf_schema *out) {
437
+ if (!in->known) {
438
+ registry_schema_unknown(out);
439
+ return TF_OK;
440
+ }
441
+
442
+ cJSON *cols = cJSON_GetObjectItemCaseSensitive(node->args, "columns");
443
+ if (!cols || !cJSON_IsArray(cols)) return TF_ERROR;
444
+ int *indices = NULL;
445
+ size_t n_indices = 0;
446
+ char *error = NULL;
447
+ int rc = tf_column_selectors_resolve_json(cols, in->col_names, in->col_types, in->n_cols,
448
+ &indices, &n_indices, &error);
449
+ if (rc != TF_OK) {
450
+ registry_set_schema_error("across", error);
451
+ free(error);
452
+ return TF_ERROR;
453
+ }
454
+ free(error);
455
+
456
+ cJSON *fns = NULL;
457
+ int n_fns = 0;
458
+ if (registry_across_functions(node, &fns, &n_fns) != TF_OK) { free(indices); return TF_ERROR; }
459
+ cJSON *names = cJSON_GetObjectItemCaseSensitive(node->args, "names");
460
+ cJSON *replace_j = cJSON_GetObjectItemCaseSensitive(node->args, "replace");
461
+ int replace = cJSON_IsBool(replace_j) ? cJSON_IsTrue(replace_j) : (n_fns == 1 && !cJSON_IsString(names));
462
+ if (replace && n_fns != 1) { free(indices); return TF_ERROR; }
463
+ const char *tmpl = cJSON_IsString(names) ? names->valuestring : (replace ? "{col}" : "{col}_{fn}");
464
+
465
+ size_t extra = 0;
466
+ size_t total = 0;
467
+ if (!replace && tf_size_mul(n_indices, (size_t)n_fns, &extra) != TF_OK) {
468
+ free(indices);
469
+ return TF_ERROR;
470
+ }
471
+ if (tf_size_add(in->n_cols, extra, &total) != TF_OK ||
472
+ registry_schema_alloc_known(out, total) != TF_OK ||
473
+ registry_schema_copy_input_columns(out, in) != TF_OK) {
474
+ free(indices);
475
+ return TF_ERROR;
476
+ }
477
+
478
+ if (replace) {
479
+ const char *fn_name = cJSON_IsArray(fns) ? cJSON_GetArrayItem(fns, 0)->valuestring : fns->valuestring;
480
+ for (size_t i = 0; i < n_indices; i++) {
481
+ size_t ci = (size_t)indices[i];
482
+ tf_type t = registry_across_output_type(fn_name, in->col_types[ci]);
483
+ if (t == TF_TYPE_NULL) { free(indices); tf_schema_free(out); return TF_ERROR; }
484
+ out->col_types[ci] = t;
485
+ }
486
+ } else {
487
+ size_t a = 0;
488
+ for (size_t i = 0; i < n_indices; i++) {
489
+ size_t ci = (size_t)indices[i];
490
+ for (int f = 0; f < n_fns; f++) {
491
+ cJSON *fn_j = cJSON_IsArray(fns) ? cJSON_GetArrayItem(fns, f) : fns;
492
+ if (!cJSON_IsString(fn_j)) { free(indices); tf_schema_free(out); return TF_ERROR; }
493
+ tf_type t = registry_across_output_type(fn_j->valuestring, in->col_types[ci]);
494
+ if (t == TF_TYPE_NULL) { free(indices); tf_schema_free(out); return TF_ERROR; }
495
+ char *name = registry_across_format_name(tmpl, in->col_names[ci], fn_j->valuestring);
496
+ if (registry_schema_take_col(out, in->n_cols + a, name, t) != TF_OK) {
497
+ free(indices);
498
+ tf_schema_free(out);
499
+ return TF_ERROR;
500
+ }
501
+ a++;
502
+ }
503
+ }
504
+ }
505
+ free(indices);
165
506
  return TF_OK;
166
507
  }
167
508
 
@@ -169,17 +510,80 @@ static int infer_schema_derive(const tf_ir_node *node,
169
510
  static int infer_schema_validate(const tf_ir_node *node,
170
511
  const tf_schema *in, tf_schema *out) {
171
512
  (void)node;
172
- if (!in->known) { out->known = false; out->col_names = NULL; out->col_types = NULL; out->n_cols = 0; return TF_OK; }
173
- out->n_cols = in->n_cols + 1;
174
- out->col_names = calloc(out->n_cols, sizeof(char *));
175
- out->col_types = calloc(out->n_cols, sizeof(tf_type));
176
- out->known = true;
177
- for (size_t i = 0; i < in->n_cols; i++) {
178
- out->col_names[i] = strdup(in->col_names[i]);
179
- out->col_types[i] = in->col_types[i];
513
+ if (!in->known) { registry_schema_unknown(out); return TF_OK; }
514
+ if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
515
+ if (registry_schema_set_col(out, in->n_cols, "_valid", TF_TYPE_BOOL) != TF_OK) {
516
+ tf_schema_free(out);
517
+ return TF_ERROR;
518
+ }
519
+ return TF_OK;
520
+ }
521
+
522
+ static int infer_schema_assert(const tf_ir_node *node,
523
+ const tf_schema *in, tf_schema *out) {
524
+ cJSON *action_j = cJSON_GetObjectItemCaseSensitive(node->args, "action");
525
+ const char *action = cJSON_IsString(action_j) ? action_j->valuestring : "fail";
526
+ if (strcmp(action, "annotate") != 0) return infer_schema_passthrough(node, in, out);
527
+ if (!in->known) { registry_schema_unknown(out); return TF_OK; }
528
+ cJSON *result_j = cJSON_GetObjectItemCaseSensitive(node->args, "result");
529
+ const char *result = cJSON_IsString(result_j) && result_j->valuestring[0]
530
+ ? result_j->valuestring : "_assert";
531
+ if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
532
+ if (registry_schema_set_col(out, in->n_cols, result, TF_TYPE_BOOL) != TF_OK) {
533
+ tf_schema_free(out);
534
+ return TF_ERROR;
535
+ }
536
+ return TF_OK;
537
+ }
538
+
539
+ static int infer_schema_schema(const tf_ir_node *node,
540
+ const tf_schema *in, tf_schema *out) {
541
+ cJSON *action_j = cJSON_GetObjectItemCaseSensitive(node->args, "action");
542
+ cJSON *mode_j = cJSON_GetObjectItemCaseSensitive(node->args, "mode");
543
+ const char *action = cJSON_IsString(action_j) ? action_j->valuestring :
544
+ (cJSON_IsString(mode_j) ? mode_j->valuestring : "fail");
545
+ if (strcmp(action, "annotate") != 0) return infer_schema_passthrough(node, in, out);
546
+ if (!in->known) { registry_schema_unknown(out); return TF_OK; }
547
+ cJSON *result_j = cJSON_GetObjectItemCaseSensitive(node->args, "result");
548
+ const char *result = cJSON_IsString(result_j) && result_j->valuestring[0]
549
+ ? result_j->valuestring : "_schema";
550
+ if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
551
+ if (registry_schema_set_col(out, in->n_cols, result, TF_TYPE_BOOL) != TF_OK) {
552
+ tf_schema_free(out);
553
+ return TF_ERROR;
554
+ }
555
+ return TF_OK;
556
+ }
557
+
558
+ static int infer_schema_schema_infer(const tf_ir_node *node,
559
+ const tf_schema *in, tf_schema *out) {
560
+ (void)node; (void)in;
561
+ static const char *names[] = {
562
+ "column", "type", "nullable", "non_null", "rows_seen",
563
+ "rows_sampled", "missing", "non_missing", "observed_types", "warning"
564
+ };
565
+ static const tf_type types[] = {
566
+ TF_TYPE_STRING, TF_TYPE_STRING, TF_TYPE_BOOL, TF_TYPE_BOOL, TF_TYPE_INT64,
567
+ TF_TYPE_INT64, TF_TYPE_INT64, TF_TYPE_INT64, TF_TYPE_STRING, TF_TYPE_STRING
568
+ };
569
+ size_t n = sizeof(names) / sizeof(names[0]);
570
+ if (registry_schema_alloc_known(out, n) != TF_OK) return TF_ERROR;
571
+ for (size_t i = 0; i < n; i++) {
572
+ if (registry_schema_set_col(out, i, names[i], types[i]) != TF_OK) {
573
+ tf_schema_free(out);
574
+ return TF_ERROR;
575
+ }
576
+ }
577
+ return TF_OK;
578
+ }
579
+
580
+ static int infer_schema_add_int64_column(const tf_schema *in, tf_schema *out, const char *name) {
581
+ if (!in->known) { registry_schema_unknown(out); return TF_OK; }
582
+ if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
583
+ if (registry_schema_set_col(out, in->n_cols, name, TF_TYPE_INT64) != TF_OK) {
584
+ tf_schema_free(out);
585
+ return TF_ERROR;
180
586
  }
181
- out->col_names[in->n_cols] = strdup("_valid");
182
- out->col_types[in->n_cols] = TF_TYPE_BOOL;
183
587
  return TF_OK;
184
588
  }
185
589
 
@@ -187,17 +591,42 @@ static int infer_schema_validate(const tf_ir_node *node,
187
591
  static int infer_schema_add_hash(const tf_ir_node *node,
188
592
  const tf_schema *in, tf_schema *out) {
189
593
  (void)node;
190
- if (!in->known) { out->known = false; out->col_names = NULL; out->col_types = NULL; out->n_cols = 0; return TF_OK; }
191
- out->n_cols = in->n_cols + 1;
192
- out->col_names = calloc(out->n_cols, sizeof(char *));
193
- out->col_types = calloc(out->n_cols, sizeof(tf_type));
194
- out->known = true;
195
- for (size_t i = 0; i < in->n_cols; i++) {
196
- out->col_names[i] = strdup(in->col_names[i]);
197
- out->col_types[i] = in->col_types[i];
594
+ return infer_schema_add_int64_column(in, out, "_hash");
595
+ }
596
+
597
+ static int infer_schema_rleid(const tf_ir_node *node,
598
+ const tf_schema *in, tf_schema *out) {
599
+ const char *name = "_rleid";
600
+ cJSON *res = cJSON_GetObjectItemCaseSensitive(node->args, "result");
601
+ if (cJSON_IsString(res) && res->valuestring[0] != '\0') name = res->valuestring;
602
+ return infer_schema_add_int64_column(in, out, name);
603
+ }
604
+
605
+ static int infer_schema_rowid(const tf_ir_node *node,
606
+ const tf_schema *in, tf_schema *out) {
607
+ const char *name = "_rowid";
608
+ cJSON *res = cJSON_GetObjectItemCaseSensitive(node->args, "result");
609
+ if (cJSON_IsString(res) && res->valuestring[0] != '\0') name = res->valuestring;
610
+ return infer_schema_add_int64_column(in, out, name);
611
+ }
612
+
613
+ static int infer_schema_source_name(const tf_ir_node *node,
614
+ const tf_schema *in, tf_schema *out) {
615
+ if (!in->known) {
616
+ registry_schema_unknown(out);
617
+ return TF_OK;
618
+ }
619
+ const char *name = "_source";
620
+ cJSON *res = cJSON_GetObjectItemCaseSensitive(node->args, "result");
621
+ if (!cJSON_IsString(res)) res = cJSON_GetObjectItemCaseSensitive(node->args, "column");
622
+ if (!cJSON_IsString(res)) res = cJSON_GetObjectItemCaseSensitive(node->args, "as");
623
+ if (cJSON_IsString(res) && res->valuestring[0] != '\0') name = res->valuestring;
624
+
625
+ if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
626
+ if (registry_schema_set_col(out, in->n_cols, name, TF_TYPE_STRING) != TF_OK) {
627
+ tf_schema_free(out);
628
+ return TF_ERROR;
198
629
  }
199
- out->col_names[in->n_cols] = strdup("_hash");
200
- out->col_types[in->n_cols] = TF_TYPE_INT64;
201
630
  return TF_OK;
202
631
  }
203
632
 
@@ -205,74 +634,382 @@ static int infer_schema_add_hash(const tf_ir_node *node,
205
634
  static int infer_schema_frequency(const tf_ir_node *node,
206
635
  const tf_schema *in, tf_schema *out) {
207
636
  (void)node; (void)in;
208
- out->n_cols = 2;
209
- out->col_names = calloc(2, sizeof(char *));
210
- out->col_types = calloc(2, sizeof(tf_type));
211
- out->known = true;
212
- out->col_names[0] = strdup("value");
213
- out->col_types[0] = TF_TYPE_STRING;
214
- out->col_names[1] = strdup("count");
215
- out->col_types[1] = TF_TYPE_INT64;
637
+ if (registry_schema_alloc_known(out, 2) != TF_OK) return TF_ERROR;
638
+ if (registry_schema_set_col(out, 0, "value", TF_TYPE_STRING) != TF_OK ||
639
+ registry_schema_set_col(out, 1, "count", TF_TYPE_INT64) != TF_OK) {
640
+ tf_schema_free(out);
641
+ return TF_ERROR;
642
+ }
216
643
  return TF_OK;
217
644
  }
218
645
 
219
- /* group-agg: group columns (string) + agg columns (float64) */
646
+ static int schema_col_index(const tf_schema *schema, const char *name);
647
+
648
+ static int infer_schema_pivot(const tf_ir_node *node,
649
+ const tf_schema *in, tf_schema *out) {
650
+ if (!node || !node->args || !in || !in->known) {
651
+ registry_schema_unknown(out);
652
+ return TF_OK;
653
+ }
654
+
655
+ cJSON *cats = cJSON_GetObjectItemCaseSensitive(node->args, "categories");
656
+ if (!cats) {
657
+ registry_schema_unknown(out);
658
+ return TF_OK;
659
+ }
660
+ if (!cJSON_IsArray(cats)) {
661
+ registry_set_schema_error("pivot", "categories must be an array");
662
+ return TF_ERROR;
663
+ }
664
+
665
+ cJSON *name_j = cJSON_GetObjectItemCaseSensitive(node->args, "name_column");
666
+ cJSON *value_j = cJSON_GetObjectItemCaseSensitive(node->args, "value_column");
667
+ const char *name_col = cJSON_IsString(name_j) ? name_j->valuestring : NULL;
668
+ const char *value_col = cJSON_IsString(value_j) ? value_j->valuestring : NULL;
669
+ if (!name_col || !name_col[0] || !value_col || !value_col[0]) {
670
+ registry_set_schema_error("pivot", "missing name_column or value_column");
671
+ return TF_ERROR;
672
+ }
673
+
674
+ int name_idx = schema_col_index(in, name_col);
675
+ int value_idx = schema_col_index(in, value_col);
676
+ if (name_idx < 0 || value_idx < 0) {
677
+ registry_set_schema_error("pivot", "name_column or value_column not found");
678
+ return TF_ERROR;
679
+ }
680
+
681
+ int n_cats = cJSON_GetArraySize(cats);
682
+ if (n_cats <= 0) {
683
+ registry_set_schema_error("pivot", "categories cannot be empty");
684
+ return TF_ERROR;
685
+ }
686
+
687
+ size_t unique_cats = 0;
688
+ for (int i = 0; i < n_cats; i++) {
689
+ cJSON *cat = cJSON_GetArrayItem(cats, i);
690
+ if (!cJSON_IsString(cat) || !cat->valuestring || !cat->valuestring[0]) {
691
+ registry_set_schema_error("pivot", "categories must be non-empty strings");
692
+ return TF_ERROR;
693
+ }
694
+ int seen = 0;
695
+ for (int j = 0; j < i; j++) {
696
+ cJSON *prev = cJSON_GetArrayItem(cats, j);
697
+ if (cJSON_IsString(prev) && prev->valuestring &&
698
+ strcmp(prev->valuestring, cat->valuestring) == 0) {
699
+ seen = 1;
700
+ break;
701
+ }
702
+ }
703
+ if (!seen) {
704
+ size_t next_unique = 0;
705
+ if (tf_size_add(unique_cats, 1, &next_unique) != TF_OK) return TF_ERROR;
706
+ unique_cats = next_unique;
707
+ }
708
+ }
709
+
710
+ size_t n_pass = 0;
711
+ for (size_t c = 0; c < in->n_cols; c++) {
712
+ if ((int)c != name_idx && (int)c != value_idx) n_pass++;
713
+ }
714
+ size_t total = 0;
715
+ if (tf_size_add(n_pass, unique_cats, &total) != TF_OK ||
716
+ registry_schema_alloc_known(out, total) != TF_OK) {
717
+ return TF_ERROR;
718
+ }
719
+
720
+ size_t out_idx = 0;
721
+ for (size_t c = 0; c < in->n_cols; c++) {
722
+ if ((int)c == name_idx || (int)c == value_idx) continue;
723
+ if (registry_schema_set_col(out, out_idx++, in->col_names[c] ? in->col_names[c] : "",
724
+ in->col_types[c]) != TF_OK) {
725
+ tf_schema_free(out);
726
+ return TF_ERROR;
727
+ }
728
+ }
729
+
730
+ cJSON *agg_j = cJSON_GetObjectItemCaseSensitive(node->args, "agg");
731
+ const char *agg = cJSON_IsString(agg_j) ? agg_j->valuestring : NULL;
732
+ tf_type pivot_type = (agg && strcmp(agg, "count") == 0) ? TF_TYPE_INT64 : TF_TYPE_FLOAT64;
733
+ for (int i = 0; i < n_cats; i++) {
734
+ cJSON *cat = cJSON_GetArrayItem(cats, i);
735
+ int seen = 0;
736
+ for (int j = 0; j < i; j++) {
737
+ cJSON *prev = cJSON_GetArrayItem(cats, j);
738
+ if (cJSON_IsString(prev) && prev->valuestring &&
739
+ strcmp(prev->valuestring, cat->valuestring) == 0) {
740
+ seen = 1;
741
+ break;
742
+ }
743
+ }
744
+ if (seen) continue;
745
+ if (registry_schema_set_col(out, out_idx++, cat->valuestring, pivot_type) != TF_OK) {
746
+ tf_schema_free(out);
747
+ return TF_ERROR;
748
+ }
749
+ }
750
+ return TF_OK;
751
+ }
752
+
753
+ static int schema_col_index(const tf_schema *schema, const char *name) {
754
+ if (!schema || !schema->known || !name) return -1;
755
+ for (size_t i = 0; i < schema->n_cols; i++) {
756
+ if (schema->col_names[i] && strcmp(schema->col_names[i], name) == 0) return (int)i;
757
+ }
758
+ return -1;
759
+ }
760
+
761
+
762
+ static int infer_schema_append_source_column(const tf_ir_node *node,
763
+ const tf_schema *in,
764
+ tf_schema *out,
765
+ const char *default_suffix) {
766
+ cJSON *col_j = cJSON_GetObjectItemCaseSensitive(node->args, "column");
767
+ const char *column = cJSON_IsString(col_j) ? col_j->valuestring : NULL;
768
+ if (!in->known || !column) {
769
+ registry_schema_unknown(out);
770
+ return TF_OK;
771
+ }
772
+
773
+ int ci = schema_col_index(in, column);
774
+ if (ci < 0) {
775
+ registry_schema_unknown(out);
776
+ return TF_OK;
777
+ }
778
+
779
+ const char *name = NULL;
780
+ cJSON *res = cJSON_GetObjectItemCaseSensitive(node->args, "result");
781
+ if (cJSON_IsString(res) && res->valuestring[0] != '\0') name = res->valuestring;
782
+
783
+ char fallback[256];
784
+ if (!name) {
785
+ snprintf(fallback, sizeof(fallback), "%s_%s", column, default_suffix);
786
+ name = fallback;
787
+ }
788
+
789
+ if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
790
+ if (registry_schema_set_col(out, in->n_cols, name, in->col_types[ci]) != TF_OK) {
791
+ tf_schema_free(out);
792
+ return TF_ERROR;
793
+ }
794
+ return TF_OK;
795
+ }
796
+
797
+ static int infer_schema_lead(const tf_ir_node *node,
798
+ const tf_schema *in, tf_schema *out) {
799
+ return infer_schema_append_source_column(node, in, out, "lead");
800
+ }
801
+
802
+ static int infer_schema_lag(const tf_ir_node *node,
803
+ const tf_schema *in, tf_schema *out) {
804
+ return infer_schema_append_source_column(node, in, out, "lag");
805
+ }
806
+
807
+ static int infer_schema_shift(const tf_ir_node *node,
808
+ const tf_schema *in, tf_schema *out) {
809
+ const char *suffix = "shift";
810
+ cJSON *type = cJSON_GetObjectItemCaseSensitive(node->args, "type");
811
+ if (cJSON_IsString(type) && strcmp(type->valuestring, "lead") == 0) suffix = "lead";
812
+ return infer_schema_append_source_column(node, in, out, suffix);
813
+ }
814
+
815
+ static tf_type json_extract_type_from_arg(const cJSON *args) {
816
+ cJSON *type = cJSON_GetObjectItemCaseSensitive(args, "type");
817
+ const char *s = cJSON_IsString(type) ? type->valuestring : "string";
818
+ if (strcmp(s, "int") == 0 || strcmp(s, "int64") == 0) return TF_TYPE_INT64;
819
+ if (strcmp(s, "float") == 0 || strcmp(s, "float64") == 0 || strcmp(s, "number") == 0) return TF_TYPE_FLOAT64;
820
+ if (strcmp(s, "bool") == 0 || strcmp(s, "boolean") == 0) return TF_TYPE_BOOL;
821
+ return TF_TYPE_STRING;
822
+ }
823
+
824
+ static int infer_schema_json_extract(const tf_ir_node *node,
825
+ const tf_schema *in, tf_schema *out) {
826
+ if (!in->known) {
827
+ registry_schema_unknown(out);
828
+ return TF_OK;
829
+ }
830
+
831
+ cJSON *res = cJSON_GetObjectItemCaseSensitive(node->args, "result");
832
+ if (!cJSON_IsString(res) || res->valuestring[0] == '\0') return TF_ERROR;
833
+
834
+ if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
835
+ if (registry_schema_set_col(out, in->n_cols, res->valuestring,
836
+ json_extract_type_from_arg(node->args)) != TF_OK) {
837
+ tf_schema_free(out);
838
+ return TF_ERROR;
839
+ }
840
+ return TF_OK;
841
+ }
842
+
843
+ static int infer_schema_json_schema(const tf_ir_node *node,
844
+ const tf_schema *in, tf_schema *out) {
845
+ cJSON *mode_j = cJSON_GetObjectItemCaseSensitive(node->args, "mode");
846
+ const char *mode = cJSON_IsString(mode_j) ? mode_j->valuestring : "annotate";
847
+ if (strcmp(mode, "filter") == 0 || strcmp(mode, "keep") == 0) {
848
+ return infer_schema_passthrough(node, in, out);
849
+ }
850
+
851
+ if (!in->known) {
852
+ registry_schema_unknown(out);
853
+ return TF_OK;
854
+ }
855
+
856
+ const char *result = "_valid";
857
+ cJSON *res = cJSON_GetObjectItemCaseSensitive(node->args, "result");
858
+ if (cJSON_IsString(res) && res->valuestring[0] != '\0') result = res->valuestring;
859
+
860
+ if (registry_schema_copy_input_plus(out, in, 1) != TF_OK) return TF_ERROR;
861
+ if (registry_schema_set_col(out, in->n_cols, result, TF_TYPE_BOOL) != TF_OK) {
862
+ tf_schema_free(out);
863
+ return TF_ERROR;
864
+ }
865
+ return TF_OK;
866
+ }
867
+
868
+ static tf_type json_flatten_type_from_string(const char *type_s) {
869
+ if (!type_s || strcmp(type_s, "string") == 0 || strcmp(type_s, "str") == 0 || strcmp(type_s, "json") == 0) return TF_TYPE_STRING;
870
+ if (strcmp(type_s, "int") == 0 || strcmp(type_s, "int64") == 0) return TF_TYPE_INT64;
871
+ if (strcmp(type_s, "float") == 0 || strcmp(type_s, "float64") == 0 || strcmp(type_s, "number") == 0) return TF_TYPE_FLOAT64;
872
+ if (strcmp(type_s, "bool") == 0 || strcmp(type_s, "boolean") == 0) return TF_TYPE_BOOL;
873
+ return TF_TYPE_NULL;
874
+ }
875
+
876
+ static int json_flatten_parse_string_field(const char *spec, char **name, tf_type *type) {
877
+ if (!spec) return TF_ERROR;
878
+ char *copy = strdup(spec);
879
+ if (!copy) return TF_ERROR;
880
+ char *first = strchr(copy, ':');
881
+ if (!first) { free(copy); return TF_ERROR; }
882
+ *first = '\0';
883
+ char *second = strchr(first + 1, ':');
884
+ if (second) *second = '\0';
885
+ tf_type parsed = json_flatten_type_from_string(second ? second + 1 : "string");
886
+ if (first[1] == '\0' || parsed == TF_TYPE_NULL) {
887
+ free(copy);
888
+ return TF_ERROR;
889
+ }
890
+ *name = strdup(first + 1);
891
+ *type = parsed;
892
+ free(copy);
893
+ return *name ? TF_OK : TF_ERROR;
894
+ }
895
+
896
+ static int json_flatten_field_schema(const cJSON *item, char **name, tf_type *type) {
897
+ *name = NULL;
898
+ *type = TF_TYPE_STRING;
899
+ if (cJSON_IsString(item)) return json_flatten_parse_string_field(item->valuestring, name, type);
900
+ if (!cJSON_IsObject(item)) return TF_ERROR;
901
+ cJSON *name_j = cJSON_GetObjectItemCaseSensitive(item, "name");
902
+ if (!cJSON_IsString(name_j)) name_j = cJSON_GetObjectItemCaseSensitive(item, "result");
903
+ cJSON *type_j = cJSON_GetObjectItemCaseSensitive(item, "type");
904
+ tf_type parsed = json_flatten_type_from_string(cJSON_IsString(type_j) ? type_j->valuestring : "string");
905
+ if (!cJSON_IsString(name_j) || name_j->valuestring[0] == '\0' || parsed == TF_TYPE_NULL) return TF_ERROR;
906
+ *name = strdup(name_j->valuestring);
907
+ *type = parsed;
908
+ return *name ? TF_OK : TF_ERROR;
909
+ }
910
+
911
+ static int infer_schema_json_flatten(const tf_ir_node *node,
912
+ const tf_schema *in, tf_schema *out) {
913
+ if (!in->known) {
914
+ registry_schema_unknown(out);
915
+ return TF_OK;
916
+ }
917
+ cJSON *fields = cJSON_GetObjectItemCaseSensitive(node->args, "fields");
918
+ if (!cJSON_IsArray(fields) || cJSON_GetArraySize(fields) <= 0) return TF_ERROR;
919
+ int n_fields = cJSON_GetArraySize(fields);
920
+ if (registry_schema_copy_input_plus(out, in, (size_t)n_fields) != TF_OK) return TF_ERROR;
921
+ for (int i = 0; i < n_fields; i++) {
922
+ char *name = NULL;
923
+ tf_type type = TF_TYPE_STRING;
924
+ if (json_flatten_field_schema(cJSON_GetArrayItem(fields, i), &name, &type) != TF_OK) {
925
+ free(name);
926
+ tf_schema_free(out);
927
+ return TF_ERROR;
928
+ }
929
+ if (registry_schema_take_col(out, in->n_cols + (size_t)i, name, type) != TF_OK) {
930
+ tf_schema_free(out);
931
+ return TF_ERROR;
932
+ }
933
+ }
934
+ return TF_OK;
935
+ }
936
+
937
+
938
+ /* group-agg: typed group columns + aggregate columns (float64) */
220
939
  static int infer_schema_group_agg(const tf_ir_node *node,
221
940
  const tf_schema *in, tf_schema *out) {
222
- (void)in;
223
941
  cJSON *group_by = cJSON_GetObjectItemCaseSensitive(node->args, "group_by");
224
942
  cJSON *aggs = cJSON_GetObjectItemCaseSensitive(node->args, "aggs");
225
943
  int ng = group_by ? cJSON_GetArraySize(group_by) : 0;
226
944
  int na = aggs ? cJSON_GetArraySize(aggs) : 0;
227
- out->n_cols = ng + na;
228
- out->col_names = calloc(out->n_cols, sizeof(char *));
229
- out->col_types = calloc(out->n_cols, sizeof(tf_type));
230
- out->known = true;
945
+ if (ng < 0 || na < 0) return TF_ERROR;
946
+ size_t total = 0;
947
+ if (tf_size_add((size_t)ng, (size_t)na, &total) != TF_OK ||
948
+ registry_schema_alloc_known(out, total) != TF_OK) return TF_ERROR;
231
949
  for (int i = 0; i < ng; i++) {
232
950
  cJSON *item = cJSON_GetArrayItem(group_by, i);
233
- out->col_names[i] = strdup(cJSON_IsString(item) ? item->valuestring : "?");
234
- out->col_types[i] = TF_TYPE_STRING;
951
+ const char *name = cJSON_IsString(item) ? item->valuestring : "?";
952
+ int input_idx = schema_col_index(in, name);
953
+ tf_type type = input_idx >= 0 ? in->col_types[input_idx] : TF_TYPE_STRING;
954
+ if (registry_schema_set_col(out, (size_t)i, name, type) != TF_OK) {
955
+ tf_schema_free(out);
956
+ return TF_ERROR;
957
+ }
235
958
  }
236
959
  for (int i = 0; i < na; i++) {
237
960
  cJSON *item = cJSON_GetArrayItem(aggs, i);
238
961
  cJSON *name_j = cJSON_GetObjectItemCaseSensitive(item, "name");
239
- out->col_names[ng + i] = strdup(name_j && cJSON_IsString(name_j) ? name_j->valuestring : "?");
240
- out->col_types[ng + i] = TF_TYPE_FLOAT64;
962
+ const char *name = name_j && cJSON_IsString(name_j) ? name_j->valuestring : "?";
963
+ if (registry_schema_set_col(out, (size_t)ng + (size_t)i, name, TF_TYPE_FLOAT64) != TF_OK) {
964
+ tf_schema_free(out);
965
+ return TF_ERROR;
966
+ }
241
967
  }
242
968
  return TF_OK;
243
969
  }
244
970
 
245
- /* stats: output schema is known: column, count, sum, avg, min, max */
971
+ /* stats/scan: output schema is known from requested or profile-default stats */
246
972
  static int infer_schema_stats(const tf_ir_node *node,
247
973
  const tf_schema *in, tf_schema *out) {
248
974
  (void)in;
249
- /* Determine which stats are requested */
250
- int w_count=0, w_sum=0, w_avg=0, w_min=0, w_max=0;
251
- int w_var=0, w_stddev=0, w_median=0, w_p25=0, w_p75=0;
252
- int w_skewness=0, w_kurtosis=0, w_distinct=0, w_hist=0, w_sample=0;
253
- cJSON *stats_arr = cJSON_GetObjectItemCaseSensitive(node->args, "stats");
254
- if (stats_arr && cJSON_IsArray(stats_arr)) {
255
- int n = cJSON_GetArraySize(stats_arr);
975
+ int w_count = 0, w_missing = 0, w_complete_rate = 0;
976
+ int w_sum = 0, w_avg = 0, w_min = 0, w_max = 0;
977
+ int w_var = 0, w_stddev = 0;
978
+ int w_median = 0, w_p25 = 0, w_p75 = 0;
979
+ int w_skewness = 0, w_kurtosis = 0;
980
+ int w_distinct = 0, w_hist = 0, w_sample = 0;
981
+
982
+ cJSON *arr = node ? cJSON_GetObjectItemCaseSensitive(node->args, "stats") : NULL;
983
+ if (arr && cJSON_IsArray(arr)) {
984
+ int n = cJSON_GetArraySize(arr);
256
985
  for (int i = 0; i < n; i++) {
257
- cJSON *item = cJSON_GetArrayItem(stats_arr, i);
986
+ cJSON *item = cJSON_GetArrayItem(arr, i);
258
987
  if (!cJSON_IsString(item)) continue;
259
988
  const char *s = item->valuestring;
260
989
  if (strcmp(s, "count") == 0) w_count = 1;
990
+ else if (strcmp(s, "missing") == 0) w_missing = 1;
991
+ else if (strcmp(s, "complete_rate") == 0) w_complete_rate = 1;
261
992
  else if (strcmp(s, "sum") == 0) w_sum = 1;
262
- else if (strcmp(s, "avg") == 0) w_avg = 1;
993
+ else if (strcmp(s, "avg") == 0 || strcmp(s, "mean") == 0) w_avg = 1;
263
994
  else if (strcmp(s, "min") == 0) w_min = 1;
264
995
  else if (strcmp(s, "max") == 0) w_max = 1;
265
- else if (strcmp(s, "var") == 0) w_var = 1;
266
- else if (strcmp(s, "stddev") == 0) w_stddev = 1;
996
+ else if (strcmp(s, "var") == 0 || strcmp(s, "variance") == 0) w_var = 1;
997
+ else if (strcmp(s, "stddev") == 0 || strcmp(s, "sd") == 0) w_stddev = 1;
267
998
  else if (strcmp(s, "median") == 0) w_median = 1;
268
- else if (strcmp(s, "p25") == 0) w_p25 = 1;
269
- else if (strcmp(s, "p75") == 0) w_p75 = 1;
270
- else if (strcmp(s, "skewness") == 0) w_skewness = 1;
271
- else if (strcmp(s, "kurtosis") == 0) w_kurtosis = 1;
999
+ else if (strcmp(s, "p25") == 0 || strcmp(s, "q1") == 0) w_p25 = 1;
1000
+ else if (strcmp(s, "p75") == 0 || strcmp(s, "q3") == 0) w_p75 = 1;
1001
+ else if (strcmp(s, "skewness") == 0 || strcmp(s, "skew") == 0) w_skewness = 1;
1002
+ else if (strcmp(s, "kurtosis") == 0 || strcmp(s, "kurt") == 0) w_kurtosis = 1;
272
1003
  else if (strcmp(s, "distinct") == 0) w_distinct = 1;
273
1004
  else if (strcmp(s, "hist") == 0) w_hist = 1;
274
1005
  else if (strcmp(s, "sample") == 0) w_sample = 1;
275
1006
  }
1007
+ } else if (node && node->op && strcmp(node->op, "scan") == 0) {
1008
+ w_count = w_missing = w_complete_rate = 1;
1009
+ w_distinct = 1;
1010
+ w_min = w_max = w_avg = w_stddev = 1;
1011
+ w_p25 = w_median = w_p75 = 1;
1012
+ w_hist = w_sample = 1;
276
1013
  } else {
277
1014
  /* Default: basic + variance + median */
278
1015
  w_count = w_sum = w_avg = w_min = w_max = 1;
@@ -281,6 +1018,8 @@ static int infer_schema_stats(const tf_ir_node *node,
281
1018
 
282
1019
  size_t n_cols = 1; /* column */
283
1020
  if (w_count) n_cols++;
1021
+ if (w_missing) n_cols++;
1022
+ if (w_complete_rate) n_cols++;
284
1023
  if (w_sum) n_cols++;
285
1024
  if (w_avg) n_cols++;
286
1025
  if (w_min) n_cols++;
@@ -296,28 +1035,37 @@ static int infer_schema_stats(const tf_ir_node *node,
296
1035
  if (w_hist) n_cols++;
297
1036
  if (w_sample) n_cols++;
298
1037
 
299
- out->col_names = calloc(n_cols, sizeof(char *));
300
- out->col_types = calloc(n_cols, sizeof(tf_type));
301
- out->n_cols = n_cols;
302
- out->known = true;
1038
+ if (registry_schema_alloc_known(out, n_cols) != TF_OK) return TF_ERROR;
303
1039
 
304
1040
  size_t ci = 0;
305
- out->col_names[ci] = strdup("column"); out->col_types[ci++] = TF_TYPE_STRING;
306
- if (w_count) { out->col_names[ci] = strdup("count"); out->col_types[ci++] = TF_TYPE_INT64; }
307
- if (w_sum) { out->col_names[ci] = strdup("sum"); out->col_types[ci++] = TF_TYPE_FLOAT64; }
308
- if (w_avg) { out->col_names[ci] = strdup("avg"); out->col_types[ci++] = TF_TYPE_FLOAT64; }
309
- if (w_min) { out->col_names[ci] = strdup("min"); out->col_types[ci++] = TF_TYPE_FLOAT64; }
310
- if (w_max) { out->col_names[ci] = strdup("max"); out->col_types[ci++] = TF_TYPE_FLOAT64; }
311
- if (w_var) { out->col_names[ci] = strdup("var"); out->col_types[ci++] = TF_TYPE_FLOAT64; }
312
- if (w_stddev) { out->col_names[ci] = strdup("stddev"); out->col_types[ci++] = TF_TYPE_FLOAT64; }
313
- if (w_median) { out->col_names[ci] = strdup("median"); out->col_types[ci++] = TF_TYPE_FLOAT64; }
314
- if (w_p25) { out->col_names[ci] = strdup("p25"); out->col_types[ci++] = TF_TYPE_FLOAT64; }
315
- if (w_p75) { out->col_names[ci] = strdup("p75"); out->col_types[ci++] = TF_TYPE_FLOAT64; }
316
- if (w_skewness) { out->col_names[ci] = strdup("skewness"); out->col_types[ci++] = TF_TYPE_FLOAT64; }
317
- if (w_kurtosis) { out->col_names[ci] = strdup("kurtosis"); out->col_types[ci++] = TF_TYPE_FLOAT64; }
318
- if (w_distinct) { out->col_names[ci] = strdup("distinct"); out->col_types[ci++] = TF_TYPE_INT64; }
319
- if (w_hist) { out->col_names[ci] = strdup("hist"); out->col_types[ci++] = TF_TYPE_STRING; }
320
- if (w_sample) { out->col_names[ci] = strdup("sample"); out->col_types[ci++] = TF_TYPE_STRING; }
1041
+ #define ADD_STAT_COLUMN(name, type) do { \
1042
+ if (registry_schema_set_col(out, ci, (name), (type)) != TF_OK) { \
1043
+ tf_schema_free(out); \
1044
+ return TF_ERROR; \
1045
+ } \
1046
+ ci++; \
1047
+ } while (0)
1048
+
1049
+ ADD_STAT_COLUMN("column", TF_TYPE_STRING);
1050
+ if (w_count) ADD_STAT_COLUMN("count", TF_TYPE_INT64);
1051
+ if (w_missing) ADD_STAT_COLUMN("missing", TF_TYPE_INT64);
1052
+ if (w_complete_rate) ADD_STAT_COLUMN("complete_rate", TF_TYPE_FLOAT64);
1053
+ if (w_sum) ADD_STAT_COLUMN("sum", TF_TYPE_FLOAT64);
1054
+ if (w_avg) ADD_STAT_COLUMN("avg", TF_TYPE_FLOAT64);
1055
+ if (w_min) ADD_STAT_COLUMN("min", TF_TYPE_FLOAT64);
1056
+ if (w_max) ADD_STAT_COLUMN("max", TF_TYPE_FLOAT64);
1057
+ if (w_var) ADD_STAT_COLUMN("var", TF_TYPE_FLOAT64);
1058
+ if (w_stddev) ADD_STAT_COLUMN("stddev", TF_TYPE_FLOAT64);
1059
+ if (w_median) ADD_STAT_COLUMN("median", TF_TYPE_FLOAT64);
1060
+ if (w_p25) ADD_STAT_COLUMN("p25", TF_TYPE_FLOAT64);
1061
+ if (w_p75) ADD_STAT_COLUMN("p75", TF_TYPE_FLOAT64);
1062
+ if (w_skewness) ADD_STAT_COLUMN("skewness", TF_TYPE_FLOAT64);
1063
+ if (w_kurtosis) ADD_STAT_COLUMN("kurtosis", TF_TYPE_FLOAT64);
1064
+ if (w_distinct) ADD_STAT_COLUMN("distinct", TF_TYPE_INT64);
1065
+ if (w_hist) ADD_STAT_COLUMN("hist", TF_TYPE_STRING);
1066
+ if (w_sample) ADD_STAT_COLUMN("sample", TF_TYPE_STRING);
1067
+ #undef ADD_STAT_COLUMN
1068
+
321
1069
  return TF_OK;
322
1070
  }
323
1071
 
@@ -328,6 +1076,22 @@ static tf_arg_desc csv_decode_args[] = {
328
1076
  {"header", "bool", false, "true"},
329
1077
  {"batch_size","int", false, "1024"},
330
1078
  {"repair", "bool", false, "false"},
1079
+ {"mode", "string", false, "permissive"},
1080
+ {"strict", "bool", false, "false"},
1081
+ {"max_error_bytes", "int", false, "4096"},
1082
+ {"max_record_bytes", "int", false, "67108864"},
1083
+ {"max_columns", "int", false, "8192"},
1084
+ {"nulls", "string|array", false, """"},
1085
+ {"na", "string|array", false, """"},
1086
+ {"quoted_nulls", "bool", false, "true"},
1087
+ {"audit", "bool", false, "false"},
1088
+ {"audit_limit", "int", false, "1000"},
1089
+ {"audit_include_row", "bool", false, "true"},
1090
+ {"audit_columns", "string[]", false, NULL},
1091
+ {"audit_redact", "string[]", false, NULL},
1092
+ {"audit_hash_columns", "string[]", false, NULL},
1093
+ {"audit_max_bytes", "size", false, "0"},
1094
+ {"audit_max_cell_bytes", "size", false, "0"},
331
1095
  };
332
1096
 
333
1097
  static tf_arg_desc csv_encode_args[] = {
@@ -337,6 +1101,10 @@ static tf_arg_desc csv_encode_args[] = {
337
1101
 
338
1102
  static tf_arg_desc jsonl_decode_args[] = {
339
1103
  {"batch_size", "int", false, "1024"},
1104
+ {"on_error", "string", false, "skip"},
1105
+ {"max_error_bytes", "int", false, "4096"},
1106
+ {"max_record_bytes", "int", false, "67108864"},
1107
+ {"max_columns", "int", false, "8192"},
340
1108
  };
341
1109
 
342
1110
  static tf_arg_desc jsonl_encode_args[] = {
@@ -345,6 +1113,9 @@ static tf_arg_desc jsonl_encode_args[] = {
345
1113
 
346
1114
  static tf_arg_desc text_decode_args[] = {
347
1115
  {"batch_size", "int", false, "1024"},
1116
+ {"max_error_bytes", "int", false, "4096"},
1117
+ {"max_record_bytes", "int", false, "67108864"},
1118
+ {"max_columns", "int", false, "8192"},
348
1119
  };
349
1120
 
350
1121
  static tf_arg_desc text_encode_args[] = {
@@ -360,12 +1131,26 @@ static tf_arg_desc grep_args[] = {
360
1131
 
361
1132
  static tf_arg_desc filter_args[] = {
362
1133
  {"expr", "string", true, NULL},
1134
+ {"audit", "bool", false, "false"},
1135
+ {"audit_limit", "int", false, "1000"},
1136
+ {"audit_include_row", "bool", false, "true"},
1137
+ {"audit_columns", "string[]", false, NULL},
1138
+ {"audit_redact", "string[]", false, NULL},
1139
+ {"audit_hash_columns", "string[]", false, NULL},
1140
+ {"audit_max_bytes", "int", false, NULL},
1141
+ {"audit_max_cell_bytes", "int", false, NULL},
363
1142
  };
364
1143
 
365
1144
  static tf_arg_desc select_args[] = {
366
1145
  {"columns", "string[]", true, NULL},
367
1146
  };
368
1147
 
1148
+ static tf_arg_desc relocate_args[] = {
1149
+ {"columns", "string[]", true, NULL},
1150
+ {"before", "string", false, NULL},
1151
+ {"after", "string", false, NULL},
1152
+ };
1153
+
369
1154
  static tf_arg_desc rename_args[] = {
370
1155
  {"mapping", "map", true, NULL},
371
1156
  };
@@ -382,20 +1167,141 @@ static tf_arg_desc derive_args[] = {
382
1167
  {"columns", "map[]", true, NULL},
383
1168
  };
384
1169
 
1170
+
1171
+ static tf_arg_desc across_args[] = {
1172
+ {"columns", "string[]", true, NULL},
1173
+ {"fn", "string", false, NULL},
1174
+ {"functions", "string[]", false, NULL},
1175
+ {"names", "string", false, NULL},
1176
+ {"replace", "bool", false, NULL},
1177
+ };
1178
+
385
1179
  static tf_arg_desc stats_args[] = {
386
1180
  {"stats", "string[]", false, NULL},
387
1181
  };
388
1182
 
1183
+ static tf_arg_desc scan_args[] = {
1184
+ {"stats", "string[]", false, NULL},
1185
+ };
1186
+
389
1187
  static tf_arg_desc unique_args[] = {
390
1188
  {"columns", "string[]", false, NULL},
1189
+ {"max_keys", "int", false, NULL},
1190
+ {"max_state_bytes", "int", false, NULL},
1191
+ {"spill_dir", "string", false, NULL},
1192
+ {"spill_memory_bytes", "int", false, NULL},
1193
+ {"spill_run_rows", "int", false, NULL},
1194
+ {"spill_output_rows", "int", false, NULL},
1195
+ {"sorted", "bool", false, "false"},
1196
+ {"mode", "string", false, "\"exact\""},
1197
+ {"approx", "bool", false, "false"},
1198
+ {"bloom_bytes", "int", false, NULL},
1199
+ {"bloom_hashes", "int", false, "7"},
391
1200
  };
392
1201
 
393
1202
  static tf_arg_desc sort_args[] = {
394
1203
  {"columns", "map[]", true, NULL},
1204
+ {"spill_dir", "string", false, NULL},
1205
+ {"spill_memory_bytes", "int", false, NULL},
1206
+ {"spill_run_rows", "int", false, NULL},
1207
+ {"spill_output_rows", "int", false, NULL},
395
1208
  };
396
1209
 
397
1210
  static tf_arg_desc validate_args[] = {
1211
+ {"expr", "string", false, NULL},
1212
+ {"rules", "array|object", false, NULL},
1213
+ {"rules_file", "string", false, NULL},
1214
+ {"audit", "bool", false, "false"},
1215
+ {"audit_limit", "int", false, "1000"},
1216
+ {"audit_include_row", "bool", false, "true"},
1217
+ {"audit_columns", "string[]", false, NULL},
1218
+ {"audit_redact", "string[]", false, NULL},
1219
+ {"audit_hash_columns", "string[]", false, NULL},
1220
+ {"audit_max_bytes", "int", false, NULL},
1221
+ {"audit_max_cell_bytes", "int", false, NULL},
1222
+ {"max_failures", "int", false, NULL},
1223
+ {"max_failure_rate", "number", false, NULL},
1224
+ {"warn_failure_rate", "number", false, NULL},
1225
+ {"name", "string", false, "\"validate\""},
1226
+ {"message", "string", false, "\"\""},
1227
+ };
1228
+
1229
+ static tf_arg_desc assert_args[] = {
1230
+ {"expr", "string", false, NULL},
1231
+ {"aggregate", "string", false, NULL},
1232
+ {"column", "string", false, NULL},
1233
+ {"op", "string", false, NULL},
1234
+ {"value", "number", false, NULL},
1235
+ {"tolerance", "number", false, "1e-12"},
1236
+ {"rel", "bool", false, "true"},
1237
+ {"action", "string", false, "\"fail\""},
1238
+ {"name", "string", false, "\"assert\""},
1239
+ {"message", "string", false, "\"\""},
1240
+ {"result", "string", false, "\"_assert\""},
1241
+ {"audit", "bool", false, "false"},
1242
+ {"audit_limit", "int", false, "1000"},
1243
+ {"audit_include_row", "bool", false, "true"},
1244
+ {"audit_columns", "string[]", false, NULL},
1245
+ {"audit_redact", "string[]", false, NULL},
1246
+ {"audit_hash_columns", "string[]", false, NULL},
1247
+ {"audit_max_bytes", "int", false, NULL},
1248
+ {"audit_max_cell_bytes", "int", false, NULL},
1249
+ };
1250
+
1251
+ static tf_arg_desc quarantine_args[] = {
398
1252
  {"expr", "string", true, NULL},
1253
+ {"name", "string", false, "\"quarantine\""},
1254
+ {"message", "string", false, "\"\""},
1255
+ {"audit_include_row", "bool", false, "true"},
1256
+ {"audit_columns", "string[]", false, NULL},
1257
+ {"audit_redact", "string[]", false, NULL},
1258
+ {"audit_hash_columns", "string[]", false, NULL},
1259
+ {"audit_max_bytes", "size", false, "0"},
1260
+ {"audit_max_cell_bytes", "size", false, "0"},
1261
+ };
1262
+
1263
+ static tf_arg_desc tee_args[] = {
1264
+ {"expr", "string", false, NULL},
1265
+ {"channel", "string", false, "\"samples\""},
1266
+ {"columns", "string[]", false, NULL},
1267
+ {"limit", "int", false, "1000"},
1268
+ {"every", "int", false, "1"},
1269
+ {"name", "string", false, "\"tee\""},
1270
+ {"include_row", "bool", false, "true"},
1271
+ {"audit_include_row", "bool", false, "true"},
1272
+ {"audit_columns", "string[]", false, NULL},
1273
+ {"audit_redact", "string[]", false, NULL},
1274
+ {"audit_hash_columns", "string[]", false, NULL},
1275
+ {"audit_max_bytes", "size", false, "0"},
1276
+ {"audit_max_cell_bytes", "size", false, "0"},
1277
+ };
1278
+
1279
+ static tf_arg_desc schema_args[] = {
1280
+ {"columns", "map", false, NULL},
1281
+ {"required", "string[]", false, NULL},
1282
+ {"non_null", "string[]", false, NULL},
1283
+ {"nullable", "string[]", false, NULL},
1284
+ {"values", "map", false, NULL},
1285
+ {"min", "map", false, NULL},
1286
+ {"max", "map", false, NULL},
1287
+ {"regex", "map", false, NULL},
1288
+ {"max_regex_pattern_bytes", "int", false, "4096"},
1289
+ {"max_regex_cell_bytes", "int", false, "65536"},
1290
+ {"audit_include_row", "bool", false, "true"},
1291
+ {"audit_columns", "string[]", false, NULL},
1292
+ {"audit_redact", "string[]", false, NULL},
1293
+ {"audit_hash_columns", "string[]", false, NULL},
1294
+ {"audit_max_bytes", "int", false, NULL},
1295
+ {"audit_max_cell_bytes", "int", false, NULL},
1296
+ {"mode", "string", false, "\"fail\""},
1297
+ {"action", "string", false, "\"fail\""},
1298
+ {"name", "string", false, "\"schema\""},
1299
+ {"message", "string", false, "\"\""},
1300
+ {"result", "string", false, "\"_schema\""},
1301
+ };
1302
+
1303
+ static tf_arg_desc schema_infer_args[] = {
1304
+ {"rows", "int", false, "10000"},
399
1305
  };
400
1306
 
401
1307
  static tf_arg_desc trim_args[] = {
@@ -404,10 +1310,26 @@ static tf_arg_desc trim_args[] = {
404
1310
 
405
1311
  static tf_arg_desc fill_null_args[] = {
406
1312
  {"mapping", "map", true, NULL},
1313
+ {"audit", "bool", false, "false"},
1314
+ {"audit_limit", "int", false, "1000"},
1315
+ {"audit_include_row", "bool", false, "true"},
1316
+ {"audit_columns", "string[]", false, NULL},
1317
+ {"audit_redact", "string[]", false, NULL},
1318
+ {"audit_hash_columns", "string[]", false, NULL},
1319
+ {"audit_max_bytes", "size", false, "0"},
1320
+ {"audit_max_cell_bytes", "size", false, "0"},
407
1321
  };
408
1322
 
409
1323
  static tf_arg_desc cast_args[] = {
410
1324
  {"mapping", "map", true, NULL},
1325
+ {"audit", "bool", false, "false"},
1326
+ {"audit_limit", "int", false, "1000"},
1327
+ {"audit_include_row", "bool", false, "true"},
1328
+ {"audit_columns", "string[]", false, NULL},
1329
+ {"audit_redact", "string[]", false, NULL},
1330
+ {"audit_hash_columns", "string[]", false, NULL},
1331
+ {"audit_max_bytes", "size", false, "0"},
1332
+ {"audit_max_cell_bytes", "size", false, "0"},
411
1333
  };
412
1334
 
413
1335
  static tf_arg_desc clip_args[] = {
@@ -421,6 +1343,14 @@ static tf_arg_desc replace_args[] = {
421
1343
  {"pattern", "string", true, NULL},
422
1344
  {"replacement", "string", true, NULL},
423
1345
  {"regex", "bool", false, "false"},
1346
+ {"audit", "bool", false, "false"},
1347
+ {"audit_limit", "int", false, "1000"},
1348
+ {"audit_include_row", "bool", false, "true"},
1349
+ {"audit_columns", "string[]", false, NULL},
1350
+ {"audit_redact", "string[]", false, NULL},
1351
+ {"audit_hash_columns", "string[]", false, NULL},
1352
+ {"audit_max_bytes", "size", false, "0"},
1353
+ {"audit_max_cell_bytes", "size", false, "0"},
424
1354
  };
425
1355
 
426
1356
  static tf_arg_desc hash_args[] = {
@@ -430,6 +1360,8 @@ static tf_arg_desc hash_args[] = {
430
1360
  static tf_arg_desc bin_args[] = {
431
1361
  {"column", "string", true, NULL},
432
1362
  {"boundaries", "float[]", true, NULL},
1363
+ {"missing", "string", false, "\"error\""},
1364
+ {"on_type_error", "string", false, "\"fail\""},
433
1365
  };
434
1366
 
435
1367
  static tf_arg_desc fill_down_args[] = {
@@ -440,6 +1372,8 @@ static tf_arg_desc step_args[] = {
440
1372
  {"column", "string", true, NULL},
441
1373
  {"func", "string", true, NULL},
442
1374
  {"result", "string", false, NULL},
1375
+ {"missing", "string", false, "\"error\""},
1376
+ {"on_type_error", "string", false, "\"fail\""},
443
1377
  };
444
1378
 
445
1379
  static tf_arg_desc window_args[] = {
@@ -447,11 +1381,32 @@ static tf_arg_desc window_args[] = {
447
1381
  {"size", "int", true, NULL},
448
1382
  {"func", "string", true, NULL},
449
1383
  {"result", "string", false, NULL},
1384
+ {"missing", "string", false, "\"error\""},
1385
+ {"on_type_error", "string", false, "\"fail\""},
1386
+ };
1387
+
1388
+ static tf_arg_desc rolling_args[] = {
1389
+ {"column", "string", true, NULL},
1390
+ {"size", "int", true, NULL},
1391
+ {"result", "string", false, NULL},
1392
+ {"missing", "string", false, "\"error\""},
1393
+ {"on_type_error", "string", false, "\"fail\""},
1394
+ };
1395
+
1396
+ static tf_arg_desc rolling_bool_args[] = {
1397
+ {"column", "string", true, NULL},
1398
+ {"size", "int", true, NULL},
1399
+ {"result", "string", false, NULL},
1400
+ {"nulls", "string", false, "\"ignore\""},
450
1401
  };
451
1402
 
452
1403
  static tf_arg_desc explode_args[] = {
453
1404
  {"column", "string", true, NULL},
454
1405
  {"delimiter", "string", false, "\",\""},
1406
+ {"max_tokens_per_row", "int", false, NULL},
1407
+ {"max_output_rows_per_input_row", "int", false, NULL},
1408
+ {"max_output_rows_per_batch", "int", false, NULL},
1409
+ {"max_token_bytes", "int", false, NULL},
455
1410
  };
456
1411
 
457
1412
  static tf_arg_desc split_args[] = {
@@ -462,6 +1417,8 @@ static tf_arg_desc split_args[] = {
462
1417
 
463
1418
  static tf_arg_desc unpivot_args[] = {
464
1419
  {"columns", "string[]", true, NULL},
1420
+ {"max_output_rows_per_input_row", "int", false, NULL},
1421
+ {"max_output_rows_per_batch", "int", false, NULL},
465
1422
  };
466
1423
 
467
1424
  static tf_arg_desc tail_args[] = {
@@ -472,36 +1429,112 @@ static tf_arg_desc top_args[] = {
472
1429
  {"n", "int", true, NULL},
473
1430
  {"column", "string", true, NULL},
474
1431
  {"desc", "bool", false, "true"},
1432
+ {"with_ties", "bool", false, "false"},
475
1433
  };
476
1434
 
477
1435
  static tf_arg_desc sample_args[] = {
478
1436
  {"n", "int", true, NULL},
1437
+ {"seed", "int|string", false, "0"},
479
1438
  };
480
1439
 
481
1440
  static tf_arg_desc group_agg_args[] = {
482
1441
  {"group_by", "string[]", true, NULL},
483
1442
  {"aggs", "map[]", true, NULL},
1443
+ {"max_groups", "int", false, NULL},
1444
+ {"max_state_bytes", "int", false, NULL},
1445
+ {"spill_dir", "string", false, NULL},
1446
+ {"spill_memory_bytes", "int", false, NULL},
1447
+ {"spill_run_rows", "int", false, NULL},
1448
+ {"spill_output_rows", "int", false, NULL},
1449
+ {"sorted", "bool", false, "false"},
484
1450
  };
485
1451
 
486
1452
  static tf_arg_desc frequency_args[] = {
487
1453
  {"columns", "string[]", false, NULL},
1454
+ {"max_values", "int", false, NULL},
1455
+ {"max_state_bytes", "int", false, NULL},
1456
+ {"overflow", "string", false, "\"error\""},
1457
+ {"other", "string", false, "\"__other__\""},
1458
+ {"audit", "bool", false, "false"},
1459
+ {"audit_limit", "int", false, "1000"},
1460
+ {"audit_include_row", "bool", false, "true"},
1461
+ {"audit_columns", "string[]", false, NULL},
1462
+ {"audit_redact", "string[]", false, NULL},
1463
+ {"audit_hash_columns", "string[]", false, NULL},
1464
+ {"audit_max_bytes", "size", false, "0"},
1465
+ {"audit_max_cell_bytes", "size", false, "0"},
488
1466
  };
489
1467
 
490
1468
  static tf_arg_desc datetime_args[] = {
491
1469
  {"column", "string", true, NULL},
492
1470
  {"extract", "string[]", false, NULL},
1471
+ {"missing", "string", false, "\"error\""},
1472
+ {"on_type_error", "string", false, "\"fail\""},
493
1473
  };
494
1474
 
495
1475
  static tf_arg_desc pivot_args[] = {
496
1476
  {"name_column", "string", true, NULL},
497
1477
  {"value_column", "string", true, NULL},
498
1478
  {"agg", "string", false, "\"first\""},
1479
+ {"categories", "array", false, NULL},
1480
+ {"max_categories", "int", false, NULL},
1481
+ {"sorted", "bool", false, "false"},
1482
+ {"spill_dir", "string", false, NULL},
1483
+ {"spill_memory_bytes", "int", false, NULL},
1484
+ {"spill_run_rows", "int", false, NULL},
1485
+ {"spill_output_rows", "int", false, NULL},
499
1486
  };
500
1487
 
501
1488
  static tf_arg_desc join_args[] = {
502
1489
  {"file", "string", true, NULL},
503
1490
  {"on", "string", true, NULL},
504
1491
  {"how", "string", false, "\"inner\""},
1492
+ {"max_lookup_rows", "int", false, NULL},
1493
+ {"max_lookup_keys", "int", false, NULL},
1494
+ {"max_lookup_bytes", "int", false, NULL},
1495
+ {"max_state_bytes", "int", false, NULL},
1496
+ {"max_matches_per_row", "int", false, NULL},
1497
+ {"max_output_rows", "int", false, NULL},
1498
+ {"spill_dir", "string", false, NULL},
1499
+ {"spill_memory_bytes", "int", false, NULL},
1500
+ {"spill_run_rows", "int", false, NULL},
1501
+ {"spill_output_rows", "int", false, NULL},
1502
+ {"sorted", "bool", false, "false"},
1503
+ };
1504
+
1505
+ static tf_arg_desc set_file_args[] = {
1506
+ {"file", "string", true, NULL},
1507
+ {"columns", "string[]", false, NULL},
1508
+ {"max_lookup_rows", "int", false, NULL},
1509
+ {"max_lookup_keys", "int", false, NULL},
1510
+ {"max_lookup_bytes", "int", false, NULL},
1511
+ {"max_output_keys", "int", false, NULL},
1512
+ {"max_state_bytes", "int", false, NULL},
1513
+ {"spill_dir", "string", false, NULL},
1514
+ {"spill_memory_bytes", "int", false, NULL},
1515
+ {"spill_run_rows", "int", false, NULL},
1516
+ {"spill_output_rows", "int", false, NULL},
1517
+ {"sorted", "bool", false, "false"},
1518
+ };
1519
+
1520
+ static tf_arg_desc union_args[] = {
1521
+ {"file", "string", true, NULL},
1522
+ {"columns", "string[]", false, NULL},
1523
+ {"max_lookup_rows", "int", false, NULL},
1524
+ {"max_lookup_bytes", "int", false, NULL},
1525
+ {"max_output_keys", "int", false, NULL},
1526
+ {"max_state_bytes", "int", false, NULL},
1527
+ {"spill_dir", "string", false, NULL},
1528
+ {"spill_memory_bytes", "int", false, NULL},
1529
+ {"spill_run_rows", "int", false, NULL},
1530
+ {"spill_output_rows", "int", false, NULL},
1531
+ {"sorted", "bool", false, "false"},
1532
+ };
1533
+
1534
+ static tf_arg_desc union_all_args[] = {
1535
+ {"file", "string", true, NULL},
1536
+ {"max_lookup_rows", "int", false, NULL},
1537
+ {"max_lookup_bytes", "int", false, NULL},
505
1538
  };
506
1539
 
507
1540
  static tf_arg_desc stack_args[] = {
@@ -516,38 +1549,80 @@ static tf_arg_desc lead_args[] = {
516
1549
  {"result", "string", false, NULL},
517
1550
  };
518
1551
 
1552
+ static tf_arg_desc lag_args[] = {
1553
+ {"column", "string", true, NULL},
1554
+ {"offset", "int", false, "1"},
1555
+ {"result", "string", false, NULL},
1556
+ };
1557
+
1558
+ static tf_arg_desc shift_args[] = {
1559
+ {"column", "string", true, NULL},
1560
+ {"offset", "int", false, "1"},
1561
+ {"result", "string", false, NULL},
1562
+ {"type", "string", false, "\"lag\""},
1563
+ };
1564
+
1565
+ static tf_arg_desc rowid_args[] = {
1566
+ {"columns", "string[]", false, NULL},
1567
+ {"result", "string", false, "\"_rowid\""},
1568
+ {"sorted", "bool", false, "false"},
1569
+ {"max_keys", "int", false, NULL},
1570
+ {"max_state_bytes", "int", false, NULL},
1571
+ };
1572
+
1573
+ static tf_arg_desc rleid_args[] = {
1574
+ {"columns", "string[]", true, NULL},
1575
+ {"result", "string", false, "\"_rleid\""},
1576
+ };
1577
+
519
1578
  static tf_arg_desc date_trunc_args[] = {
520
1579
  {"column", "string", true, NULL},
521
1580
  {"trunc", "string", true, NULL},
522
1581
  {"result", "string", false, NULL},
1582
+ {"missing", "string", false, "\"error\""},
1583
+ {"on_type_error", "string", false, "\"fail\""},
523
1584
  };
524
1585
 
525
1586
  static tf_arg_desc onehot_args[] = {
526
1587
  {"column", "string", true, NULL},
527
1588
  {"drop", "bool", false, "false"},
1589
+ {"categories", "array", false, NULL},
1590
+ {"max_categories", "int", false, NULL},
1591
+ {"max_state_bytes", "int", false, NULL},
1592
+ {"unknown", "string", false, NULL},
528
1593
  };
529
1594
 
530
1595
  static tf_arg_desc label_encode_args[] = {
531
1596
  {"column", "string", true, NULL},
532
1597
  {"result", "string", false, NULL},
1598
+ {"categories", "array", false, NULL},
1599
+ {"max_categories", "int", false, NULL},
1600
+ {"max_state_bytes", "int", false, NULL},
1601
+ {"unknown", "string", false, NULL},
533
1602
  };
534
1603
 
535
1604
  static tf_arg_desc ewma_args[] = {
536
1605
  {"column", "string", true, NULL},
537
1606
  {"alpha", "float", true, NULL},
538
1607
  {"result", "string", false, NULL},
1608
+ {"missing", "string", false, "\"error\""},
1609
+ {"on_type_error", "string", false, "\"fail\""},
539
1610
  };
540
1611
 
541
1612
  static tf_arg_desc diff_args[] = {
542
1613
  {"column", "string", true, NULL},
543
1614
  {"order", "int", false, "1"},
544
1615
  {"result", "string", false, NULL},
1616
+ {"missing", "string", false, "\"error\""},
1617
+ {"on_type_error", "string", false, "\"fail\""},
545
1618
  };
546
1619
 
547
1620
  static tf_arg_desc anomaly_args[] = {
548
1621
  {"column", "string", true, NULL},
549
1622
  {"threshold", "float", false, "3.0"},
550
1623
  {"result", "string", false, NULL},
1624
+ {"missing", "string", false, "\"error\""},
1625
+ {"on_type_error", "string", false, "\"fail\""},
551
1626
  };
552
1627
 
553
1628
  static tf_arg_desc split_data_args[] = {
@@ -556,19 +1631,73 @@ static tf_arg_desc split_data_args[] = {
556
1631
  {"seed", "int", false, "42"},
557
1632
  };
558
1633
 
1634
+ static tf_arg_desc source_name_args[] = {
1635
+ {"result", "string", false, "\"_source\""},
1636
+ {"default", "string", false, "\"\""},
1637
+ };
1638
+
559
1639
  static tf_arg_desc interpolate_args[] = {
560
1640
  {"column", "string", true, NULL},
561
1641
  {"method", "string", false, "\"linear\""},
1642
+ {"missing", "string", false, "\"error\""},
1643
+ {"on_type_error", "string", false, "\"fail\""},
562
1644
  };
563
1645
 
564
1646
  static tf_arg_desc normalize_args[] = {
565
1647
  {"columns", "string[]", true, NULL},
566
1648
  {"method", "string", false, "\"minmax\""},
1649
+ {"missing", "string", false, "\"error\""},
1650
+ {"on_type_error", "string", false, "\"fail\""},
1651
+ {"audit", "bool", false, "false"},
1652
+ {"audit_limit", "int", false, "1000"},
1653
+ {"audit_include_row", "bool", false, "true"},
1654
+ {"audit_columns", "string[]", false, NULL},
1655
+ {"audit_redact", "string[]", false, NULL},
1656
+ {"audit_hash_columns", "string[]", false, NULL},
1657
+ {"audit_max_bytes", "size", false, "0"},
1658
+ {"audit_max_cell_bytes", "size", false, "0"},
567
1659
  };
568
1660
 
569
1661
  static tf_arg_desc acf_args[] = {
570
1662
  {"column", "string", true, NULL},
571
1663
  {"lags", "int", false, "20"},
1664
+ {"missing", "string", false, "\"error\""},
1665
+ {"on_type_error", "string", false, "\"fail\""},
1666
+ };
1667
+
1668
+ static tf_arg_desc json_extract_args[] = {
1669
+ {"column", "string", false, "\"_line\""},
1670
+ {"path", "string", true, NULL},
1671
+ {"result", "string", true, NULL},
1672
+ {"type", "string", false, "\"string\""},
1673
+ };
1674
+
1675
+ static tf_arg_desc json_filter_args[] = {
1676
+ {"column", "string", false, "\"_line\""},
1677
+ {"path", "string", true, NULL},
1678
+ {"op", "string", false, "\"exists\""},
1679
+ {"value", "string", false, NULL},
1680
+ {"type", "string", false, "\"auto\""},
1681
+ };
1682
+
1683
+ static tf_arg_desc json_schema_args[] = {
1684
+ {"column", "string", false, "\"_line\""},
1685
+ {"schema", "object|string|bool", true, NULL},
1686
+ {"mode", "string", false, "\"annotate\""},
1687
+ {"result", "string", false, "\"_valid\""},
1688
+ {"audit", "bool", false, "false"},
1689
+ {"audit_limit", "int", false, "1000"},
1690
+ {"audit_include_row", "bool", false, "true"},
1691
+ {"audit_columns", "string[]", false, NULL},
1692
+ {"audit_redact", "string[]", false, NULL},
1693
+ {"audit_hash_columns", "string[]", false, NULL},
1694
+ {"audit_max_bytes", "int", false, NULL},
1695
+ {"audit_max_cell_bytes", "int", false, NULL},
1696
+ };
1697
+
1698
+ static tf_arg_desc json_flatten_args[] = {
1699
+ {"column", "string", false, "\"_line\""},
1700
+ {"fields", "object[]|string[]", true, NULL},
572
1701
  };
573
1702
 
574
1703
  static tf_arg_desc table_encode_args[] = {
@@ -576,14 +1705,49 @@ static tf_arg_desc table_encode_args[] = {
576
1705
  {"max_rows", "int", false, "0"},
577
1706
  };
578
1707
 
1708
+
1709
+ const char *tf_memory_class_name(tf_memory_class cls) {
1710
+ switch (cls) {
1711
+ case TF_MEM_ROW_LOCAL: return "row_local";
1712
+ case TF_MEM_BOUNDED_STATE: return "bounded_state";
1713
+ case TF_MEM_KEY_STATE: return "key_state";
1714
+ case TF_MEM_BLOCKING: return "blocking";
1715
+ case TF_MEM_EXTERNAL: return "external";
1716
+ default: return "unknown";
1717
+ }
1718
+ }
1719
+
1720
+ const char *tf_emit_class_name(tf_emit_class cls) {
1721
+ switch (cls) {
1722
+ case TF_EMIT_PER_BATCH: return "per_batch";
1723
+ case TF_EMIT_ON_FLUSH: return "on_flush";
1724
+ case TF_EMIT_SIDE_ONLY: return "side_only";
1725
+ case TF_EMIT_MIXED: return "mixed";
1726
+ default: return "unknown";
1727
+ }
1728
+ }
1729
+
1730
+ const char *tf_schema_class_name(tf_schema_class cls) {
1731
+ switch (cls) {
1732
+ case TF_SCHEMA_STABLE: return "stable";
1733
+ case TF_SCHEMA_PARAMETRIC: return "parametric";
1734
+ case TF_SCHEMA_DATA_DEPENDENT: return "data_dependent";
1735
+ default: return "unknown";
1736
+ }
1737
+ }
1738
+
579
1739
  static tf_op_entry builtin_ops[] = {
580
1740
  {
581
1741
  .name = "codec.csv.decode",
582
1742
  .kind = TF_OP_DECODER,
583
1743
  .tier = TF_TIER_CORE,
584
1744
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1745
+ .memory_class = TF_MEM_BOUNDED_STATE,
1746
+ .emit_class = TF_EMIT_PER_BATCH,
1747
+ .schema_class = TF_SCHEMA_DATA_DEPENDENT,
1748
+ .state_estimate = "O(batch_size * columns)",
585
1749
  .args = csv_decode_args,
586
- .n_args = 4,
1750
+ .n_args = 20,
587
1751
  .infer_schema = infer_schema_unknown,
588
1752
  .create_native = (void *(*)(const cJSON *))tf_csv_decoder_create,
589
1753
  },
@@ -592,6 +1756,10 @@ static tf_op_entry builtin_ops[] = {
592
1756
  .kind = TF_OP_ENCODER,
593
1757
  .tier = TF_TIER_CORE,
594
1758
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1759
+ .memory_class = TF_MEM_BOUNDED_STATE,
1760
+ .emit_class = TF_EMIT_PER_BATCH,
1761
+ .schema_class = TF_SCHEMA_STABLE,
1762
+ .state_estimate = "O(output_chunk)",
595
1763
  .args = csv_encode_args,
596
1764
  .n_args = 2,
597
1765
  .infer_schema = infer_schema_sink,
@@ -602,8 +1770,12 @@ static tf_op_entry builtin_ops[] = {
602
1770
  .kind = TF_OP_DECODER,
603
1771
  .tier = TF_TIER_CORE,
604
1772
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1773
+ .memory_class = TF_MEM_BOUNDED_STATE,
1774
+ .emit_class = TF_EMIT_PER_BATCH,
1775
+ .schema_class = TF_SCHEMA_DATA_DEPENDENT,
1776
+ .state_estimate = "O(batch_size)",
605
1777
  .args = jsonl_decode_args,
606
- .n_args = 1,
1778
+ .n_args = 3,
607
1779
  .infer_schema = infer_schema_unknown,
608
1780
  .create_native = (void *(*)(const cJSON *))tf_jsonl_decoder_create,
609
1781
  },
@@ -612,6 +1784,10 @@ static tf_op_entry builtin_ops[] = {
612
1784
  .kind = TF_OP_ENCODER,
613
1785
  .tier = TF_TIER_CORE,
614
1786
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1787
+ .memory_class = TF_MEM_BOUNDED_STATE,
1788
+ .emit_class = TF_EMIT_PER_BATCH,
1789
+ .schema_class = TF_SCHEMA_STABLE,
1790
+ .state_estimate = "O(output_chunk)",
615
1791
  .args = jsonl_encode_args,
616
1792
  .n_args = 0,
617
1793
  .infer_schema = infer_schema_sink,
@@ -622,6 +1798,10 @@ static tf_op_entry builtin_ops[] = {
622
1798
  .kind = TF_OP_DECODER,
623
1799
  .tier = TF_TIER_CORE,
624
1800
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1801
+ .memory_class = TF_MEM_BOUNDED_STATE,
1802
+ .emit_class = TF_EMIT_PER_BATCH,
1803
+ .schema_class = TF_SCHEMA_PARAMETRIC,
1804
+ .state_estimate = "O(batch_size)",
625
1805
  .args = text_decode_args,
626
1806
  .n_args = 1,
627
1807
  .infer_schema = infer_schema_unknown,
@@ -632,6 +1812,10 @@ static tf_op_entry builtin_ops[] = {
632
1812
  .kind = TF_OP_ENCODER,
633
1813
  .tier = TF_TIER_CORE,
634
1814
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1815
+ .memory_class = TF_MEM_BOUNDED_STATE,
1816
+ .emit_class = TF_EMIT_PER_BATCH,
1817
+ .schema_class = TF_SCHEMA_STABLE,
1818
+ .state_estimate = "O(output_chunk)",
635
1819
  .args = text_encode_args,
636
1820
  .n_args = 0,
637
1821
  .infer_schema = infer_schema_sink,
@@ -642,6 +1826,10 @@ static tf_op_entry builtin_ops[] = {
642
1826
  .kind = TF_OP_TRANSFORM,
643
1827
  .tier = TF_TIER_CORE,
644
1828
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1829
+ .memory_class = TF_MEM_ROW_LOCAL,
1830
+ .emit_class = TF_EMIT_PER_BATCH,
1831
+ .schema_class = TF_SCHEMA_STABLE,
1832
+ .state_estimate = "O(batch_rows)",
645
1833
  .args = grep_args,
646
1834
  .n_args = 4,
647
1835
  .infer_schema = infer_schema_passthrough,
@@ -652,8 +1840,12 @@ static tf_op_entry builtin_ops[] = {
652
1840
  .kind = TF_OP_TRANSFORM,
653
1841
  .tier = TF_TIER_CORE,
654
1842
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1843
+ .memory_class = TF_MEM_ROW_LOCAL,
1844
+ .emit_class = TF_EMIT_PER_BATCH,
1845
+ .schema_class = TF_SCHEMA_STABLE,
1846
+ .state_estimate = "O(batch_rows * columns)",
655
1847
  .args = filter_args,
656
- .n_args = 1,
1848
+ .n_args = 9,
657
1849
  .infer_schema = infer_schema_passthrough,
658
1850
  .create_native = (void *(*)(const cJSON *))tf_filter_create,
659
1851
  },
@@ -662,6 +1854,10 @@ static tf_op_entry builtin_ops[] = {
662
1854
  .kind = TF_OP_TRANSFORM,
663
1855
  .tier = TF_TIER_CORE,
664
1856
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1857
+ .memory_class = TF_MEM_ROW_LOCAL,
1858
+ .emit_class = TF_EMIT_PER_BATCH,
1859
+ .schema_class = TF_SCHEMA_PARAMETRIC,
1860
+ .state_estimate = "O(batch_rows * selected_columns)",
665
1861
  .args = select_args,
666
1862
  .n_args = 1,
667
1863
  .infer_schema = infer_schema_select,
@@ -672,6 +1868,10 @@ static tf_op_entry builtin_ops[] = {
672
1868
  .kind = TF_OP_TRANSFORM,
673
1869
  .tier = TF_TIER_CORE,
674
1870
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1871
+ .memory_class = TF_MEM_ROW_LOCAL,
1872
+ .emit_class = TF_EMIT_PER_BATCH,
1873
+ .schema_class = TF_SCHEMA_PARAMETRIC,
1874
+ .state_estimate = "O(columns)",
675
1875
  .args = rename_args,
676
1876
  .n_args = 1,
677
1877
  .infer_schema = infer_schema_rename,
@@ -682,48 +1882,126 @@ static tf_op_entry builtin_ops[] = {
682
1882
  .kind = TF_OP_TRANSFORM,
683
1883
  .tier = TF_TIER_CORE,
684
1884
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1885
+ .memory_class = TF_MEM_BOUNDED_STATE,
1886
+ .emit_class = TF_EMIT_PER_BATCH,
1887
+ .schema_class = TF_SCHEMA_STABLE,
1888
+ .state_estimate = "O(1)",
685
1889
  .args = head_args,
686
1890
  .n_args = 1,
687
1891
  .infer_schema = infer_schema_passthrough,
688
1892
  .create_native = (void *(*)(const cJSON *))tf_head_create,
689
1893
  },
690
1894
  {
691
- .name = "skip",
1895
+ .name = "slice-head",
1896
+ .kind = TF_OP_TRANSFORM,
1897
+ .tier = TF_TIER_CORE,
1898
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1899
+ .memory_class = TF_MEM_BOUNDED_STATE,
1900
+ .emit_class = TF_EMIT_PER_BATCH,
1901
+ .schema_class = TF_SCHEMA_STABLE,
1902
+ .state_estimate = "O(1)",
1903
+ .args = head_args,
1904
+ .n_args = 1,
1905
+ .infer_schema = infer_schema_passthrough,
1906
+ .create_native = (void *(*)(const cJSON *))tf_head_create,
1907
+ },
1908
+ {
1909
+ .name = "skip",
1910
+ .kind = TF_OP_TRANSFORM,
1911
+ .tier = TF_TIER_CORE,
1912
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1913
+ .memory_class = TF_MEM_BOUNDED_STATE,
1914
+ .emit_class = TF_EMIT_PER_BATCH,
1915
+ .schema_class = TF_SCHEMA_STABLE,
1916
+ .state_estimate = "O(1)",
1917
+ .args = skip_args,
1918
+ .n_args = 1,
1919
+ .infer_schema = infer_schema_passthrough,
1920
+ .create_native = (void *(*)(const cJSON *))tf_skip_create,
1921
+ },
1922
+ {
1923
+ .name = "derive",
1924
+ .kind = TF_OP_TRANSFORM,
1925
+ .tier = TF_TIER_CORE,
1926
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1927
+ .memory_class = TF_MEM_ROW_LOCAL,
1928
+ .emit_class = TF_EMIT_PER_BATCH,
1929
+ .schema_class = TF_SCHEMA_PARAMETRIC,
1930
+ .state_estimate = "O(batch_rows * derived_columns)",
1931
+ .args = derive_args,
1932
+ .n_args = 1,
1933
+ .infer_schema = infer_schema_derive,
1934
+ .create_native = (void *(*)(const cJSON *))tf_derive_create,
1935
+ },
1936
+
1937
+ {
1938
+ .name = "source-name",
1939
+ .kind = TF_OP_TRANSFORM,
1940
+ .tier = TF_TIER_CORE,
1941
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1942
+ .memory_class = TF_MEM_ROW_LOCAL,
1943
+ .emit_class = TF_EMIT_PER_BATCH,
1944
+ .schema_class = TF_SCHEMA_PARAMETRIC,
1945
+ .state_estimate = "O(batch_rows)",
1946
+ .args = source_name_args,
1947
+ .n_args = 2,
1948
+ .infer_schema = infer_schema_source_name,
1949
+ .create_native = (void *(*)(const cJSON *))tf_source_name_create,
1950
+ },
1951
+
1952
+ {
1953
+ .name = "across",
692
1954
  .kind = TF_OP_TRANSFORM,
693
1955
  .tier = TF_TIER_CORE,
694
1956
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
695
- .args = skip_args,
696
- .n_args = 1,
697
- .infer_schema = infer_schema_passthrough,
698
- .create_native = (void *(*)(const cJSON *))tf_skip_create,
1957
+ .memory_class = TF_MEM_ROW_LOCAL,
1958
+ .emit_class = TF_EMIT_PER_BATCH,
1959
+ .schema_class = TF_SCHEMA_PARAMETRIC,
1960
+ .state_estimate = "O(batch_rows * selected_columns * functions)",
1961
+ .args = across_args,
1962
+ .n_args = 5,
1963
+ .infer_schema = infer_schema_across,
1964
+ .create_native = (void *(*)(const cJSON *))tf_across_create,
699
1965
  },
700
1966
  {
701
- .name = "derive",
1967
+ .name = "stats",
702
1968
  .kind = TF_OP_TRANSFORM,
703
1969
  .tier = TF_TIER_CORE,
704
1970
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
705
- .args = derive_args,
1971
+ .memory_class = TF_MEM_BOUNDED_STATE,
1972
+ .emit_class = TF_EMIT_ON_FLUSH,
1973
+ .schema_class = TF_SCHEMA_PARAMETRIC,
1974
+ .state_estimate = "O(columns * requested_stats)",
1975
+ .args = stats_args,
706
1976
  .n_args = 1,
707
- .infer_schema = infer_schema_derive,
708
- .create_native = (void *(*)(const cJSON *))tf_derive_create,
1977
+ .infer_schema = infer_schema_stats,
1978
+ .create_native = (void *(*)(const cJSON *))tf_stats_create,
709
1979
  },
710
1980
  {
711
- .name = "stats",
1981
+ .name = "scan",
712
1982
  .kind = TF_OP_TRANSFORM,
713
1983
  .tier = TF_TIER_CORE,
714
1984
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
715
- .args = stats_args,
1985
+ .memory_class = TF_MEM_BOUNDED_STATE,
1986
+ .emit_class = TF_EMIT_ON_FLUSH,
1987
+ .schema_class = TF_SCHEMA_PARAMETRIC,
1988
+ .state_estimate = "O(columns * profile_stats)",
1989
+ .args = scan_args,
716
1990
  .n_args = 1,
717
1991
  .infer_schema = infer_schema_stats,
718
- .create_native = (void *(*)(const cJSON *))tf_stats_create,
1992
+ .create_native = (void *(*)(const cJSON *))tf_scan_create,
719
1993
  },
720
1994
  {
721
1995
  .name = "unique",
722
1996
  .kind = TF_OP_TRANSFORM,
723
1997
  .tier = TF_TIER_CORE,
724
1998
  .caps = TF_CAP_STREAMING | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
1999
+ .memory_class = TF_MEM_KEY_STATE,
2000
+ .emit_class = TF_EMIT_PER_BATCH,
2001
+ .schema_class = TF_SCHEMA_STABLE,
2002
+ .state_estimate = "O(1) sorted, O(distinct_keys) exact unsorted or capped by max_keys/max_state_bytes, O(bloom_bytes) approximate, or O(spill_run_rows * columns) RAM with spill_dir",
725
2003
  .args = unique_args,
726
- .n_args = 1,
2004
+ .n_args = 12,
727
2005
  .infer_schema = infer_schema_passthrough,
728
2006
  .create_native = (void *(*)(const cJSON *))tf_unique_create,
729
2007
  },
@@ -732,8 +2010,12 @@ static tf_op_entry builtin_ops[] = {
732
2010
  .kind = TF_OP_TRANSFORM,
733
2011
  .tier = TF_TIER_CORE,
734
2012
  .caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2013
+ .memory_class = TF_MEM_BLOCKING,
2014
+ .emit_class = TF_EMIT_ON_FLUSH,
2015
+ .schema_class = TF_SCHEMA_STABLE,
2016
+ .state_estimate = "O(input_rows)",
735
2017
  .args = sort_args,
736
- .n_args = 1,
2018
+ .n_args = 5,
737
2019
  .infer_schema = infer_schema_passthrough,
738
2020
  .create_native = (void *(*)(const cJSON *))tf_sort_create,
739
2021
  },
@@ -743,18 +2025,40 @@ static tf_op_entry builtin_ops[] = {
743
2025
  .kind = TF_OP_TRANSFORM,
744
2026
  .tier = TF_TIER_CORE,
745
2027
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2028
+ .memory_class = TF_MEM_ROW_LOCAL,
2029
+ .emit_class = TF_EMIT_PER_BATCH,
2030
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2031
+ .state_estimate = "O(batch_rows * selected_columns)",
746
2032
  .args = select_args,
747
2033
  .n_args = 1,
748
2034
  .infer_schema = infer_schema_select,
749
2035
  .create_native = (void *(*)(const cJSON *))tf_select_create,
750
2036
  },
2037
+ {
2038
+ .name = "relocate",
2039
+ .kind = TF_OP_TRANSFORM,
2040
+ .tier = TF_TIER_CORE,
2041
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2042
+ .memory_class = TF_MEM_ROW_LOCAL,
2043
+ .emit_class = TF_EMIT_PER_BATCH,
2044
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2045
+ .state_estimate = "O(batch_rows * columns)",
2046
+ .args = relocate_args,
2047
+ .n_args = 3,
2048
+ .infer_schema = infer_schema_relocate,
2049
+ .create_native = (void *(*)(const cJSON *))tf_relocate_create,
2050
+ },
751
2051
  {
752
2052
  .name = "dedup",
753
2053
  .kind = TF_OP_TRANSFORM,
754
2054
  .tier = TF_TIER_CORE,
755
2055
  .caps = TF_CAP_STREAMING | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2056
+ .memory_class = TF_MEM_KEY_STATE,
2057
+ .emit_class = TF_EMIT_PER_BATCH,
2058
+ .schema_class = TF_SCHEMA_STABLE,
2059
+ .state_estimate = "O(1) sorted, O(distinct_keys) exact unsorted or capped by max_keys/max_state_bytes, O(bloom_bytes) approximate, or O(spill_run_rows * columns) RAM with spill_dir",
756
2060
  .args = unique_args,
757
- .n_args = 1,
2061
+ .n_args = 12,
758
2062
  .infer_schema = infer_schema_passthrough,
759
2063
  .create_native = (void *(*)(const cJSON *))tf_unique_create,
760
2064
  },
@@ -764,16 +2068,94 @@ static tf_op_entry builtin_ops[] = {
764
2068
  .kind = TF_OP_TRANSFORM,
765
2069
  .tier = TF_TIER_CORE,
766
2070
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2071
+ .memory_class = TF_MEM_ROW_LOCAL,
2072
+ .emit_class = TF_EMIT_MIXED,
2073
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2074
+ .state_estimate = "O(batch_rows * columns)",
767
2075
  .args = validate_args,
768
- .n_args = 1,
2076
+ .n_args = 16,
769
2077
  .infer_schema = infer_schema_validate,
770
2078
  .create_native = (void *(*)(const cJSON *))tf_validate_create,
771
2079
  },
2080
+ {
2081
+ .name = "assert",
2082
+ .kind = TF_OP_TRANSFORM,
2083
+ .tier = TF_TIER_CORE,
2084
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2085
+ .memory_class = TF_MEM_ROW_LOCAL,
2086
+ .emit_class = TF_EMIT_MIXED,
2087
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2088
+ .state_estimate = "O(batch_rows * columns) row mode, O(1) aggregate mode",
2089
+ .args = assert_args,
2090
+ .n_args = 19,
2091
+ .infer_schema = infer_schema_assert,
2092
+ .create_native = (void *(*)(const cJSON *))tf_assert_create,
2093
+ },
2094
+ {
2095
+ .name = "quarantine",
2096
+ .kind = TF_OP_TRANSFORM,
2097
+ .tier = TF_TIER_CORE,
2098
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2099
+ .memory_class = TF_MEM_ROW_LOCAL,
2100
+ .emit_class = TF_EMIT_MIXED,
2101
+ .schema_class = TF_SCHEMA_STABLE,
2102
+ .state_estimate = "O(batch_rows * columns)",
2103
+ .args = quarantine_args,
2104
+ .n_args = 9,
2105
+ .infer_schema = infer_schema_passthrough,
2106
+ .create_native = (void *(*)(const cJSON *))tf_quarantine_create,
2107
+ },
2108
+ {
2109
+ .name = "tee",
2110
+ .kind = TF_OP_TRANSFORM,
2111
+ .tier = TF_TIER_CORE,
2112
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2113
+ .memory_class = TF_MEM_ROW_LOCAL,
2114
+ .emit_class = TF_EMIT_MIXED,
2115
+ .schema_class = TF_SCHEMA_STABLE,
2116
+ .state_estimate = "O(batch_rows * selected_columns)",
2117
+ .args = tee_args,
2118
+ .n_args = 13,
2119
+ .infer_schema = infer_schema_passthrough,
2120
+ .create_native = (void *(*)(const cJSON *))tf_tee_create,
2121
+ },
2122
+ {
2123
+ .name = "schema",
2124
+ .kind = TF_OP_TRANSFORM,
2125
+ .tier = TF_TIER_CORE,
2126
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2127
+ .memory_class = TF_MEM_ROW_LOCAL,
2128
+ .emit_class = TF_EMIT_MIXED,
2129
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2130
+ .state_estimate = "O(batch_rows * declared_columns)",
2131
+ .args = schema_args,
2132
+ .n_args = 13,
2133
+ .infer_schema = infer_schema_schema,
2134
+ .create_native = (void *(*)(const cJSON *))tf_schema_create,
2135
+ },
2136
+ {
2137
+ .name = "schema-infer",
2138
+ .kind = TF_OP_TRANSFORM,
2139
+ .tier = TF_TIER_CORE,
2140
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2141
+ .memory_class = TF_MEM_BOUNDED_STATE,
2142
+ .emit_class = TF_EMIT_ON_FLUSH,
2143
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2144
+ .state_estimate = "O(columns)",
2145
+ .args = schema_infer_args,
2146
+ .n_args = 1,
2147
+ .infer_schema = infer_schema_schema_infer,
2148
+ .create_native = (void *(*)(const cJSON *))tf_schema_infer_create,
2149
+ },
772
2150
  {
773
2151
  .name = "trim",
774
2152
  .kind = TF_OP_TRANSFORM,
775
2153
  .tier = TF_TIER_CORE,
776
2154
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2155
+ .memory_class = TF_MEM_ROW_LOCAL,
2156
+ .emit_class = TF_EMIT_PER_BATCH,
2157
+ .schema_class = TF_SCHEMA_STABLE,
2158
+ .state_estimate = "O(batch_rows * columns)",
777
2159
  .args = trim_args,
778
2160
  .n_args = 1,
779
2161
  .infer_schema = infer_schema_passthrough,
@@ -784,8 +2166,12 @@ static tf_op_entry builtin_ops[] = {
784
2166
  .kind = TF_OP_TRANSFORM,
785
2167
  .tier = TF_TIER_CORE,
786
2168
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2169
+ .memory_class = TF_MEM_ROW_LOCAL,
2170
+ .emit_class = TF_EMIT_PER_BATCH,
2171
+ .schema_class = TF_SCHEMA_STABLE,
2172
+ .state_estimate = "O(batch_rows * columns)",
787
2173
  .args = fill_null_args,
788
- .n_args = 1,
2174
+ .n_args = 9,
789
2175
  .infer_schema = infer_schema_passthrough,
790
2176
  .create_native = (void *(*)(const cJSON *))tf_fill_null_create,
791
2177
  },
@@ -794,8 +2180,12 @@ static tf_op_entry builtin_ops[] = {
794
2180
  .kind = TF_OP_TRANSFORM,
795
2181
  .tier = TF_TIER_CORE,
796
2182
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2183
+ .memory_class = TF_MEM_ROW_LOCAL,
2184
+ .emit_class = TF_EMIT_PER_BATCH,
2185
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2186
+ .state_estimate = "O(batch_rows * columns)",
797
2187
  .args = cast_args,
798
- .n_args = 1,
2188
+ .n_args = 9,
799
2189
  .infer_schema = infer_schema_passthrough, /* type changes at runtime */
800
2190
  .create_native = (void *(*)(const cJSON *))tf_cast_create,
801
2191
  },
@@ -804,6 +2194,10 @@ static tf_op_entry builtin_ops[] = {
804
2194
  .kind = TF_OP_TRANSFORM,
805
2195
  .tier = TF_TIER_CORE,
806
2196
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2197
+ .memory_class = TF_MEM_ROW_LOCAL,
2198
+ .emit_class = TF_EMIT_PER_BATCH,
2199
+ .schema_class = TF_SCHEMA_STABLE,
2200
+ .state_estimate = "O(batch_rows)",
807
2201
  .args = clip_args,
808
2202
  .n_args = 3,
809
2203
  .infer_schema = infer_schema_passthrough,
@@ -814,8 +2208,12 @@ static tf_op_entry builtin_ops[] = {
814
2208
  .kind = TF_OP_TRANSFORM,
815
2209
  .tier = TF_TIER_CORE,
816
2210
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2211
+ .memory_class = TF_MEM_ROW_LOCAL,
2212
+ .emit_class = TF_EMIT_PER_BATCH,
2213
+ .schema_class = TF_SCHEMA_STABLE,
2214
+ .state_estimate = "O(batch_rows)",
817
2215
  .args = replace_args,
818
- .n_args = 4,
2216
+ .n_args = 12,
819
2217
  .infer_schema = infer_schema_passthrough,
820
2218
  .create_native = (void *(*)(const cJSON *))tf_replace_create,
821
2219
  },
@@ -824,6 +2222,10 @@ static tf_op_entry builtin_ops[] = {
824
2222
  .kind = TF_OP_TRANSFORM,
825
2223
  .tier = TF_TIER_CORE,
826
2224
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2225
+ .memory_class = TF_MEM_ROW_LOCAL,
2226
+ .emit_class = TF_EMIT_PER_BATCH,
2227
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2228
+ .state_estimate = "O(batch_rows * selected_columns)",
827
2229
  .args = hash_args,
828
2230
  .n_args = 1,
829
2231
  .infer_schema = infer_schema_add_hash,
@@ -834,8 +2236,12 @@ static tf_op_entry builtin_ops[] = {
834
2236
  .kind = TF_OP_TRANSFORM,
835
2237
  .tier = TF_TIER_CORE,
836
2238
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2239
+ .memory_class = TF_MEM_ROW_LOCAL,
2240
+ .emit_class = TF_EMIT_PER_BATCH,
2241
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2242
+ .state_estimate = "O(boundaries)",
837
2243
  .args = bin_args,
838
- .n_args = 2,
2244
+ .n_args = 4,
839
2245
  .infer_schema = infer_schema_passthrough, /* adds column at runtime */
840
2246
  .create_native = (void *(*)(const cJSON *))tf_bin_create,
841
2247
  },
@@ -845,6 +2251,10 @@ static tf_op_entry builtin_ops[] = {
845
2251
  .kind = TF_OP_TRANSFORM,
846
2252
  .tier = TF_TIER_CORE,
847
2253
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2254
+ .memory_class = TF_MEM_BOUNDED_STATE,
2255
+ .emit_class = TF_EMIT_PER_BATCH,
2256
+ .schema_class = TF_SCHEMA_STABLE,
2257
+ .state_estimate = "O(columns)",
848
2258
  .args = fill_down_args,
849
2259
  .n_args = 1,
850
2260
  .infer_schema = infer_schema_passthrough,
@@ -855,6 +2265,10 @@ static tf_op_entry builtin_ops[] = {
855
2265
  .kind = TF_OP_TRANSFORM,
856
2266
  .tier = TF_TIER_CORE,
857
2267
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2268
+ .memory_class = TF_MEM_BOUNDED_STATE,
2269
+ .emit_class = TF_EMIT_PER_BATCH,
2270
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2271
+ .state_estimate = "O(1)",
858
2272
  .args = step_args,
859
2273
  .n_args = 3,
860
2274
  .infer_schema = infer_schema_passthrough, /* adds column at runtime */
@@ -865,17 +2279,109 @@ static tf_op_entry builtin_ops[] = {
865
2279
  .kind = TF_OP_TRANSFORM,
866
2280
  .tier = TF_TIER_CORE,
867
2281
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2282
+ .memory_class = TF_MEM_BOUNDED_STATE,
2283
+ .emit_class = TF_EMIT_PER_BATCH,
2284
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2285
+ .state_estimate = "O(window_size)",
868
2286
  .args = window_args,
869
- .n_args = 4,
2287
+ .n_args = 6,
870
2288
  .infer_schema = infer_schema_passthrough, /* adds column at runtime */
871
2289
  .create_native = (void *(*)(const cJSON *))tf_window_create,
872
2290
  },
2291
+ {
2292
+ .name = "rolling-sum",
2293
+ .kind = TF_OP_TRANSFORM,
2294
+ .tier = TF_TIER_CORE,
2295
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2296
+ .memory_class = TF_MEM_BOUNDED_STATE,
2297
+ .emit_class = TF_EMIT_PER_BATCH,
2298
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2299
+ .state_estimate = "O(window_size)",
2300
+ .args = rolling_args,
2301
+ .n_args = 5,
2302
+ .infer_schema = infer_schema_passthrough,
2303
+ .create_native = (void *(*)(const cJSON *))tf_rolling_sum_create,
2304
+ },
2305
+ {
2306
+ .name = "rolling-mean",
2307
+ .kind = TF_OP_TRANSFORM,
2308
+ .tier = TF_TIER_CORE,
2309
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2310
+ .memory_class = TF_MEM_BOUNDED_STATE,
2311
+ .emit_class = TF_EMIT_PER_BATCH,
2312
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2313
+ .state_estimate = "O(window_size)",
2314
+ .args = rolling_args,
2315
+ .n_args = 5,
2316
+ .infer_schema = infer_schema_passthrough,
2317
+ .create_native = (void *(*)(const cJSON *))tf_rolling_mean_create,
2318
+ },
2319
+ {
2320
+ .name = "rolling-min",
2321
+ .kind = TF_OP_TRANSFORM,
2322
+ .tier = TF_TIER_CORE,
2323
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2324
+ .memory_class = TF_MEM_BOUNDED_STATE,
2325
+ .emit_class = TF_EMIT_PER_BATCH,
2326
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2327
+ .state_estimate = "O(window_size)",
2328
+ .args = rolling_args,
2329
+ .n_args = 5,
2330
+ .infer_schema = infer_schema_passthrough,
2331
+ .create_native = (void *(*)(const cJSON *))tf_rolling_min_create,
2332
+ },
2333
+ {
2334
+ .name = "rolling-max",
2335
+ .kind = TF_OP_TRANSFORM,
2336
+ .tier = TF_TIER_CORE,
2337
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2338
+ .memory_class = TF_MEM_BOUNDED_STATE,
2339
+ .emit_class = TF_EMIT_PER_BATCH,
2340
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2341
+ .state_estimate = "O(window_size)",
2342
+ .args = rolling_args,
2343
+ .n_args = 5,
2344
+ .infer_schema = infer_schema_passthrough,
2345
+ .create_native = (void *(*)(const cJSON *))tf_rolling_max_create,
2346
+ },
2347
+ {
2348
+ .name = "rolling-any",
2349
+ .kind = TF_OP_TRANSFORM,
2350
+ .tier = TF_TIER_CORE,
2351
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2352
+ .memory_class = TF_MEM_BOUNDED_STATE,
2353
+ .emit_class = TF_EMIT_PER_BATCH,
2354
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2355
+ .state_estimate = "O(window_size)",
2356
+ .args = rolling_bool_args,
2357
+ .n_args = 4,
2358
+ .infer_schema = infer_schema_passthrough,
2359
+ .create_native = (void *(*)(const cJSON *))tf_rolling_any_create,
2360
+ },
2361
+ {
2362
+ .name = "rolling-all",
2363
+ .kind = TF_OP_TRANSFORM,
2364
+ .tier = TF_TIER_CORE,
2365
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2366
+ .memory_class = TF_MEM_BOUNDED_STATE,
2367
+ .emit_class = TF_EMIT_PER_BATCH,
2368
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2369
+ .state_estimate = "O(window_size)",
2370
+ .args = rolling_bool_args,
2371
+ .n_args = 4,
2372
+ .infer_schema = infer_schema_passthrough,
2373
+ .create_native = (void *(*)(const cJSON *))tf_rolling_all_create,
2374
+ },
873
2375
  /* ---- Row-multiplying streaming ---- */
874
2376
  {
875
2377
  .name = "explode",
876
2378
  .kind = TF_OP_TRANSFORM,
877
2379
  .tier = TF_TIER_CORE,
878
2380
  .caps = TF_CAP_STREAMING | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2381
+ .memory_class = TF_MEM_ROW_LOCAL,
2382
+ .emit_class = TF_EMIT_PER_BATCH,
2383
+ .schema_class = TF_SCHEMA_STABLE,
2384
+ .state_estimate = "O(batch_rows)",
879
2385
  .args = explode_args,
880
2386
  .n_args = 2,
881
2387
  .infer_schema = infer_schema_passthrough,
@@ -886,6 +2392,10 @@ static tf_op_entry builtin_ops[] = {
886
2392
  .kind = TF_OP_TRANSFORM,
887
2393
  .tier = TF_TIER_CORE,
888
2394
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2395
+ .memory_class = TF_MEM_ROW_LOCAL,
2396
+ .emit_class = TF_EMIT_PER_BATCH,
2397
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2398
+ .state_estimate = "O(batch_rows * output_columns)",
889
2399
  .args = split_args,
890
2400
  .n_args = 3,
891
2401
  .infer_schema = infer_schema_passthrough, /* adds columns at runtime */
@@ -896,6 +2406,10 @@ static tf_op_entry builtin_ops[] = {
896
2406
  .kind = TF_OP_TRANSFORM,
897
2407
  .tier = TF_TIER_CORE,
898
2408
  .caps = TF_CAP_STREAMING | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2409
+ .memory_class = TF_MEM_ROW_LOCAL,
2410
+ .emit_class = TF_EMIT_PER_BATCH,
2411
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2412
+ .state_estimate = "O(batch_rows * unpivot_columns)",
899
2413
  .args = unpivot_args,
900
2414
  .n_args = 1,
901
2415
  .infer_schema = infer_schema_passthrough, /* schema changes at runtime */
@@ -907,6 +2421,24 @@ static tf_op_entry builtin_ops[] = {
907
2421
  .kind = TF_OP_TRANSFORM,
908
2422
  .tier = TF_TIER_CORE,
909
2423
  .caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2424
+ .memory_class = TF_MEM_BOUNDED_STATE,
2425
+ .emit_class = TF_EMIT_ON_FLUSH,
2426
+ .schema_class = TF_SCHEMA_STABLE,
2427
+ .state_estimate = "O(n)",
2428
+ .args = tail_args,
2429
+ .n_args = 1,
2430
+ .infer_schema = infer_schema_passthrough,
2431
+ .create_native = (void *(*)(const cJSON *))tf_tail_create,
2432
+ },
2433
+ {
2434
+ .name = "slice-tail",
2435
+ .kind = TF_OP_TRANSFORM,
2436
+ .tier = TF_TIER_CORE,
2437
+ .caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2438
+ .memory_class = TF_MEM_BOUNDED_STATE,
2439
+ .emit_class = TF_EMIT_ON_FLUSH,
2440
+ .schema_class = TF_SCHEMA_STABLE,
2441
+ .state_estimate = "O(n)",
910
2442
  .args = tail_args,
911
2443
  .n_args = 1,
912
2444
  .infer_schema = infer_schema_passthrough,
@@ -917,18 +2449,82 @@ static tf_op_entry builtin_ops[] = {
917
2449
  .kind = TF_OP_TRANSFORM,
918
2450
  .tier = TF_TIER_CORE,
919
2451
  .caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2452
+ .memory_class = TF_MEM_BOUNDED_STATE,
2453
+ .emit_class = TF_EMIT_ON_FLUSH,
2454
+ .schema_class = TF_SCHEMA_STABLE,
2455
+ .state_estimate = "O(n)",
920
2456
  .args = top_args,
921
2457
  .n_args = 3,
922
2458
  .infer_schema = infer_schema_passthrough,
923
2459
  .create_native = (void *(*)(const cJSON *))tf_top_create,
924
2460
  },
2461
+ {
2462
+ .name = "top-k",
2463
+ .kind = TF_OP_TRANSFORM,
2464
+ .tier = TF_TIER_CORE,
2465
+ .caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2466
+ .memory_class = TF_MEM_BOUNDED_STATE,
2467
+ .emit_class = TF_EMIT_ON_FLUSH,
2468
+ .schema_class = TF_SCHEMA_STABLE,
2469
+ .state_estimate = "O(n)",
2470
+ .args = top_args,
2471
+ .n_args = 3,
2472
+ .infer_schema = infer_schema_passthrough,
2473
+ .create_native = (void *(*)(const cJSON *))tf_top_k_create,
2474
+ },
2475
+ {
2476
+ .name = "bottom-k",
2477
+ .kind = TF_OP_TRANSFORM,
2478
+ .tier = TF_TIER_CORE,
2479
+ .caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2480
+ .memory_class = TF_MEM_BOUNDED_STATE,
2481
+ .emit_class = TF_EMIT_ON_FLUSH,
2482
+ .schema_class = TF_SCHEMA_STABLE,
2483
+ .state_estimate = "O(n)",
2484
+ .args = top_args,
2485
+ .n_args = 3,
2486
+ .infer_schema = infer_schema_passthrough,
2487
+ .create_native = (void *(*)(const cJSON *))tf_bottom_k_create,
2488
+ },
2489
+ {
2490
+ .name = "slice-min",
2491
+ .kind = TF_OP_TRANSFORM,
2492
+ .tier = TF_TIER_CORE,
2493
+ .caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2494
+ .memory_class = TF_MEM_BOUNDED_STATE,
2495
+ .emit_class = TF_EMIT_ON_FLUSH,
2496
+ .schema_class = TF_SCHEMA_STABLE,
2497
+ .state_estimate = "O(n)",
2498
+ .args = top_args,
2499
+ .n_args = 4,
2500
+ .infer_schema = infer_schema_passthrough,
2501
+ .create_native = (void *(*)(const cJSON *))tf_slice_min_create,
2502
+ },
2503
+ {
2504
+ .name = "slice-max",
2505
+ .kind = TF_OP_TRANSFORM,
2506
+ .tier = TF_TIER_CORE,
2507
+ .caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2508
+ .memory_class = TF_MEM_BOUNDED_STATE,
2509
+ .emit_class = TF_EMIT_ON_FLUSH,
2510
+ .schema_class = TF_SCHEMA_STABLE,
2511
+ .state_estimate = "O(n)",
2512
+ .args = top_args,
2513
+ .n_args = 4,
2514
+ .infer_schema = infer_schema_passthrough,
2515
+ .create_native = (void *(*)(const cJSON *))tf_slice_max_create,
2516
+ },
925
2517
  {
926
2518
  .name = "sample",
927
2519
  .kind = TF_OP_TRANSFORM,
928
2520
  .tier = TF_TIER_CORE,
929
2521
  .caps = TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE,
2522
+ .memory_class = TF_MEM_BOUNDED_STATE,
2523
+ .emit_class = TF_EMIT_ON_FLUSH,
2524
+ .schema_class = TF_SCHEMA_STABLE,
2525
+ .state_estimate = "O(n)",
930
2526
  .args = sample_args,
931
- .n_args = 1,
2527
+ .n_args = 2,
932
2528
  .infer_schema = infer_schema_passthrough,
933
2529
  .create_native = (void *(*)(const cJSON *))tf_sample_create,
934
2530
  },
@@ -937,8 +2533,12 @@ static tf_op_entry builtin_ops[] = {
937
2533
  .kind = TF_OP_TRANSFORM,
938
2534
  .tier = TF_TIER_CORE,
939
2535
  .caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2536
+ .memory_class = TF_MEM_KEY_STATE,
2537
+ .emit_class = TF_EMIT_MIXED,
2538
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2539
+ .state_estimate = "O(1) sorted, O(distinct_groups) unsorted or capped by max_groups/max_state_bytes, or O(spill_run_rows * columns) RAM with spill_dir",
940
2540
  .args = group_agg_args,
941
- .n_args = 2,
2541
+ .n_args = 9,
942
2542
  .infer_schema = infer_schema_group_agg,
943
2543
  .create_native = (void *(*)(const cJSON *))tf_group_agg_create,
944
2544
  },
@@ -947,8 +2547,12 @@ static tf_op_entry builtin_ops[] = {
947
2547
  .kind = TF_OP_TRANSFORM,
948
2548
  .tier = TF_TIER_CORE,
949
2549
  .caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2550
+ .memory_class = TF_MEM_KEY_STATE,
2551
+ .emit_class = TF_EMIT_ON_FLUSH,
2552
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2553
+ .state_estimate = "O(distinct_values) or capped by max_values/max_state_bytes",
950
2554
  .args = frequency_args,
951
- .n_args = 1,
2555
+ .n_args = 13,
952
2556
  .infer_schema = infer_schema_frequency,
953
2557
  .create_native = (void *(*)(const cJSON *))tf_frequency_create,
954
2558
  },
@@ -958,16 +2562,80 @@ static tf_op_entry builtin_ops[] = {
958
2562
  .kind = TF_OP_TRANSFORM,
959
2563
  .tier = TF_TIER_CORE,
960
2564
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2565
+ .memory_class = TF_MEM_ROW_LOCAL,
2566
+ .emit_class = TF_EMIT_PER_BATCH,
2567
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2568
+ .state_estimate = "O(batch_rows)",
961
2569
  .args = datetime_args,
962
- .n_args = 2,
2570
+ .n_args = 4,
963
2571
  .infer_schema = infer_schema_passthrough, /* adds columns at runtime */
964
2572
  .create_native = (void *(*)(const cJSON *))tf_datetime_create,
965
2573
  },
2574
+ {
2575
+ .name = "json-extract",
2576
+ .kind = TF_OP_TRANSFORM,
2577
+ .tier = TF_TIER_CORE,
2578
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2579
+ .memory_class = TF_MEM_ROW_LOCAL,
2580
+ .emit_class = TF_EMIT_PER_BATCH,
2581
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2582
+ .state_estimate = "O(batch_rows + path_segments)",
2583
+ .args = json_extract_args,
2584
+ .n_args = 4,
2585
+ .infer_schema = infer_schema_json_extract,
2586
+ .create_native = (void *(*)(const cJSON *))tf_json_extract_create,
2587
+ },
2588
+ {
2589
+ .name = "json-filter",
2590
+ .kind = TF_OP_TRANSFORM,
2591
+ .tier = TF_TIER_CORE,
2592
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2593
+ .memory_class = TF_MEM_ROW_LOCAL,
2594
+ .emit_class = TF_EMIT_PER_BATCH,
2595
+ .schema_class = TF_SCHEMA_STABLE,
2596
+ .state_estimate = "O(batch_rows + path_segments)",
2597
+ .args = json_filter_args,
2598
+ .n_args = 5,
2599
+ .infer_schema = infer_schema_passthrough,
2600
+ .create_native = (void *(*)(const cJSON *))tf_json_filter_create,
2601
+ },
2602
+ {
2603
+ .name = "json-schema",
2604
+ .kind = TF_OP_TRANSFORM,
2605
+ .tier = TF_TIER_CORE,
2606
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2607
+ .memory_class = TF_MEM_ROW_LOCAL,
2608
+ .emit_class = TF_EMIT_PER_BATCH,
2609
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2610
+ .state_estimate = "O(batch_rows + schema_nodes)",
2611
+ .args = json_schema_args,
2612
+ .n_args = 12,
2613
+ .infer_schema = infer_schema_json_schema,
2614
+ .create_native = (void *(*)(const cJSON *))tf_json_schema_create,
2615
+ },
2616
+ {
2617
+ .name = "json-flatten",
2618
+ .kind = TF_OP_TRANSFORM,
2619
+ .tier = TF_TIER_CORE,
2620
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2621
+ .memory_class = TF_MEM_ROW_LOCAL,
2622
+ .emit_class = TF_EMIT_PER_BATCH,
2623
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2624
+ .state_estimate = "O(batch_rows * fields + path_segments)",
2625
+ .args = json_flatten_args,
2626
+ .n_args = 2,
2627
+ .infer_schema = infer_schema_json_flatten,
2628
+ .create_native = (void *(*)(const cJSON *))tf_json_flatten_create,
2629
+ },
966
2630
  {
967
2631
  .name = "flatten",
968
2632
  .kind = TF_OP_TRANSFORM,
969
2633
  .tier = TF_TIER_CORE,
970
2634
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2635
+ .memory_class = TF_MEM_ROW_LOCAL,
2636
+ .emit_class = TF_EMIT_PER_BATCH,
2637
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2638
+ .state_estimate = "O(batch_rows * fields)",
971
2639
  .args = NULL,
972
2640
  .n_args = 0,
973
2641
  .infer_schema = infer_schema_passthrough,
@@ -979,9 +2647,13 @@ static tf_op_entry builtin_ops[] = {
979
2647
  .kind = TF_OP_TRANSFORM,
980
2648
  .tier = TF_TIER_CORE,
981
2649
  .caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2650
+ .memory_class = TF_MEM_BLOCKING,
2651
+ .emit_class = TF_EMIT_ON_FLUSH,
2652
+ .schema_class = TF_SCHEMA_DATA_DEPENDENT,
2653
+ .state_estimate = "O(input_rows + distinct_categories), O(current_group + categories) with categories+sorted=true, or external spill with categories/max_categories",
982
2654
  .args = pivot_args,
983
- .n_args = 3,
984
- .infer_schema = infer_schema_passthrough, /* schema changes at runtime */
2655
+ .n_args = 10,
2656
+ .infer_schema = infer_schema_pivot,
985
2657
  .create_native = (void *(*)(const cJSON *))tf_pivot_create,
986
2658
  },
987
2659
  {
@@ -989,16 +2661,136 @@ static tf_op_entry builtin_ops[] = {
989
2661
  .kind = TF_OP_TRANSFORM,
990
2662
  .tier = TF_TIER_CORE,
991
2663
  .caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
2664
+ .memory_class = TF_MEM_KEY_STATE,
2665
+ .emit_class = TF_EMIT_PER_BATCH,
2666
+ .schema_class = TF_SCHEMA_DATA_DEPENDENT,
2667
+ .state_estimate = "O(current_lookup_run) sorted, O(lookup_rows + lookup_keys + output_batch) unsorted or capped by max_lookup_*/max_state_bytes/max_output_rows, or O(spill_run_rows * columns + max_matches_per_row) RAM with spill_dir",
992
2668
  .args = join_args,
993
- .n_args = 3,
2669
+ .n_args = 14,
994
2670
  .infer_schema = infer_schema_passthrough, /* schema depends on lookup file */
995
2671
  .create_native = (void *(*)(const cJSON *))tf_join_create,
996
2672
  },
2673
+ {
2674
+ .name = "semi-join",
2675
+ .kind = TF_OP_TRANSFORM,
2676
+ .tier = TF_TIER_CORE,
2677
+ .caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
2678
+ .memory_class = TF_MEM_KEY_STATE,
2679
+ .emit_class = TF_EMIT_PER_BATCH,
2680
+ .schema_class = TF_SCHEMA_STABLE,
2681
+ .state_estimate = "O(1) sorted, O(lookup_keys) unsorted or capped by max_lookup_*/max_state_bytes/max_output_rows, or O(spill_run_rows * columns) RAM with spill_dir",
2682
+ .args = join_args,
2683
+ .n_args = 14,
2684
+ .infer_schema = infer_schema_passthrough,
2685
+ .create_native = (void *(*)(const cJSON *))tf_semi_join_create,
2686
+ },
2687
+ {
2688
+ .name = "anti-join",
2689
+ .kind = TF_OP_TRANSFORM,
2690
+ .tier = TF_TIER_CORE,
2691
+ .caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
2692
+ .memory_class = TF_MEM_KEY_STATE,
2693
+ .emit_class = TF_EMIT_PER_BATCH,
2694
+ .schema_class = TF_SCHEMA_STABLE,
2695
+ .state_estimate = "O(1) sorted, O(lookup_keys) unsorted or capped by max_lookup_*/max_state_bytes/max_output_rows, or O(spill_run_rows * columns) RAM with spill_dir",
2696
+ .args = join_args,
2697
+ .n_args = 14,
2698
+ .infer_schema = infer_schema_passthrough,
2699
+ .create_native = (void *(*)(const cJSON *))tf_anti_join_create,
2700
+ },
2701
+ {
2702
+ .name = "intersect",
2703
+ .kind = TF_OP_TRANSFORM,
2704
+ .tier = TF_TIER_CORE,
2705
+ .caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
2706
+ .memory_class = TF_MEM_KEY_STATE,
2707
+ .emit_class = TF_EMIT_PER_BATCH,
2708
+ .schema_class = TF_SCHEMA_STABLE,
2709
+ .state_estimate = "O(1) sorted, O(lookup_keys + emitted_keys) unsorted or capped by max_lookup_*/max_output_keys/max_state_bytes",
2710
+ .args = set_file_args,
2711
+ .n_args = 12,
2712
+ .infer_schema = infer_schema_passthrough,
2713
+ .create_native = (void *(*)(const cJSON *))tf_intersect_create,
2714
+ },
2715
+ {
2716
+ .name = "setdiff",
2717
+ .kind = TF_OP_TRANSFORM,
2718
+ .tier = TF_TIER_CORE,
2719
+ .caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
2720
+ .memory_class = TF_MEM_KEY_STATE,
2721
+ .emit_class = TF_EMIT_PER_BATCH,
2722
+ .schema_class = TF_SCHEMA_STABLE,
2723
+ .state_estimate = "O(1) sorted, O(lookup_keys + emitted_keys) unsorted or capped by max_lookup_*/max_output_keys/max_state_bytes",
2724
+ .args = set_file_args,
2725
+ .n_args = 12,
2726
+ .infer_schema = infer_schema_passthrough,
2727
+ .create_native = (void *(*)(const cJSON *))tf_setdiff_create,
2728
+ },
2729
+ {
2730
+ .name = "intersect-all",
2731
+ .kind = TF_OP_TRANSFORM,
2732
+ .tier = TF_TIER_CORE,
2733
+ .caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
2734
+ .memory_class = TF_MEM_KEY_STATE,
2735
+ .emit_class = TF_EMIT_PER_BATCH,
2736
+ .schema_class = TF_SCHEMA_STABLE,
2737
+ .state_estimate = "O(lookup_keys) unsorted with max_lookup_keys/max_state_bytes; O(spill_run_rows * columns) RAM with spill_dir; O(1) current-run state with sorted=true; emits min(left_count, lookup_count) rows per key",
2738
+ .args = set_file_args,
2739
+ .n_args = 12,
2740
+ .infer_schema = infer_schema_passthrough,
2741
+ .create_native = (void *(*)(const cJSON *))tf_intersect_all_create,
2742
+ },
2743
+ {
2744
+ .name = "setdiff-all",
2745
+ .kind = TF_OP_TRANSFORM,
2746
+ .tier = TF_TIER_CORE,
2747
+ .caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
2748
+ .memory_class = TF_MEM_KEY_STATE,
2749
+ .emit_class = TF_EMIT_PER_BATCH,
2750
+ .schema_class = TF_SCHEMA_STABLE,
2751
+ .state_estimate = "O(lookup_keys) unsorted with max_lookup_keys/max_state_bytes; O(spill_run_rows * columns) RAM with spill_dir; O(1) current-run state with sorted=true; emits max(left_count - lookup_count, 0) rows per key",
2752
+ .args = set_file_args,
2753
+ .n_args = 12,
2754
+ .infer_schema = infer_schema_passthrough,
2755
+ .create_native = (void *(*)(const cJSON *))tf_setdiff_all_create,
2756
+ },
2757
+ {
2758
+ .name = "union",
2759
+ .kind = TF_OP_TRANSFORM,
2760
+ .tier = TF_TIER_CORE,
2761
+ .caps = TF_CAP_STREAMING | TF_CAP_FS | TF_CAP_DETERMINISTIC,
2762
+ .memory_class = TF_MEM_KEY_STATE,
2763
+ .emit_class = TF_EMIT_MIXED,
2764
+ .schema_class = TF_SCHEMA_STABLE,
2765
+ .state_estimate = "O(emitted_keys), capped by max_output_keys/max_state_bytes; O(previous_left_key + current_file_key) with sorted=true; O(spill_run_rows * columns) RAM with spill_dir",
2766
+ .args = union_args,
2767
+ .n_args = 11,
2768
+ .infer_schema = infer_schema_passthrough,
2769
+ .create_native = (void *(*)(const cJSON *))tf_union_create,
2770
+ },
2771
+ {
2772
+ .name = "union-all",
2773
+ .kind = TF_OP_TRANSFORM,
2774
+ .tier = TF_TIER_CORE,
2775
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_FS | TF_CAP_DETERMINISTIC,
2776
+ .memory_class = TF_MEM_BOUNDED_STATE,
2777
+ .emit_class = TF_EMIT_MIXED,
2778
+ .schema_class = TF_SCHEMA_STABLE,
2779
+ .state_estimate = "O(batch_rows)",
2780
+ .args = union_all_args,
2781
+ .n_args = 3,
2782
+ .infer_schema = infer_schema_passthrough,
2783
+ .create_native = (void *(*)(const cJSON *))tf_union_all_create,
2784
+ },
997
2785
  {
998
2786
  .name = "stack",
999
2787
  .kind = TF_OP_TRANSFORM,
1000
2788
  .tier = TF_TIER_CORE,
1001
- .caps = TF_CAP_FS | TF_CAP_DETERMINISTIC,
2789
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_FS | TF_CAP_DETERMINISTIC,
2790
+ .memory_class = TF_MEM_BOUNDED_STATE,
2791
+ .emit_class = TF_EMIT_MIXED,
2792
+ .schema_class = TF_SCHEMA_STABLE,
2793
+ .state_estimate = "O(batch_rows)",
1002
2794
  .args = stack_args,
1003
2795
  .n_args = 3,
1004
2796
  .infer_schema = infer_schema_passthrough,
@@ -1009,18 +2801,82 @@ static tf_op_entry builtin_ops[] = {
1009
2801
  .kind = TF_OP_TRANSFORM,
1010
2802
  .tier = TF_TIER_CORE,
1011
2803
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2804
+ .memory_class = TF_MEM_BOUNDED_STATE,
2805
+ .emit_class = TF_EMIT_PER_BATCH,
2806
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2807
+ .state_estimate = "O(offset)",
1012
2808
  .args = lead_args,
1013
2809
  .n_args = 3,
1014
- .infer_schema = infer_schema_passthrough,
2810
+ .infer_schema = infer_schema_lead,
1015
2811
  .create_native = (void *(*)(const cJSON *))tf_lead_create,
1016
2812
  },
2813
+ {
2814
+ .name = "lag",
2815
+ .kind = TF_OP_TRANSFORM,
2816
+ .tier = TF_TIER_CORE,
2817
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2818
+ .memory_class = TF_MEM_BOUNDED_STATE,
2819
+ .emit_class = TF_EMIT_PER_BATCH,
2820
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2821
+ .state_estimate = "O(offset * row_width)",
2822
+ .args = lag_args,
2823
+ .n_args = 3,
2824
+ .infer_schema = infer_schema_lag,
2825
+ .create_native = (void *(*)(const cJSON *))tf_lag_create,
2826
+ },
2827
+ {
2828
+ .name = "shift",
2829
+ .kind = TF_OP_TRANSFORM,
2830
+ .tier = TF_TIER_CORE,
2831
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2832
+ .memory_class = TF_MEM_BOUNDED_STATE,
2833
+ .emit_class = TF_EMIT_PER_BATCH,
2834
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2835
+ .state_estimate = "O(offset * row_width)",
2836
+ .args = shift_args,
2837
+ .n_args = 4,
2838
+ .infer_schema = infer_schema_shift,
2839
+ .create_native = (void *(*)(const cJSON *))tf_shift_create,
2840
+ },
2841
+ {
2842
+ .name = "rowid",
2843
+ .kind = TF_OP_TRANSFORM,
2844
+ .tier = TF_TIER_CORE,
2845
+ .caps = TF_CAP_STREAMING | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2846
+ .memory_class = TF_MEM_KEY_STATE,
2847
+ .emit_class = TF_EMIT_PER_BATCH,
2848
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2849
+ .state_estimate = "O(1) global/sorted, O(distinct_keys) unsorted or capped by max_keys/max_state_bytes",
2850
+ .args = rowid_args,
2851
+ .n_args = 5,
2852
+ .infer_schema = infer_schema_rowid,
2853
+ .create_native = (void *(*)(const cJSON *))tf_rowid_create,
2854
+ },
2855
+ {
2856
+ .name = "rleid",
2857
+ .kind = TF_OP_TRANSFORM,
2858
+ .tier = TF_TIER_CORE,
2859
+ .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2860
+ .memory_class = TF_MEM_BOUNDED_STATE,
2861
+ .emit_class = TF_EMIT_PER_BATCH,
2862
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2863
+ .state_estimate = "O(key_width)",
2864
+ .args = rleid_args,
2865
+ .n_args = 2,
2866
+ .infer_schema = infer_schema_rleid,
2867
+ .create_native = (void *(*)(const cJSON *))tf_rleid_create,
2868
+ },
1017
2869
  {
1018
2870
  .name = "date-trunc",
1019
2871
  .kind = TF_OP_TRANSFORM,
1020
2872
  .tier = TF_TIER_CORE,
1021
2873
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2874
+ .memory_class = TF_MEM_ROW_LOCAL,
2875
+ .emit_class = TF_EMIT_PER_BATCH,
2876
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2877
+ .state_estimate = "O(batch_rows)",
1022
2878
  .args = date_trunc_args,
1023
- .n_args = 3,
2879
+ .n_args = 5,
1024
2880
  .infer_schema = infer_schema_passthrough,
1025
2881
  .create_native = (void *(*)(const cJSON *))tf_date_trunc_create,
1026
2882
  },
@@ -1029,8 +2885,12 @@ static tf_op_entry builtin_ops[] = {
1029
2885
  .kind = TF_OP_TRANSFORM,
1030
2886
  .tier = TF_TIER_CORE,
1031
2887
  .caps = TF_CAP_STREAMING | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2888
+ .memory_class = TF_MEM_KEY_STATE,
2889
+ .emit_class = TF_EMIT_PER_BATCH,
2890
+ .schema_class = TF_SCHEMA_DATA_DEPENDENT,
2891
+ .state_estimate = "O(distinct_categories) or capped by max_categories/max_state_bytes",
1032
2892
  .args = onehot_args,
1033
- .n_args = 2,
2893
+ .n_args = 6,
1034
2894
  .infer_schema = infer_schema_passthrough,
1035
2895
  .create_native = (void *(*)(const cJSON *))tf_onehot_create,
1036
2896
  },
@@ -1039,8 +2899,12 @@ static tf_op_entry builtin_ops[] = {
1039
2899
  .kind = TF_OP_TRANSFORM,
1040
2900
  .tier = TF_TIER_CORE,
1041
2901
  .caps = TF_CAP_STREAMING | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2902
+ .memory_class = TF_MEM_KEY_STATE,
2903
+ .emit_class = TF_EMIT_PER_BATCH,
2904
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2905
+ .state_estimate = "O(distinct_categories) or capped by max_categories/max_state_bytes",
1042
2906
  .args = label_encode_args,
1043
- .n_args = 2,
2907
+ .n_args = 6,
1044
2908
  .infer_schema = infer_schema_passthrough,
1045
2909
  .create_native = (void *(*)(const cJSON *))tf_label_encode_create,
1046
2910
  },
@@ -1049,8 +2913,12 @@ static tf_op_entry builtin_ops[] = {
1049
2913
  .kind = TF_OP_TRANSFORM,
1050
2914
  .tier = TF_TIER_CORE,
1051
2915
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2916
+ .memory_class = TF_MEM_BOUNDED_STATE,
2917
+ .emit_class = TF_EMIT_PER_BATCH,
2918
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2919
+ .state_estimate = "O(1)",
1052
2920
  .args = ewma_args,
1053
- .n_args = 3,
2921
+ .n_args = 5,
1054
2922
  .infer_schema = infer_schema_passthrough,
1055
2923
  .create_native = (void *(*)(const cJSON *))tf_ewma_create,
1056
2924
  },
@@ -1059,6 +2927,10 @@ static tf_op_entry builtin_ops[] = {
1059
2927
  .kind = TF_OP_TRANSFORM,
1060
2928
  .tier = TF_TIER_CORE,
1061
2929
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2930
+ .memory_class = TF_MEM_BOUNDED_STATE,
2931
+ .emit_class = TF_EMIT_PER_BATCH,
2932
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2933
+ .state_estimate = "O(order)",
1062
2934
  .args = diff_args,
1063
2935
  .n_args = 3,
1064
2936
  .infer_schema = infer_schema_passthrough,
@@ -1069,8 +2941,12 @@ static tf_op_entry builtin_ops[] = {
1069
2941
  .kind = TF_OP_TRANSFORM,
1070
2942
  .tier = TF_TIER_CORE,
1071
2943
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2944
+ .memory_class = TF_MEM_BOUNDED_STATE,
2945
+ .emit_class = TF_EMIT_PER_BATCH,
2946
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2947
+ .state_estimate = "O(1)",
1072
2948
  .args = anomaly_args,
1073
- .n_args = 3,
2949
+ .n_args = 5,
1074
2950
  .infer_schema = infer_schema_passthrough,
1075
2951
  .create_native = (void *(*)(const cJSON *))tf_anomaly_create,
1076
2952
  },
@@ -1079,6 +2955,10 @@ static tf_op_entry builtin_ops[] = {
1079
2955
  .kind = TF_OP_TRANSFORM,
1080
2956
  .tier = TF_TIER_CORE,
1081
2957
  .caps = TF_CAP_STREAMING | TF_CAP_BOUNDED_MEMORY | TF_CAP_BROWSER_SAFE,
2958
+ .memory_class = TF_MEM_BOUNDED_STATE,
2959
+ .emit_class = TF_EMIT_PER_BATCH,
2960
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2961
+ .state_estimate = "O(1)",
1082
2962
  .args = split_data_args,
1083
2963
  .n_args = 3,
1084
2964
  .infer_schema = infer_schema_passthrough,
@@ -1089,8 +2969,12 @@ static tf_op_entry builtin_ops[] = {
1089
2969
  .kind = TF_OP_TRANSFORM,
1090
2970
  .tier = TF_TIER_CORE,
1091
2971
  .caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2972
+ .memory_class = TF_MEM_BLOCKING,
2973
+ .emit_class = TF_EMIT_ON_FLUSH,
2974
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2975
+ .state_estimate = "O(input_rows)",
1092
2976
  .args = interpolate_args,
1093
- .n_args = 2,
2977
+ .n_args = 4,
1094
2978
  .infer_schema = infer_schema_passthrough,
1095
2979
  .create_native = (void *(*)(const cJSON *))tf_interpolate_create,
1096
2980
  },
@@ -1099,8 +2983,12 @@ static tf_op_entry builtin_ops[] = {
1099
2983
  .kind = TF_OP_TRANSFORM,
1100
2984
  .tier = TF_TIER_CORE,
1101
2985
  .caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
2986
+ .memory_class = TF_MEM_BLOCKING,
2987
+ .emit_class = TF_EMIT_ON_FLUSH,
2988
+ .schema_class = TF_SCHEMA_PARAMETRIC,
2989
+ .state_estimate = "O(input_rows)",
1102
2990
  .args = normalize_args,
1103
- .n_args = 2,
2991
+ .n_args = 12,
1104
2992
  .infer_schema = infer_schema_passthrough,
1105
2993
  .create_native = (void *(*)(const cJSON *))tf_normalize_create,
1106
2994
  },
@@ -1109,8 +2997,12 @@ static tf_op_entry builtin_ops[] = {
1109
2997
  .kind = TF_OP_TRANSFORM,
1110
2998
  .tier = TF_TIER_CORE,
1111
2999
  .caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
3000
+ .memory_class = TF_MEM_BLOCKING,
3001
+ .emit_class = TF_EMIT_ON_FLUSH,
3002
+ .schema_class = TF_SCHEMA_PARAMETRIC,
3003
+ .state_estimate = "O(input_rows)",
1112
3004
  .args = acf_args,
1113
- .n_args = 2,
3005
+ .n_args = 4,
1114
3006
  .infer_schema = infer_schema_passthrough,
1115
3007
  .create_native = (void *(*)(const cJSON *))tf_acf_create,
1116
3008
  },
@@ -1119,6 +3011,10 @@ static tf_op_entry builtin_ops[] = {
1119
3011
  .kind = TF_OP_ENCODER,
1120
3012
  .tier = TF_TIER_CORE,
1121
3013
  .caps = TF_CAP_BROWSER_SAFE | TF_CAP_DETERMINISTIC,
3014
+ .memory_class = TF_MEM_BLOCKING,
3015
+ .emit_class = TF_EMIT_ON_FLUSH,
3016
+ .schema_class = TF_SCHEMA_STABLE,
3017
+ .state_estimate = "O(input_rows)",
1122
3018
  .args = table_encode_args,
1123
3019
  .n_args = 2,
1124
3020
  .infer_schema = infer_schema_sink,