tranfi 0.1.2 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/LICENSE +177 -0
  2. package/NOTICE +8 -0
  3. package/README.md +443 -51
  4. package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
  5. package/app/index.html +1 -1
  6. package/binding.gyp +55 -3
  7. package/csrc/arena.c +7 -5
  8. package/csrc/batch.c +818 -71
  9. package/csrc/buffer.c +84 -8
  10. package/csrc/cJSON.c +262 -19
  11. package/csrc/cJSON.h +17 -1
  12. package/csrc/codec_csv.c +1074 -181
  13. package/csrc/codec_jsonl.c +830 -118
  14. package/csrc/codec_table.c +108 -78
  15. package/csrc/codec_text.c +286 -68
  16. package/csrc/compiler.c +31 -3
  17. package/csrc/config.h +21 -0
  18. package/csrc/dsl.c +4722 -485
  19. package/csrc/expr.c +363 -55
  20. package/csrc/expr.h +2 -0
  21. package/csrc/internal.h +316 -27
  22. package/csrc/ir.c +65 -18
  23. package/csrc/ir.h +41 -0
  24. package/csrc/ir_schema.c +20 -5
  25. package/csrc/ir_serialize.c +68 -6
  26. package/csrc/ir_sql.c +796 -185
  27. package/csrc/ir_validate.c +462 -6
  28. package/csrc/json_path.c +210 -0
  29. package/csrc/main.c +879 -30
  30. package/csrc/memory_estimate.c +477 -0
  31. package/csrc/op_acf.c +171 -21
  32. package/csrc/op_across.c +477 -0
  33. package/csrc/op_anomaly.c +167 -32
  34. package/csrc/op_assert.c +761 -0
  35. package/csrc/op_bin.c +168 -29
  36. package/csrc/op_cast.c +383 -55
  37. package/csrc/op_clip.c +30 -19
  38. package/csrc/op_date_trunc.c +208 -34
  39. package/csrc/op_datetime.c +259 -77
  40. package/csrc/op_derive.c +65 -97
  41. package/csrc/op_diff.c +146 -30
  42. package/csrc/op_ewma.c +149 -30
  43. package/csrc/op_explode.c +124 -26
  44. package/csrc/op_fill_down.c +125 -53
  45. package/csrc/op_fill_null.c +176 -31
  46. package/csrc/op_filter.c +89 -40
  47. package/csrc/op_frequency.c +571 -43
  48. package/csrc/op_grep.c +36 -18
  49. package/csrc/op_group_agg.c +1790 -119
  50. package/csrc/op_hash.c +48 -15
  51. package/csrc/op_head.c +21 -86
  52. package/csrc/op_interpolate.c +268 -62
  53. package/csrc/op_join.c +2700 -182
  54. package/csrc/op_json_extract.c +227 -0
  55. package/csrc/op_json_filter.c +384 -0
  56. package/csrc/op_json_flatten.c +293 -0
  57. package/csrc/op_json_schema.c +503 -0
  58. package/csrc/op_label_encode.c +328 -53
  59. package/csrc/op_lag.c +181 -0
  60. package/csrc/op_lead.c +141 -89
  61. package/csrc/op_normalize.c +363 -79
  62. package/csrc/op_onehot.c +345 -73
  63. package/csrc/op_pivot.c +1546 -162
  64. package/csrc/op_quarantine.c +189 -0
  65. package/csrc/op_registry.c +2062 -166
  66. package/csrc/op_rename.c +41 -50
  67. package/csrc/op_replace.c +270 -118
  68. package/csrc/op_rleid.c +297 -0
  69. package/csrc/op_rowid.c +559 -0
  70. package/csrc/op_sample.c +80 -23
  71. package/csrc/op_schema.c +1341 -0
  72. package/csrc/op_schema_infer.c +252 -0
  73. package/csrc/op_select.c +265 -65
  74. package/csrc/op_set.c +3449 -0
  75. package/csrc/op_skip.c +30 -87
  76. package/csrc/op_sort.c +670 -124
  77. package/csrc/op_source_name.c +120 -0
  78. package/csrc/op_split.c +65 -28
  79. package/csrc/op_split_data.c +41 -9
  80. package/csrc/op_stack.c +178 -222
  81. package/csrc/op_stats.c +206 -110
  82. package/csrc/op_step.c +217 -55
  83. package/csrc/op_tail.c +21 -12
  84. package/csrc/op_tee.c +338 -0
  85. package/csrc/op_top.c +260 -53
  86. package/csrc/op_trim.c +48 -19
  87. package/csrc/op_unique.c +1193 -150
  88. package/csrc/op_unpivot.c +100 -66
  89. package/csrc/op_validate.c +601 -24
  90. package/csrc/op_window.c +492 -51
  91. package/csrc/path_policy.c +85 -0
  92. package/csrc/pipeline.c +872 -99
  93. package/csrc/recipes.c +3 -1
  94. package/csrc/report.c +73 -30
  95. package/csrc/selector.c +1097 -0
  96. package/csrc/size_utils.c +352 -0
  97. package/csrc/spill.c +317 -0
  98. package/csrc/spill.h +21 -0
  99. package/csrc/tranfi.h +169 -1
  100. package/csrc/transform.h +209 -0
  101. package/csrc/transform_api.c +2237 -0
  102. package/csrc/transform_categorical.c +923 -0
  103. package/csrc/transform_internal.h +472 -0
  104. package/csrc/transform_json.c +3812 -0
  105. package/csrc/transform_numeric.c +1966 -0
  106. package/csrc/transform_sha256.c +154 -0
  107. package/csrc/transform_wasm.h +162 -0
  108. package/csrc/transform_wasm_api.c +1373 -0
  109. package/csrc/wasm_api.c +70 -9
  110. package/napi_api.c +219 -11
  111. package/napi_transform.c +1648 -0
  112. package/napi_transform.h +8 -0
  113. package/package.json +27 -11
  114. package/scripts/install-native.js +76 -0
  115. package/scripts/prepack.js +64 -0
  116. package/scripts/sync-csrc.js +23 -0
  117. package/src/cli.js +81 -41
  118. package/src/engines/duckdb.js +45 -12
  119. package/src/index.js +661 -42
  120. package/src/memory_policy.js +411 -0
  121. package/src/native.js +1 -5
  122. package/src/pipeline.js +454 -31
  123. package/src/recipe_json.js +80 -0
  124. package/src/server.js +10 -8
  125. package/src/transform.js +403 -0
  126. package/src/transform_error.js +10 -0
  127. package/src/wasm.js +6 -4
  128. package/wasm/index.js +498 -10
  129. package/wasm/tranfi_core.js +0 -0
  130. package/wasm/transform.js +1156 -0
  131. package/wasm/worker.js +786 -0
  132. package/csrc/plan.c +0 -206
@@ -1,7 +1,11 @@
1
1
  /*
2
2
  * op_label_encode.c — Map categorical values to sequential integers.
3
3
  *
4
- * Config: {"column": "city", "result": "city_encoded"}
4
+ * Config: {"column": "city", "result": "city_encoded",
5
+ * "categories": ["Paris", "London"],
6
+ * "max_categories": 1000,
7
+ * "max_state_bytes": 1048576,
8
+ * "unknown": "error" | "other" | "null"}
5
9
  */
6
10
 
7
11
  #include "internal.h"
@@ -10,20 +14,69 @@
10
14
  #include <string.h>
11
15
  #include <stdio.h>
12
16
 
17
+ typedef enum {
18
+ TF_CAT_UNKNOWN_ADD,
19
+ TF_CAT_UNKNOWN_ERROR,
20
+ TF_CAT_UNKNOWN_OTHER,
21
+ TF_CAT_UNKNOWN_NULL
22
+ } tf_cat_unknown_policy;
23
+
13
24
  typedef struct label_entry {
14
25
  char *value;
15
26
  int64_t label;
16
27
  } label_entry;
17
28
 
18
29
  typedef struct {
19
- char *column;
20
- char *result;
21
- label_entry *entries;
22
- size_t n_entries;
23
- size_t cap;
24
- int64_t next_label;
30
+ char *column;
31
+ char *result;
32
+ size_t max_categories; /* 0 = unlimited */
33
+ size_t max_state_bytes; /* 0 = unlimited */
34
+ int categories_declared;
35
+ tf_cat_unknown_policy unknown;
36
+ label_entry *entries;
37
+ size_t n_entries;
38
+ size_t cap;
39
+ int64_t next_label;
25
40
  } label_encode_state;
26
41
 
42
+ static const char *OTHER_CATEGORY = "__other__";
43
+
44
+ static size_t label_retained_state_bytes(const label_encode_state *st);
45
+
46
+ static int label_write_error(tf_side_channels *side, const char *msg) {
47
+ return tf_side_write_error(side, msg);
48
+ }
49
+
50
+ static int label_unknown_error(const label_encode_state *st, const char *val,
51
+ tf_side_channels *side) {
52
+ char msg[256];
53
+ snprintf(msg, sizeof(msg),
54
+ "label-encode: unknown category '%s' for column '%s'",
55
+ val ? val : "", st->column ? st->column : "");
56
+ return label_write_error(side, msg);
57
+ }
58
+
59
+ static int label_limit_error(const label_encode_state *st, const char *val,
60
+ tf_side_channels *side) {
61
+ char msg[256];
62
+ snprintf(msg, sizeof(msg),
63
+ "label-encode: max_categories=%zu exceeded while tracking category '%s'",
64
+ st->max_categories, val ? val : "");
65
+ return label_write_error(side, msg);
66
+ }
67
+
68
+ static int label_check_state_bytes(label_encode_state *st, tf_side_channels *side) {
69
+ if (!st || st->max_state_bytes == 0) return TF_OK;
70
+ size_t retained = label_retained_state_bytes(st);
71
+ if (retained <= st->max_state_bytes) return TF_OK;
72
+ char msg[208];
73
+ snprintf(msg, sizeof(msg),
74
+ "label-encode: max_state_bytes=%zu exceeded while tracking categories (%zu bytes retained)",
75
+ st->max_state_bytes, retained);
76
+ if (label_write_error(side, msg) != TF_OK) return TF_ERROR;
77
+ return TF_ERROR;
78
+ }
79
+
27
80
  static const char *get_string_value(const tf_batch *b, size_t r, int ci, char *buf, size_t bufsz) {
28
81
  if (tf_batch_is_null(b, r, ci)) return NULL;
29
82
  switch (b->col_types[ci]) {
@@ -40,59 +93,154 @@ static const char *get_string_value(const tf_batch *b, size_t r, int ci, char *b
40
93
  }
41
94
  }
42
95
 
43
- static int64_t lookup_or_assign(label_encode_state *st, const char *val) {
96
+ static int find_entry(const label_encode_state *st, const char *val) {
44
97
  for (size_t i = 0; i < st->n_entries; i++) {
45
- if (strcmp(st->entries[i].value, val) == 0)
46
- return st->entries[i].label;
98
+ if (strcmp(st->entries[i].value, val) == 0) return (int)i;
47
99
  }
48
- /* New value — assign next label */
100
+ return -1;
101
+ }
102
+
103
+ static int assign_entry(label_encode_state *st, const char *val) {
104
+ int existing = find_entry(st, val);
105
+ if (existing >= 0) return existing;
49
106
  if (st->n_entries >= st->cap) {
50
- size_t newcap = st->cap ? st->cap * 2 : 16;
51
- label_entry *tmp = realloc(st->entries, newcap * sizeof(label_entry));
107
+ size_t min_cap = 0, newcap = 0;
108
+ if (tf_size_add(st->n_entries, 1, &min_cap) != TF_OK ||
109
+ tf_size_grow_pow2(st->cap, min_cap, 16, &newcap) != TF_OK) {
110
+ return -1;
111
+ }
112
+ label_entry *tmp = tf_reallocarray_checked(st->entries, newcap,
113
+ sizeof(label_entry));
52
114
  if (!tmp) return -1;
53
115
  st->entries = tmp;
54
116
  st->cap = newcap;
55
117
  }
56
- st->entries[st->n_entries].value = strdup(val);
57
- st->entries[st->n_entries].label = st->next_label;
118
+ char *value = tf_strdup_checked(val);
119
+ if (!value) return -1;
120
+ st->entries[st->n_entries].value = value;
121
+ st->entries[st->n_entries].label = st->next_label++;
58
122
  st->n_entries++;
59
- return st->next_label++;
123
+ return (int)(st->n_entries - 1);
124
+ }
125
+
126
+ static int resolve_other(label_encode_state *st, int64_t *label,
127
+ tf_side_channels *side) {
128
+ int other = find_entry(st, OTHER_CATEGORY);
129
+ if (other >= 0) { *label = st->entries[other].label; return 0; }
130
+ if (st->max_categories > 0 && st->n_entries >= st->max_categories) {
131
+ if (label_limit_error(st, OTHER_CATEGORY, side) != TF_OK) return -1;
132
+ return -1;
133
+ }
134
+ other = assign_entry(st, OTHER_CATEGORY);
135
+ if (other < 0) return -1;
136
+ if (label_check_state_bytes(st, side) != TF_OK) return -1;
137
+ *label = st->entries[other].label;
138
+ return 0;
139
+ }
140
+
141
+ static int resolve_unknown(label_encode_state *st, const char *val,
142
+ int64_t *label, int *is_null,
143
+ tf_side_channels *side) {
144
+ switch (st->unknown) {
145
+ case TF_CAT_UNKNOWN_NULL:
146
+ *is_null = 1;
147
+ return 0;
148
+ case TF_CAT_UNKNOWN_OTHER:
149
+ return resolve_other(st, label, side);
150
+ case TF_CAT_UNKNOWN_ERROR:
151
+ if (label_unknown_error(st, val, side) != TF_OK) return -1;
152
+ return -1;
153
+ case TF_CAT_UNKNOWN_ADD:
154
+ default:
155
+ break;
156
+ }
157
+ return 1;
158
+ }
159
+
160
+ static int resolve_label(label_encode_state *st, const char *val,
161
+ int64_t *label, int *is_null,
162
+ tf_side_channels *side) {
163
+ *label = 0;
164
+ *is_null = 0;
165
+ if (!val) { *is_null = 1; return 0; }
166
+
167
+ int existing = find_entry(st, val);
168
+ if (existing >= 0) { *label = st->entries[existing].label; return 0; }
169
+
170
+ if (st->categories_declared) {
171
+ int rc = resolve_unknown(st, val, label, is_null, side);
172
+ return rc == 0 ? 0 : -1;
173
+ }
174
+
175
+ if (st->max_categories > 0 && st->unknown == TF_CAT_UNKNOWN_OTHER) {
176
+ int other = find_entry(st, OTHER_CATEGORY);
177
+ size_t reserve = other >= 0 ? 0 : 1;
178
+ if (st->n_entries + reserve >= st->max_categories) {
179
+ return resolve_other(st, label, side);
180
+ }
181
+ }
182
+
183
+ if (st->max_categories > 0 && st->n_entries >= st->max_categories) {
184
+ int rc = resolve_unknown(st, val, label, is_null, side);
185
+ if (rc == 0) return 0;
186
+ if (label_limit_error(st, val, side) != TF_OK) return -1;
187
+ return -1;
188
+ }
189
+
190
+ int added = assign_entry(st, val);
191
+ if (added < 0) return -1;
192
+ if (label_check_state_bytes(st, side) != TF_OK) return -1;
193
+ *label = st->entries[added].label;
194
+ return 0;
60
195
  }
61
196
 
62
197
  static int label_encode_process(tf_step *self, tf_batch *in, tf_batch **out,
63
198
  tf_side_channels *side) {
64
- (void)side;
65
199
  label_encode_state *st = self->state;
66
200
  *out = NULL;
67
201
 
68
- tf_batch *ob = tf_batch_create(in->n_cols + 1, in->n_rows);
202
+ size_t out_cols = 0;
203
+ if (tf_size_add(in->n_cols, 1, &out_cols) != TF_OK) return TF_ERROR;
204
+ tf_batch *ob = tf_batch_create(out_cols, in->n_rows);
69
205
  if (!ob) return TF_ERROR;
70
- for (size_t c = 0; c < in->n_cols; c++)
71
- tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
72
- tf_batch_set_schema(ob, in->n_cols, st->result, TF_TYPE_INT64);
206
+ const char *extra_names[1] = { st->result };
207
+ const tf_type extra_types[1] = { TF_TYPE_INT64 };
208
+ if (tf_batch_clone_with_extra_cols(ob, in, extra_names, extra_types, 1) != TF_OK) {
209
+ tf_batch_free(ob);
210
+ return TF_ERROR;
211
+ }
73
212
 
74
213
  int ci = tf_batch_col_index(in, st->column);
75
214
 
76
215
  for (size_t r = 0; r < in->n_rows; r++) {
77
- tf_batch_copy_row(ob, r, in, r);
216
+ if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
217
+ tf_batch_free(ob);
218
+ return TF_ERROR;
219
+ }
78
220
 
221
+ int rc = TF_OK;
79
222
  if (ci < 0 || tf_batch_is_null(in, r, ci)) {
80
- tf_batch_set_null(ob, r, in->n_cols);
81
- ob->n_rows = r + 1;
82
- continue;
223
+ rc = tf_batch_set_null(ob, r, in->n_cols);
224
+ } else {
225
+ char buf[64];
226
+ const char *val = get_string_value(in, r, ci, buf, sizeof(buf));
227
+ int64_t label = 0;
228
+ int is_null = 0;
229
+ if (resolve_label(st, val, &label, &is_null, side) != 0) {
230
+ tf_batch_free(ob);
231
+ return TF_ERROR;
232
+ }
233
+ rc = is_null ? tf_batch_set_null(ob, r, in->n_cols)
234
+ : tf_batch_set_int64(ob, r, in->n_cols, label);
83
235
  }
84
-
85
- char buf[64];
86
- const char *val = get_string_value(in, r, ci, buf, sizeof(buf));
87
- if (!val) {
88
- tf_batch_set_null(ob, r, in->n_cols);
89
- ob->n_rows = r + 1;
90
- continue;
236
+ if (rc != TF_OK) {
237
+ tf_batch_free(ob);
238
+ return TF_ERROR;
239
+ }
240
+ if (tf_batch_expose_row(ob, r) != TF_OK) {
241
+ tf_batch_free(ob);
242
+ return TF_ERROR;
91
243
  }
92
-
93
- int64_t label = lookup_or_assign(st, val);
94
- tf_batch_set_int64(ob, r, in->n_cols, label);
95
- ob->n_rows = r + 1;
96
244
  }
97
245
 
98
246
  *out = ob;
@@ -103,17 +251,71 @@ static int label_encode_flush(tf_step *self, tf_batch **out, tf_side_channels *s
103
251
  (void)self; (void)side; *out = NULL; return TF_OK;
104
252
  }
105
253
 
106
- static void label_encode_destroy(tf_step *self) {
254
+ static size_t label_category_value_bytes(const label_encode_state *st) {
255
+ size_t total = 0;
256
+ if (!st) return 0;
257
+ for (size_t i = 0; i < st->n_entries; i++) {
258
+ if (st->entries[i].value) total += strlen(st->entries[i].value) + 1;
259
+ }
260
+ return total;
261
+ }
262
+
263
+ static size_t label_retained_state_bytes(const label_encode_state *st) {
264
+ if (!st) return 0;
265
+ size_t total = st->cap * sizeof(label_entry);
266
+ total += label_category_value_bytes(st);
267
+ if (st->column) total += strlen(st->column) + 1;
268
+ if (st->result) total += strlen(st->result) + 1;
269
+ return total;
270
+ }
271
+
272
+ static int label_encode_append_stats(tf_step *self, tf_buffer *out) {
273
+ if (!self || !self->state || !out) return TF_ERROR;
107
274
  label_encode_state *st = self->state;
108
- if (st) {
109
- for (size_t i = 0; i < st->n_entries; i++)
110
- free(st->entries[i].value);
111
- free(st->entries);
112
- free(st->column);
113
- free(st->result);
114
- free(st);
275
+ char buf[280];
276
+ snprintf(buf, sizeof(buf),
277
+ ",\"tracked_categories\":%zu,\"category_value_bytes\":%zu,"
278
+ "\"retained_state_bytes\":%zu,\"max_state_bytes\":%zu",
279
+ st->n_entries, label_category_value_bytes(st),
280
+ label_retained_state_bytes(st), st->max_state_bytes);
281
+ return tf_buffer_write_str(out, buf);
282
+ }
283
+
284
+ static void label_encode_state_free(label_encode_state *st) {
285
+ if (!st) return;
286
+ for (size_t i = 0; i < st->n_entries; i++)
287
+ free(st->entries[i].value);
288
+ free(st->entries);
289
+ free(st->column);
290
+ free(st->result);
291
+ free(st);
292
+ }
293
+
294
+ static void label_encode_destroy(tf_step *self) {
295
+ if (self) {
296
+ label_encode_state_free(self->state);
297
+ free(self);
115
298
  }
116
- free(self);
299
+ }
300
+
301
+ static int parse_unknown_policy(const cJSON *args, tf_cat_unknown_policy *policy,
302
+ int *specified) {
303
+ cJSON *unknown_j = cJSON_GetObjectItemCaseSensitive(args, "unknown");
304
+ *specified = 0;
305
+ if (!unknown_j) return 0;
306
+ if (!cJSON_IsString(unknown_j)) {
307
+ tf_set_last_error("label-encode: unknown must be one of error, other, null");
308
+ return -1;
309
+ }
310
+ *specified = 1;
311
+ if (strcmp(unknown_j->valuestring, "error") == 0) *policy = TF_CAT_UNKNOWN_ERROR;
312
+ else if (strcmp(unknown_j->valuestring, "other") == 0) *policy = TF_CAT_UNKNOWN_OTHER;
313
+ else if (strcmp(unknown_j->valuestring, "null") == 0) *policy = TF_CAT_UNKNOWN_NULL;
314
+ else {
315
+ tf_set_last_error("label-encode: unknown must be one of error, other, null");
316
+ return -1;
317
+ }
318
+ return 0;
117
319
  }
118
320
 
119
321
  tf_step *tf_label_encode_create(const cJSON *args) {
@@ -121,23 +323,96 @@ tf_step *tf_label_encode_create(const cJSON *args) {
121
323
  cJSON *col_j = cJSON_GetObjectItemCaseSensitive(args, "column");
122
324
  if (!cJSON_IsString(col_j)) return NULL;
123
325
 
124
- label_encode_state *st = calloc(1, sizeof(label_encode_state));
326
+ label_encode_state *st = tf_callocarray_checked(1, sizeof(label_encode_state));
125
327
  if (!st) return NULL;
126
- st->column = strdup(col_j->valuestring);
328
+ st->column = tf_strdup_checked(col_j->valuestring);
329
+ if (!st->column) { label_encode_state_free(st); return NULL; }
330
+ st->unknown = TF_CAT_UNKNOWN_ADD;
127
331
 
128
332
  cJSON *res_j = cJSON_GetObjectItemCaseSensitive(args, "result");
129
333
  if (cJSON_IsString(res_j)) {
130
- st->result = strdup(res_j->valuestring);
334
+ st->result = tf_strdup_checked(res_j->valuestring);
131
335
  } else {
132
- char buf[256];
133
- snprintf(buf, sizeof(buf), "%s_encoded", st->column);
134
- st->result = strdup(buf);
336
+ st->result = tf_string_append_suffix_checked(st->column, "_encoded");
337
+ }
338
+ if (!st->result) { label_encode_state_free(st); return NULL; }
339
+
340
+ size_t parsed_size = 0;
341
+ int has_max_categories = tf_json_get_size_arg(args, "max_categories",
342
+ 1, TF_MAX_COUNT_ARG,
343
+ &parsed_size, "label-encode");
344
+ if (has_max_categories < 0) { label_encode_state_free(st); return NULL; }
345
+ if (has_max_categories > 0) st->max_categories = parsed_size;
346
+
347
+ int has_max_state = tf_json_get_size_arg(args, "max_state_bytes",
348
+ 1, TF_MAX_STATE_BYTES,
349
+ &parsed_size, "label-encode");
350
+ if (has_max_state < 0) { label_encode_state_free(st); return NULL; }
351
+ if (has_max_state > 0) st->max_state_bytes = parsed_size;
352
+
353
+ int unknown_specified = 0;
354
+ if (parse_unknown_policy(args, &st->unknown, &unknown_specified) != 0) {
355
+ label_encode_state_free(st);
356
+ return NULL;
357
+ }
358
+
359
+ cJSON *cats_j = cJSON_GetObjectItemCaseSensitive(args, "categories");
360
+ if (cats_j) {
361
+ if (!cJSON_IsArray(cats_j)) {
362
+ tf_set_last_error("label-encode: categories must be an array");
363
+ label_encode_state_free(st);
364
+ return NULL;
365
+ }
366
+ st->categories_declared = 1;
367
+ cJSON *item = NULL;
368
+ cJSON_ArrayForEach(item, cats_j) {
369
+ if (!cJSON_IsString(item)) {
370
+ tf_set_last_error("label-encode: categories must contain strings");
371
+ label_encode_state_free(st);
372
+ return NULL;
373
+ }
374
+ if (find_entry(st, item->valuestring) < 0 &&
375
+ st->max_categories > 0 && st->n_entries >= st->max_categories) {
376
+ tf_set_last_error("label-encode: categories exceed max_categories");
377
+ label_encode_state_free(st);
378
+ return NULL;
379
+ }
380
+ if (assign_entry(st, item->valuestring) < 0) {
381
+ label_encode_state_free(st);
382
+ return NULL;
383
+ }
384
+ if (label_check_state_bytes(st, NULL) != TF_OK) {
385
+ label_encode_state_free(st);
386
+ return NULL;
387
+ }
388
+ }
389
+ }
390
+
391
+ if (st->categories_declared && !unknown_specified)
392
+ st->unknown = TF_CAT_UNKNOWN_ERROR;
393
+
394
+ if (st->categories_declared && st->unknown == TF_CAT_UNKNOWN_OTHER &&
395
+ find_entry(st, OTHER_CATEGORY) < 0) {
396
+ if (st->max_categories > 0 && st->n_entries >= st->max_categories) {
397
+ tf_set_last_error("label-encode: max_categories leaves no room for other category");
398
+ label_encode_state_free(st);
399
+ return NULL;
400
+ }
401
+ if (assign_entry(st, OTHER_CATEGORY) < 0) {
402
+ label_encode_state_free(st);
403
+ return NULL;
404
+ }
405
+ if (label_check_state_bytes(st, NULL) != TF_OK) {
406
+ label_encode_state_free(st);
407
+ return NULL;
408
+ }
135
409
  }
136
410
 
137
- tf_step *step = malloc(sizeof(tf_step));
138
- if (!step) { free(st->column); free(st->result); free(st); return NULL; }
411
+ tf_step *step = tf_callocarray_checked(1, sizeof(tf_step));
412
+ if (!step) { label_encode_state_free(st); return NULL; }
139
413
  step->process = label_encode_process;
140
414
  step->flush = label_encode_flush;
415
+ step->append_stats = label_encode_append_stats;
141
416
  step->destroy = label_encode_destroy;
142
417
  step->state = st;
143
418
  return step;
package/csrc/op_lag.c ADDED
@@ -0,0 +1,181 @@
1
+ /*
2
+ * op_lag.c - Bounded previous-row shift.
3
+ *
4
+ * Config: {"column": "price", "offset": 1, "result": "prev_price"}
5
+ *
6
+ * Appends the value from `offset` rows behind the current row. State retained
7
+ * across batches is a ring buffer of the previous `offset` full rows so the
8
+ * appended column preserves the source type and nulls.
9
+ */
10
+
11
+ #include "internal.h"
12
+ #include "cJSON.h"
13
+ #include <stdlib.h>
14
+ #include <string.h>
15
+ #include <stdio.h>
16
+
17
+ /* tf_shift_create supports type="lead" by delegating to the existing lead op. */
18
+ tf_step *tf_lead_create(const cJSON *args);
19
+
20
+ typedef struct {
21
+ char *column;
22
+ char *result;
23
+ size_t offset;
24
+ tf_batch *history;
25
+ size_t hist_count;
26
+ size_t hist_pos;
27
+ } lag_state;
28
+
29
+ static int lag_set_error(tf_side_channels *side, const char *msg) {
30
+ return tf_side_write_error(side, msg);
31
+ }
32
+
33
+ static int ensure_history(lag_state *st, const tf_batch *in) {
34
+ if (st->history) return TF_OK;
35
+
36
+ tf_batch *h = tf_batch_create(in->n_cols, st->offset);
37
+ if (!h) return TF_ERROR;
38
+ if (tf_batch_clone_schema(h, in) != TF_OK) {
39
+ tf_batch_free(h);
40
+ return TF_ERROR;
41
+ }
42
+ if (tf_batch_expose_row(h, st->offset - 1) != TF_OK) {
43
+ tf_batch_free(h);
44
+ return TF_ERROR;
45
+ }
46
+ st->history = h;
47
+ return TF_OK;
48
+ }
49
+
50
+ static int lag_process(tf_step *self, tf_batch *in, tf_batch **out,
51
+ tf_side_channels *side) {
52
+ lag_state *st = self->state;
53
+ *out = NULL;
54
+
55
+ int ci = tf_batch_col_index(in, st->column);
56
+ if (ci < 0) {
57
+ char msg[256];
58
+ snprintf(msg, sizeof(msg), "lag: column '%s' not found", st->column);
59
+ if (lag_set_error(side, msg) != TF_OK) return TF_ERROR;
60
+ return TF_ERROR;
61
+ }
62
+ if (ensure_history(st, in) != TF_OK) return TF_ERROR;
63
+
64
+ const char *extra_names[1] = {st->result};
65
+ tf_type extra_types[1] = {in->col_types[ci]};
66
+ tf_batch *ob = tf_batch_create(in->n_cols + 1, in->n_rows);
67
+ if (!ob) return TF_ERROR;
68
+ if (tf_batch_clone_with_extra_cols(ob, in, extra_names, extra_types, 1) != TF_OK) {
69
+ tf_batch_free(ob);
70
+ return TF_ERROR;
71
+ }
72
+
73
+ for (size_t r = 0; r < in->n_rows; r++) {
74
+ if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
75
+ tf_batch_free(ob);
76
+ return TF_ERROR;
77
+ }
78
+
79
+ if (st->hist_count < st->offset) {
80
+ if (tf_batch_set_null(ob, r, in->n_cols) != TF_OK) {
81
+ tf_batch_free(ob);
82
+ return TF_ERROR;
83
+ }
84
+ } else if (tf_batch_copy_cell(ob, r, in->n_cols, st->history, st->hist_pos, (size_t)ci) != TF_OK) {
85
+ tf_batch_free(ob);
86
+ return TF_ERROR;
87
+ }
88
+
89
+ if (tf_batch_copy_row(st->history, st->hist_pos, in, r) != TF_OK) {
90
+ tf_batch_free(ob);
91
+ return TF_ERROR;
92
+ }
93
+ if (tf_batch_expose_row(ob, r) != TF_OK) {
94
+ tf_batch_free(ob);
95
+ return TF_ERROR;
96
+ }
97
+ st->hist_pos = (st->hist_pos + 1) % st->offset;
98
+ if (st->hist_count < st->offset) st->hist_count++;
99
+ }
100
+
101
+ *out = ob;
102
+ return TF_OK;
103
+ }
104
+
105
+ static int lag_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
106
+ (void)self;
107
+ (void)side;
108
+ *out = NULL;
109
+ return TF_OK;
110
+ }
111
+
112
+ static void lag_destroy(tf_step *self) {
113
+ lag_state *st = self->state;
114
+ if (st) {
115
+ free(st->column);
116
+ free(st->result);
117
+ if (st->history) tf_batch_free(st->history);
118
+ free(st);
119
+ }
120
+ free(self);
121
+ }
122
+
123
+ static tf_step *lag_create_with_suffix(const cJSON *args, const char *suffix) {
124
+ if (!args) return NULL;
125
+ cJSON *col_j = cJSON_GetObjectItemCaseSensitive(args, "column");
126
+ if (!cJSON_IsString(col_j) || !col_j->valuestring[0]) return NULL;
127
+
128
+ lag_state *st = tf_callocarray_checked(1, sizeof(lag_state));
129
+ if (!st) return NULL;
130
+ st->column = tf_strdup_checked(col_j->valuestring);
131
+ if (!st->column) { free(st); return NULL; }
132
+
133
+ size_t offset = 1;
134
+ int has_offset = tf_json_get_size_arg(args, "offset",
135
+ 1, TF_MAX_WINDOW_SIZE,
136
+ &offset, "lag");
137
+ if (has_offset < 0) { free(st->column); free(st); return NULL; }
138
+ st->offset = offset;
139
+
140
+ cJSON *res_j = cJSON_GetObjectItemCaseSensitive(args, "result");
141
+ if (cJSON_IsString(res_j) && res_j->valuestring[0]) {
142
+ st->result = tf_strdup_checked(res_j->valuestring);
143
+ } else {
144
+ char suffix_buf[64];
145
+ int n = snprintf(suffix_buf, sizeof(suffix_buf), "_%s", suffix ? suffix : "");
146
+ if (n < 0 || (size_t)n >= sizeof(suffix_buf)) {
147
+ free(st->column);
148
+ free(st);
149
+ return NULL;
150
+ }
151
+ st->result = tf_string_append_suffix_checked(st->column, suffix_buf);
152
+ }
153
+ if (!st->result) { free(st->column); free(st); return NULL; }
154
+
155
+ tf_step *step = tf_callocarray_checked(1, sizeof(tf_step));
156
+ if (!step) {
157
+ free(st->column);
158
+ free(st->result);
159
+ free(st);
160
+ return NULL;
161
+ }
162
+ step->process = lag_process;
163
+ step->flush = lag_flush;
164
+ step->destroy = lag_destroy;
165
+ step->state = st;
166
+ return step;
167
+ }
168
+
169
+ tf_step *tf_lag_create(const cJSON *args) {
170
+ return lag_create_with_suffix(args, "lag");
171
+ }
172
+
173
+ tf_step *tf_shift_create(const cJSON *args) {
174
+ cJSON *type_j = cJSON_GetObjectItemCaseSensitive(args, "type");
175
+ const char *type = cJSON_IsString(type_j) ? type_j->valuestring : "lag";
176
+ if (strcmp(type, "lead") == 0) return tf_lead_create(args);
177
+ if (strcmp(type, "lag") == 0 || strcmp(type, "shift") == 0) {
178
+ return lag_create_with_suffix(args, "shift");
179
+ }
180
+ return NULL;
181
+ }