tranfi 0.1.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -0
- package/NOTICE +8 -0
- package/README.md +443 -51
- package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
- package/app/index.html +1 -1
- package/binding.gyp +55 -3
- package/csrc/arena.c +7 -5
- package/csrc/batch.c +818 -71
- package/csrc/buffer.c +84 -8
- package/csrc/cJSON.c +262 -19
- package/csrc/cJSON.h +17 -1
- package/csrc/codec_csv.c +1074 -181
- package/csrc/codec_jsonl.c +830 -118
- package/csrc/codec_table.c +108 -78
- package/csrc/codec_text.c +286 -68
- package/csrc/compiler.c +31 -3
- package/csrc/config.h +21 -0
- package/csrc/dsl.c +4722 -485
- package/csrc/expr.c +363 -55
- package/csrc/expr.h +2 -0
- package/csrc/internal.h +316 -27
- package/csrc/ir.c +65 -18
- package/csrc/ir.h +41 -0
- package/csrc/ir_schema.c +20 -5
- package/csrc/ir_serialize.c +68 -6
- package/csrc/ir_sql.c +796 -185
- package/csrc/ir_validate.c +462 -6
- package/csrc/json_path.c +210 -0
- package/csrc/main.c +879 -30
- package/csrc/memory_estimate.c +477 -0
- package/csrc/op_acf.c +171 -21
- package/csrc/op_across.c +477 -0
- package/csrc/op_anomaly.c +167 -32
- package/csrc/op_assert.c +761 -0
- package/csrc/op_bin.c +168 -29
- package/csrc/op_cast.c +383 -55
- package/csrc/op_clip.c +30 -19
- package/csrc/op_date_trunc.c +208 -34
- package/csrc/op_datetime.c +259 -77
- package/csrc/op_derive.c +65 -97
- package/csrc/op_diff.c +146 -30
- package/csrc/op_ewma.c +149 -30
- package/csrc/op_explode.c +124 -26
- package/csrc/op_fill_down.c +125 -53
- package/csrc/op_fill_null.c +176 -31
- package/csrc/op_filter.c +89 -40
- package/csrc/op_frequency.c +571 -43
- package/csrc/op_grep.c +36 -18
- package/csrc/op_group_agg.c +1790 -119
- package/csrc/op_hash.c +48 -15
- package/csrc/op_head.c +21 -86
- package/csrc/op_interpolate.c +268 -62
- package/csrc/op_join.c +2700 -182
- package/csrc/op_json_extract.c +227 -0
- package/csrc/op_json_filter.c +384 -0
- package/csrc/op_json_flatten.c +293 -0
- package/csrc/op_json_schema.c +503 -0
- package/csrc/op_label_encode.c +328 -53
- package/csrc/op_lag.c +181 -0
- package/csrc/op_lead.c +141 -89
- package/csrc/op_normalize.c +363 -79
- package/csrc/op_onehot.c +345 -73
- package/csrc/op_pivot.c +1546 -162
- package/csrc/op_quarantine.c +189 -0
- package/csrc/op_registry.c +2062 -166
- package/csrc/op_rename.c +41 -50
- package/csrc/op_replace.c +270 -118
- package/csrc/op_rleid.c +297 -0
- package/csrc/op_rowid.c +559 -0
- package/csrc/op_sample.c +80 -23
- package/csrc/op_schema.c +1341 -0
- package/csrc/op_schema_infer.c +252 -0
- package/csrc/op_select.c +265 -65
- package/csrc/op_set.c +3449 -0
- package/csrc/op_skip.c +30 -87
- package/csrc/op_sort.c +670 -124
- package/csrc/op_source_name.c +120 -0
- package/csrc/op_split.c +65 -28
- package/csrc/op_split_data.c +41 -9
- package/csrc/op_stack.c +178 -222
- package/csrc/op_stats.c +206 -110
- package/csrc/op_step.c +217 -55
- package/csrc/op_tail.c +21 -12
- package/csrc/op_tee.c +338 -0
- package/csrc/op_top.c +260 -53
- package/csrc/op_trim.c +48 -19
- package/csrc/op_unique.c +1193 -150
- package/csrc/op_unpivot.c +100 -66
- package/csrc/op_validate.c +601 -24
- package/csrc/op_window.c +492 -51
- package/csrc/path_policy.c +85 -0
- package/csrc/pipeline.c +872 -99
- package/csrc/recipes.c +3 -1
- package/csrc/report.c +73 -30
- package/csrc/selector.c +1097 -0
- package/csrc/size_utils.c +352 -0
- package/csrc/spill.c +317 -0
- package/csrc/spill.h +21 -0
- package/csrc/tranfi.h +169 -1
- package/csrc/transform.h +209 -0
- package/csrc/transform_api.c +2237 -0
- package/csrc/transform_categorical.c +923 -0
- package/csrc/transform_internal.h +472 -0
- package/csrc/transform_json.c +3812 -0
- package/csrc/transform_numeric.c +1966 -0
- package/csrc/transform_sha256.c +154 -0
- package/csrc/transform_wasm.h +162 -0
- package/csrc/transform_wasm_api.c +1373 -0
- package/csrc/wasm_api.c +70 -9
- package/napi_api.c +219 -11
- package/napi_transform.c +1648 -0
- package/napi_transform.h +8 -0
- package/package.json +27 -11
- package/scripts/install-native.js +76 -0
- package/scripts/prepack.js +64 -0
- package/scripts/sync-csrc.js +23 -0
- package/src/cli.js +81 -41
- package/src/engines/duckdb.js +45 -12
- package/src/index.js +661 -42
- package/src/memory_policy.js +411 -0
- package/src/native.js +1 -5
- package/src/pipeline.js +454 -31
- package/src/recipe_json.js +80 -0
- package/src/server.js +10 -8
- package/src/transform.js +403 -0
- package/src/transform_error.js +10 -0
- package/src/wasm.js +6 -4
- package/wasm/index.js +498 -10
- package/wasm/tranfi_core.js +0 -0
- package/wasm/transform.js +1156 -0
- package/wasm/worker.js +786 -0
- package/csrc/plan.c +0 -206
package/csrc/op_grep.c
CHANGED
|
@@ -34,15 +34,22 @@ static int grep_process(tf_step *self, tf_batch *in, tf_batch **out,
|
|
|
34
34
|
if (col_idx < 0) {
|
|
35
35
|
/* Column not found — pass through or drop all depending on invert */
|
|
36
36
|
if (st->invert) {
|
|
37
|
-
*out = in;
|
|
38
|
-
/* Don't free — caller owns it, but we need to copy */
|
|
39
37
|
tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
|
|
40
38
|
if (!ob) return TF_ERROR;
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
39
|
+
if (tf_batch_clone_schema(ob, in) != TF_OK) {
|
|
40
|
+
tf_batch_free(ob);
|
|
41
|
+
return TF_ERROR;
|
|
42
|
+
}
|
|
43
|
+
for (size_t r = 0; r < in->n_rows; r++) {
|
|
44
|
+
if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
|
|
45
|
+
tf_batch_free(ob);
|
|
46
|
+
return TF_ERROR;
|
|
47
|
+
}
|
|
48
|
+
if (tf_batch_expose_row(ob, r) != TF_OK) {
|
|
49
|
+
tf_batch_free(ob);
|
|
50
|
+
return TF_ERROR;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
46
53
|
*out = ob;
|
|
47
54
|
}
|
|
48
55
|
return TF_OK;
|
|
@@ -51,8 +58,10 @@ static int grep_process(tf_step *self, tf_batch *in, tf_batch **out,
|
|
|
51
58
|
/* Create output batch with same schema */
|
|
52
59
|
tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
|
|
53
60
|
if (!ob) return TF_ERROR;
|
|
54
|
-
|
|
55
|
-
|
|
61
|
+
if (tf_batch_clone_schema(ob, in) != TF_OK) {
|
|
62
|
+
tf_batch_free(ob);
|
|
63
|
+
return TF_ERROR;
|
|
64
|
+
}
|
|
56
65
|
|
|
57
66
|
size_t out_row = 0;
|
|
58
67
|
for (size_t r = 0; r < in->n_rows; r++) {
|
|
@@ -77,10 +86,16 @@ static int grep_process(tf_step *self, tf_batch *in, tf_batch **out,
|
|
|
77
86
|
return TF_ERROR;
|
|
78
87
|
}
|
|
79
88
|
|
|
80
|
-
tf_batch_copy_row(ob, out_row, in, r)
|
|
89
|
+
if (tf_batch_copy_row(ob, out_row, in, r) != TF_OK) {
|
|
90
|
+
tf_batch_free(ob);
|
|
91
|
+
return TF_ERROR;
|
|
92
|
+
}
|
|
93
|
+
if (tf_batch_expose_row(ob, out_row) != TF_OK) {
|
|
94
|
+
tf_batch_free(ob);
|
|
95
|
+
return TF_ERROR;
|
|
96
|
+
}
|
|
81
97
|
out_row++;
|
|
82
98
|
}
|
|
83
|
-
ob->n_rows = out_row;
|
|
84
99
|
|
|
85
100
|
st->rows_in += in->n_rows;
|
|
86
101
|
st->rows_out += out_row;
|
|
@@ -88,9 +103,12 @@ static int grep_process(tf_step *self, tf_batch *in, tf_batch **out,
|
|
|
88
103
|
if (side && side->stats) {
|
|
89
104
|
char stats_buf[128];
|
|
90
105
|
snprintf(stats_buf, sizeof(stats_buf),
|
|
91
|
-
"{\"op\":\"grep\",\"rows_in\":%zu,\"rows_out\":%zu}
|
|
106
|
+
"{\"op\":\"grep\",\"rows_in\":%zu,\"rows_out\":%zu}",
|
|
92
107
|
in->n_rows, out_row);
|
|
93
|
-
|
|
108
|
+
if (tf_buffer_write_line(side->stats, stats_buf) != TF_OK) {
|
|
109
|
+
tf_batch_free(ob);
|
|
110
|
+
return TF_ERROR;
|
|
111
|
+
}
|
|
94
112
|
}
|
|
95
113
|
|
|
96
114
|
if (out_row > 0) {
|
|
@@ -124,17 +142,17 @@ tf_step *tf_grep_create(const cJSON *args) {
|
|
|
124
142
|
cJSON *pattern_json = cJSON_GetObjectItemCaseSensitive(args, "pattern");
|
|
125
143
|
if (!cJSON_IsString(pattern_json)) return NULL;
|
|
126
144
|
|
|
127
|
-
grep_state *st =
|
|
145
|
+
grep_state *st = tf_callocarray_checked(1, sizeof(grep_state));
|
|
128
146
|
if (!st) return NULL;
|
|
129
147
|
|
|
130
|
-
st->pattern =
|
|
148
|
+
st->pattern = tf_strdup_checked(pattern_json->valuestring);
|
|
131
149
|
if (!st->pattern) { free(st); return NULL; }
|
|
132
150
|
|
|
133
151
|
cJSON *column_json = cJSON_GetObjectItemCaseSensitive(args, "column");
|
|
134
152
|
if (cJSON_IsString(column_json))
|
|
135
|
-
st->column =
|
|
153
|
+
st->column = tf_strdup_checked(column_json->valuestring);
|
|
136
154
|
else
|
|
137
|
-
st->column =
|
|
155
|
+
st->column = tf_strdup_checked("_line");
|
|
138
156
|
if (!st->column) { free(st->pattern); free(st); return NULL; }
|
|
139
157
|
|
|
140
158
|
cJSON *invert_json = cJSON_GetObjectItemCaseSensitive(args, "invert");
|
|
@@ -153,7 +171,7 @@ tf_step *tf_grep_create(const cJSON *args) {
|
|
|
153
171
|
}
|
|
154
172
|
}
|
|
155
173
|
|
|
156
|
-
tf_step *step =
|
|
174
|
+
tf_step *step = tf_callocarray_checked(1, sizeof(tf_step));
|
|
157
175
|
if (!step) { free(st->pattern); free(st->column); free(st); return NULL; }
|
|
158
176
|
step->process = grep_process;
|
|
159
177
|
step->flush = grep_flush;
|