tranfi 0.1.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -0
- package/NOTICE +8 -0
- package/README.md +443 -51
- package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
- package/app/index.html +1 -1
- package/binding.gyp +55 -3
- package/csrc/arena.c +7 -5
- package/csrc/batch.c +818 -71
- package/csrc/buffer.c +84 -8
- package/csrc/cJSON.c +262 -19
- package/csrc/cJSON.h +17 -1
- package/csrc/codec_csv.c +1074 -181
- package/csrc/codec_jsonl.c +830 -118
- package/csrc/codec_table.c +108 -78
- package/csrc/codec_text.c +286 -68
- package/csrc/compiler.c +31 -3
- package/csrc/config.h +21 -0
- package/csrc/dsl.c +4722 -485
- package/csrc/expr.c +363 -55
- package/csrc/expr.h +2 -0
- package/csrc/internal.h +316 -27
- package/csrc/ir.c +65 -18
- package/csrc/ir.h +41 -0
- package/csrc/ir_schema.c +20 -5
- package/csrc/ir_serialize.c +68 -6
- package/csrc/ir_sql.c +796 -185
- package/csrc/ir_validate.c +462 -6
- package/csrc/json_path.c +210 -0
- package/csrc/main.c +879 -30
- package/csrc/memory_estimate.c +477 -0
- package/csrc/op_acf.c +171 -21
- package/csrc/op_across.c +477 -0
- package/csrc/op_anomaly.c +167 -32
- package/csrc/op_assert.c +761 -0
- package/csrc/op_bin.c +168 -29
- package/csrc/op_cast.c +383 -55
- package/csrc/op_clip.c +30 -19
- package/csrc/op_date_trunc.c +208 -34
- package/csrc/op_datetime.c +259 -77
- package/csrc/op_derive.c +65 -97
- package/csrc/op_diff.c +146 -30
- package/csrc/op_ewma.c +149 -30
- package/csrc/op_explode.c +124 -26
- package/csrc/op_fill_down.c +125 -53
- package/csrc/op_fill_null.c +176 -31
- package/csrc/op_filter.c +89 -40
- package/csrc/op_frequency.c +571 -43
- package/csrc/op_grep.c +36 -18
- package/csrc/op_group_agg.c +1790 -119
- package/csrc/op_hash.c +48 -15
- package/csrc/op_head.c +21 -86
- package/csrc/op_interpolate.c +268 -62
- package/csrc/op_join.c +2700 -182
- package/csrc/op_json_extract.c +227 -0
- package/csrc/op_json_filter.c +384 -0
- package/csrc/op_json_flatten.c +293 -0
- package/csrc/op_json_schema.c +503 -0
- package/csrc/op_label_encode.c +328 -53
- package/csrc/op_lag.c +181 -0
- package/csrc/op_lead.c +141 -89
- package/csrc/op_normalize.c +363 -79
- package/csrc/op_onehot.c +345 -73
- package/csrc/op_pivot.c +1546 -162
- package/csrc/op_quarantine.c +189 -0
- package/csrc/op_registry.c +2062 -166
- package/csrc/op_rename.c +41 -50
- package/csrc/op_replace.c +270 -118
- package/csrc/op_rleid.c +297 -0
- package/csrc/op_rowid.c +559 -0
- package/csrc/op_sample.c +80 -23
- package/csrc/op_schema.c +1341 -0
- package/csrc/op_schema_infer.c +252 -0
- package/csrc/op_select.c +265 -65
- package/csrc/op_set.c +3449 -0
- package/csrc/op_skip.c +30 -87
- package/csrc/op_sort.c +670 -124
- package/csrc/op_source_name.c +120 -0
- package/csrc/op_split.c +65 -28
- package/csrc/op_split_data.c +41 -9
- package/csrc/op_stack.c +178 -222
- package/csrc/op_stats.c +206 -110
- package/csrc/op_step.c +217 -55
- package/csrc/op_tail.c +21 -12
- package/csrc/op_tee.c +338 -0
- package/csrc/op_top.c +260 -53
- package/csrc/op_trim.c +48 -19
- package/csrc/op_unique.c +1193 -150
- package/csrc/op_unpivot.c +100 -66
- package/csrc/op_validate.c +601 -24
- package/csrc/op_window.c +492 -51
- package/csrc/path_policy.c +85 -0
- package/csrc/pipeline.c +872 -99
- package/csrc/recipes.c +3 -1
- package/csrc/report.c +73 -30
- package/csrc/selector.c +1097 -0
- package/csrc/size_utils.c +352 -0
- package/csrc/spill.c +317 -0
- package/csrc/spill.h +21 -0
- package/csrc/tranfi.h +169 -1
- package/csrc/transform.h +209 -0
- package/csrc/transform_api.c +2237 -0
- package/csrc/transform_categorical.c +923 -0
- package/csrc/transform_internal.h +472 -0
- package/csrc/transform_json.c +3812 -0
- package/csrc/transform_numeric.c +1966 -0
- package/csrc/transform_sha256.c +154 -0
- package/csrc/transform_wasm.h +162 -0
- package/csrc/transform_wasm_api.c +1373 -0
- package/csrc/wasm_api.c +70 -9
- package/napi_api.c +219 -11
- package/napi_transform.c +1648 -0
- package/napi_transform.h +8 -0
- package/package.json +27 -11
- package/scripts/install-native.js +76 -0
- package/scripts/prepack.js +64 -0
- package/scripts/sync-csrc.js +23 -0
- package/src/cli.js +81 -41
- package/src/engines/duckdb.js +45 -12
- package/src/index.js +661 -42
- package/src/memory_policy.js +411 -0
- package/src/native.js +1 -5
- package/src/pipeline.js +454 -31
- package/src/recipe_json.js +80 -0
- package/src/server.js +10 -8
- package/src/transform.js +403 -0
- package/src/transform_error.js +10 -0
- package/src/wasm.js +6 -4
- package/wasm/index.js +498 -10
- package/wasm/tranfi_core.js +0 -0
- package/wasm/transform.js +1156 -0
- package/wasm/worker.js +786 -0
- package/csrc/plan.c +0 -206
package/csrc/codec_table.c
CHANGED
|
@@ -16,10 +16,8 @@
|
|
|
16
16
|
|
|
17
17
|
#include "internal.h"
|
|
18
18
|
#include "cJSON.h"
|
|
19
|
-
#include "date_utils.h"
|
|
20
19
|
#include <stdlib.h>
|
|
21
20
|
#include <string.h>
|
|
22
|
-
#include <stdio.h>
|
|
23
21
|
|
|
24
22
|
#define TABLE_MAX_COLS 256
|
|
25
23
|
#define TABLE_DEFAULT_WIDTH 40
|
|
@@ -36,33 +34,57 @@ typedef struct {
|
|
|
36
34
|
|
|
37
35
|
/* Format a cell value as a string. Caller must free. */
|
|
38
36
|
static char *cell_to_string(tf_batch *b, size_t row, size_t col) {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
char tsbuf[40];
|
|
60
|
-
tf_timestamp_format(tf_batch_get_timestamp(b, row, col), tsbuf, sizeof(tsbuf));
|
|
61
|
-
return strdup(tsbuf);
|
|
37
|
+
char buf[64];
|
|
38
|
+
const char *text = NULL;
|
|
39
|
+
if (tf_batch_format_cell_as_string(b, row, col, TF_CELL_STRING_HUMAN,
|
|
40
|
+
buf, sizeof(buf), &text) != TF_OK) {
|
|
41
|
+
return NULL;
|
|
42
|
+
}
|
|
43
|
+
return strdup(text ? text : "");
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
static int table_capture_schema(table_encoder_state *st, const tf_batch *in) {
|
|
47
|
+
size_t n_cols = in->n_cols < TABLE_MAX_COLS ? in->n_cols : TABLE_MAX_COLS;
|
|
48
|
+
if (n_cols == 0) return TF_OK;
|
|
49
|
+
char **names = tf_callocarray_checked(n_cols, sizeof(char *));
|
|
50
|
+
if (!names) return TF_ERROR;
|
|
51
|
+
for (size_t i = 0; i < n_cols; i++) {
|
|
52
|
+
names[i] = strdup(in->col_names[i] ? in->col_names[i] : "");
|
|
53
|
+
if (!names[i]) {
|
|
54
|
+
for (size_t j = 0; j < i; j++) free(names[j]);
|
|
55
|
+
free(names);
|
|
56
|
+
return TF_ERROR;
|
|
62
57
|
}
|
|
63
|
-
default:
|
|
64
|
-
return strdup("");
|
|
65
58
|
}
|
|
59
|
+
st->col_names = names;
|
|
60
|
+
st->n_cols = n_cols;
|
|
61
|
+
return TF_OK;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
static int table_ensure_value_capacity(table_encoder_state *st, size_t min_rows) {
|
|
65
|
+
if (st->n_cols == 0 || min_rows <= st->capacity) return TF_OK;
|
|
66
|
+
size_t new_cap = 0;
|
|
67
|
+
size_t cells = 0;
|
|
68
|
+
if (tf_size_grow_pow2(st->capacity, min_rows, 64, &new_cap) != TF_OK ||
|
|
69
|
+
tf_size_mul(new_cap, st->n_cols, &cells) != TF_OK) {
|
|
70
|
+
return TF_ERROR;
|
|
71
|
+
}
|
|
72
|
+
char **new_values = tf_reallocarray_checked(st->values, cells, sizeof(char *));
|
|
73
|
+
if (!new_values) return TF_ERROR;
|
|
74
|
+
st->values = new_values;
|
|
75
|
+
st->capacity = new_cap;
|
|
76
|
+
return TF_OK;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
static int table_write_repeat(tf_buffer *out, char ch, size_t count) {
|
|
80
|
+
char pad[256];
|
|
81
|
+
memset(pad, ch, sizeof(pad));
|
|
82
|
+
while (count > 0) {
|
|
83
|
+
size_t chunk = count < sizeof(pad) ? count : sizeof(pad);
|
|
84
|
+
if (tf_buffer_write(out, (const uint8_t *)pad, chunk) != TF_OK) return TF_ERROR;
|
|
85
|
+
count -= chunk;
|
|
86
|
+
}
|
|
87
|
+
return TF_OK;
|
|
66
88
|
}
|
|
67
89
|
|
|
68
90
|
static int table_encode(tf_encoder *self, tf_batch *in, tf_buffer *out) {
|
|
@@ -71,29 +93,32 @@ static int table_encode(tf_encoder *self, tf_batch *in, tf_buffer *out) {
|
|
|
71
93
|
|
|
72
94
|
/* Capture column names on first batch */
|
|
73
95
|
if (!st->col_names && in->n_cols > 0) {
|
|
74
|
-
st
|
|
75
|
-
st->col_names = malloc(st->n_cols * sizeof(char *));
|
|
76
|
-
for (size_t i = 0; i < st->n_cols; i++) {
|
|
77
|
-
st->col_names[i] = strdup(in->col_names[i]);
|
|
78
|
-
}
|
|
96
|
+
if (table_capture_schema(st, in) != TF_OK) return TF_ERROR;
|
|
79
97
|
}
|
|
80
98
|
|
|
81
99
|
/* Buffer all cell values as strings */
|
|
82
100
|
for (size_t r = 0; r < in->n_rows; r++) {
|
|
83
101
|
if (st->max_rows > 0 && st->n_rows >= st->max_rows) break;
|
|
84
|
-
|
|
85
|
-
if (st->n_rows
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
if (!st->values) return TF_ERROR;
|
|
89
|
-
st->capacity = new_cap;
|
|
102
|
+
size_t next_rows = 0;
|
|
103
|
+
if (tf_size_add(st->n_rows, 1, &next_rows) != TF_OK ||
|
|
104
|
+
table_ensure_value_capacity(st, next_rows) != TF_OK) {
|
|
105
|
+
return TF_ERROR;
|
|
90
106
|
}
|
|
91
107
|
|
|
92
|
-
size_t base =
|
|
93
|
-
|
|
108
|
+
size_t base = 0;
|
|
109
|
+
if (tf_size_mul(st->n_rows, st->n_cols, &base) != TF_OK) return TF_ERROR;
|
|
110
|
+
size_t c = 0;
|
|
111
|
+
for (; c < st->n_cols; c++) {
|
|
94
112
|
st->values[base + c] = cell_to_string(in, r, c);
|
|
113
|
+
if (!st->values[base + c]) {
|
|
114
|
+
for (size_t j = 0; j < c; j++) {
|
|
115
|
+
free(st->values[base + j]);
|
|
116
|
+
st->values[base + j] = NULL;
|
|
117
|
+
}
|
|
118
|
+
return TF_ERROR;
|
|
119
|
+
}
|
|
95
120
|
}
|
|
96
|
-
st->n_rows
|
|
121
|
+
st->n_rows = next_rows;
|
|
97
122
|
}
|
|
98
123
|
|
|
99
124
|
return TF_OK;
|
|
@@ -104,7 +129,8 @@ static int table_flush(tf_encoder *self, tf_buffer *out) {
|
|
|
104
129
|
if (!st->col_names || st->n_cols == 0) return TF_OK;
|
|
105
130
|
|
|
106
131
|
/* Compute column widths */
|
|
107
|
-
size_t *widths =
|
|
132
|
+
size_t *widths = tf_callocarray_checked(st->n_cols, sizeof(size_t));
|
|
133
|
+
if (!widths) return TF_ERROR;
|
|
108
134
|
for (size_t c = 0; c < st->n_cols; c++) {
|
|
109
135
|
widths[c] = strlen(st->col_names[c]);
|
|
110
136
|
}
|
|
@@ -120,59 +146,55 @@ static int table_flush(tf_encoder *self, tf_buffer *out) {
|
|
|
120
146
|
widths[c] = st->max_width;
|
|
121
147
|
}
|
|
122
148
|
|
|
123
|
-
|
|
149
|
+
#define TABLE_WRITE(data, len) \
|
|
150
|
+
do { \
|
|
151
|
+
if (tf_buffer_write(out, (const uint8_t *)(data), (len)) != TF_OK) goto fail; \
|
|
152
|
+
} while (0)
|
|
124
153
|
|
|
125
154
|
/* Header row */
|
|
126
|
-
|
|
155
|
+
TABLE_WRITE("| ", 2);
|
|
127
156
|
for (size_t c = 0; c < st->n_cols; c++) {
|
|
128
|
-
if (c > 0)
|
|
157
|
+
if (c > 0) TABLE_WRITE(" | ", 3);
|
|
129
158
|
const char *name = st->col_names[c];
|
|
130
159
|
size_t nlen = strlen(name);
|
|
131
160
|
size_t w = widths[c];
|
|
132
161
|
if (nlen > w) nlen = w;
|
|
133
|
-
|
|
134
|
-
if (nlen < w)
|
|
135
|
-
size_t plen = w - nlen;
|
|
136
|
-
if (plen > sizeof(pad)) plen = sizeof(pad);
|
|
137
|
-
memset(pad, ' ', plen);
|
|
138
|
-
tf_buffer_write(out, (const uint8_t *)pad, plen);
|
|
139
|
-
}
|
|
162
|
+
TABLE_WRITE(name, nlen);
|
|
163
|
+
if (nlen < w && table_write_repeat(out, ' ', w - nlen) != TF_OK) goto fail;
|
|
140
164
|
}
|
|
141
|
-
|
|
165
|
+
TABLE_WRITE(" |\n", 3);
|
|
142
166
|
|
|
143
167
|
/* Separator row */
|
|
144
|
-
|
|
168
|
+
TABLE_WRITE("| ", 2);
|
|
145
169
|
for (size_t c = 0; c < st->n_cols; c++) {
|
|
146
|
-
if (c > 0)
|
|
147
|
-
|
|
148
|
-
if (w > sizeof(pad)) w = sizeof(pad);
|
|
149
|
-
memset(pad, '-', w);
|
|
150
|
-
tf_buffer_write(out, (const uint8_t *)pad, w);
|
|
170
|
+
if (c > 0) TABLE_WRITE(" | ", 3);
|
|
171
|
+
if (table_write_repeat(out, '-', widths[c]) != TF_OK) goto fail;
|
|
151
172
|
}
|
|
152
|
-
|
|
173
|
+
TABLE_WRITE(" |\n", 3);
|
|
153
174
|
|
|
154
175
|
/* Data rows */
|
|
155
176
|
for (size_t r = 0; r < st->n_rows; r++) {
|
|
156
|
-
|
|
177
|
+
TABLE_WRITE("| ", 2);
|
|
157
178
|
for (size_t c = 0; c < st->n_cols; c++) {
|
|
158
|
-
if (c > 0)
|
|
179
|
+
if (c > 0) TABLE_WRITE(" | ", 3);
|
|
159
180
|
const char *val = st->values[r * st->n_cols + c];
|
|
160
181
|
size_t vlen = strlen(val);
|
|
161
182
|
size_t w = widths[c];
|
|
162
183
|
if (vlen > w) vlen = w;
|
|
163
|
-
|
|
164
|
-
if (vlen < w)
|
|
165
|
-
size_t plen = w - vlen;
|
|
166
|
-
if (plen > sizeof(pad)) plen = sizeof(pad);
|
|
167
|
-
memset(pad, ' ', plen);
|
|
168
|
-
tf_buffer_write(out, (const uint8_t *)pad, plen);
|
|
169
|
-
}
|
|
184
|
+
TABLE_WRITE(val, vlen);
|
|
185
|
+
if (vlen < w && table_write_repeat(out, ' ', w - vlen) != TF_OK) goto fail;
|
|
170
186
|
}
|
|
171
|
-
|
|
187
|
+
TABLE_WRITE(" |\n", 3);
|
|
172
188
|
}
|
|
173
189
|
|
|
190
|
+
#undef TABLE_WRITE
|
|
174
191
|
free(widths);
|
|
175
192
|
return TF_OK;
|
|
193
|
+
|
|
194
|
+
fail:
|
|
195
|
+
#undef TABLE_WRITE
|
|
196
|
+
free(widths);
|
|
197
|
+
return TF_ERROR;
|
|
176
198
|
}
|
|
177
199
|
|
|
178
200
|
static void table_encoder_destroy(tf_encoder *self) {
|
|
@@ -183,7 +205,10 @@ static void table_encoder_destroy(tf_encoder *self) {
|
|
|
183
205
|
free(st->col_names);
|
|
184
206
|
}
|
|
185
207
|
if (st->values) {
|
|
186
|
-
|
|
208
|
+
size_t n_cells = 0;
|
|
209
|
+
if (tf_size_mul(st->n_rows, st->n_cols, &n_cells) == TF_OK) {
|
|
210
|
+
for (size_t i = 0; i < n_cells; i++) free(st->values[i]);
|
|
211
|
+
}
|
|
187
212
|
free(st->values);
|
|
188
213
|
}
|
|
189
214
|
free(st);
|
|
@@ -199,13 +224,18 @@ tf_encoder *tf_table_encoder_create(const cJSON *args) {
|
|
|
199
224
|
st->max_rows = 0;
|
|
200
225
|
|
|
201
226
|
if (args) {
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
if (
|
|
208
|
-
|
|
227
|
+
size_t parsed_size = 0;
|
|
228
|
+
int has_max_width = tf_json_get_size_arg(args, "max_width",
|
|
229
|
+
1, TF_MAX_TABLE_WIDTH,
|
|
230
|
+
&parsed_size, "table");
|
|
231
|
+
if (has_max_width < 0) { free(st); return NULL; }
|
|
232
|
+
if (has_max_width > 0) st->max_width = parsed_size;
|
|
233
|
+
|
|
234
|
+
int has_max_rows = tf_json_get_size_arg(args, "max_rows",
|
|
235
|
+
0, TF_MAX_TABLE_ROWS,
|
|
236
|
+
&parsed_size, "table");
|
|
237
|
+
if (has_max_rows < 0) { free(st); return NULL; }
|
|
238
|
+
if (has_max_rows > 0) st->max_rows = parsed_size;
|
|
209
239
|
}
|
|
210
240
|
|
|
211
241
|
tf_encoder *enc = malloc(sizeof(tf_encoder));
|