tranfi 0.0.2 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +395 -0
- package/app/assets/index-6quYZ5Ap.css +5 -0
- package/app/assets/index-pDFMluyz.js +160 -0
- package/app/assets/materialdesignicons-webfont-B7mPwVP_.ttf +0 -0
- package/app/assets/materialdesignicons-webfont-CSr8KVlo.eot +0 -0
- package/app/assets/materialdesignicons-webfont-Dp5v-WZN.woff2 +0 -0
- package/app/assets/materialdesignicons-webfont-PXm3-2wK.woff +0 -0
- package/app/index.html +13 -0
- package/binding.gyp +69 -0
- package/csrc/arena.c +91 -0
- package/csrc/batch.c +229 -0
- package/csrc/buffer.c +78 -0
- package/csrc/cJSON.c +3143 -0
- package/csrc/cJSON.h +300 -0
- package/csrc/codec_csv.c +1058 -0
- package/csrc/codec_jsonl.c +374 -0
- package/csrc/codec_table.c +218 -0
- package/csrc/codec_text.c +229 -0
- package/csrc/compiler.c +102 -0
- package/csrc/date_utils.h +94 -0
- package/csrc/dsl.c +1180 -0
- package/csrc/dsl.h +22 -0
- package/csrc/expr.c +1245 -0
- package/csrc/expr.h +56 -0
- package/csrc/internal.h +250 -0
- package/csrc/ir.c +119 -0
- package/csrc/ir.h +167 -0
- package/csrc/ir_schema.c +60 -0
- package/csrc/ir_serialize.c +104 -0
- package/csrc/ir_sql.c +1211 -0
- package/csrc/ir_validate.c +120 -0
- package/csrc/main.c +392 -0
- package/csrc/op_acf.c +133 -0
- package/csrc/op_anomaly.c +120 -0
- package/csrc/op_bin.c +109 -0
- package/csrc/op_cast.c +195 -0
- package/csrc/op_clip.c +88 -0
- package/csrc/op_date_trunc.c +181 -0
- package/csrc/op_datetime.c +212 -0
- package/csrc/op_derive.c +248 -0
- package/csrc/op_diff.c +134 -0
- package/csrc/op_ewma.c +103 -0
- package/csrc/op_explode.c +108 -0
- package/csrc/op_fill_down.c +163 -0
- package/csrc/op_fill_null.c +123 -0
- package/csrc/op_filter.c +132 -0
- package/csrc/op_frequency.c +193 -0
- package/csrc/op_grep.c +163 -0
- package/csrc/op_group_agg.c +285 -0
- package/csrc/op_hash.c +126 -0
- package/csrc/op_head.c +149 -0
- package/csrc/op_interpolate.c +239 -0
- package/csrc/op_join.c +384 -0
- package/csrc/op_label_encode.c +144 -0
- package/csrc/op_lead.c +190 -0
- package/csrc/op_normalize.c +226 -0
- package/csrc/op_onehot.c +185 -0
- package/csrc/op_pivot.c +370 -0
- package/csrc/op_registry.c +1148 -0
- package/csrc/op_rename.c +138 -0
- package/csrc/op_replace.c +202 -0
- package/csrc/op_sample.c +101 -0
- package/csrc/op_select.c +140 -0
- package/csrc/op_skip.c +152 -0
- package/csrc/op_sort.c +273 -0
- package/csrc/op_split.c +114 -0
- package/csrc/op_split_data.c +87 -0
- package/csrc/op_stack.c +315 -0
- package/csrc/op_stats.c +779 -0
- package/csrc/op_step.c +171 -0
- package/csrc/op_tail.c +96 -0
- package/csrc/op_top.c +150 -0
- package/csrc/op_trim.c +109 -0
- package/csrc/op_unique.c +300 -0
- package/csrc/op_unpivot.c +159 -0
- package/csrc/op_validate.c +71 -0
- package/csrc/op_window.c +150 -0
- package/csrc/pipeline.c +315 -0
- package/csrc/plan.c +206 -0
- package/csrc/recipes.c +102 -0
- package/csrc/recipes.h +27 -0
- package/csrc/report.c +463 -0
- package/csrc/report.h +22 -0
- package/csrc/tranfi.h +123 -0
- package/csrc/wasm_api.c +157 -0
- package/napi_api.c +326 -0
- package/package.json +46 -57
- package/src/cli.js +193 -0
- package/src/engines/duckdb.js +109 -0
- package/src/index.js +306 -0
- package/src/native.js +22 -0
- package/src/pipeline.js +286 -0
- package/src/server.js +277 -0
- package/src/wasm.js +19 -0
- package/wasm/index.js +244 -0
- package/wasm/package.json +1 -0
- package/wasm/tranfi_core.js +0 -0
- package/LICENSE +0 -21
- package/dist/bundle.js +0 -1
- package/index.html +0 -18
- package/src/app.css +0 -169
- package/src/app.js +0 -203
- package/src/app.vue +0 -250
- package/src/bulma-input.vue +0 -110
- package/src/common-inputs.js +0 -28
- package/src/main.js +0 -20
- package/src/transforms.js +0 -166
- package/webpack.config.js +0 -108
package/csrc/op_unique.c
ADDED
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* op_unique.c — Deduplicate rows by key columns using a hash set.
|
|
3
|
+
*
|
|
4
|
+
* Config: {"columns": ["name", "city"]}
|
|
5
|
+
* or {} for dedup by all columns.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
#include "internal.h"
|
|
9
|
+
#include "cJSON.h"
|
|
10
|
+
#include "date_utils.h"
|
|
11
|
+
#include <stdlib.h>
|
|
12
|
+
#include <string.h>
|
|
13
|
+
#include <stdio.h>
|
|
14
|
+
|
|
15
|
+
/* ---- Simple hash set (open addressing) ---- */
|
|
16
|
+
|
|
17
|
+
typedef struct {
|
|
18
|
+
char **keys;
|
|
19
|
+
size_t count;
|
|
20
|
+
size_t cap;
|
|
21
|
+
} hash_set;
|
|
22
|
+
|
|
23
|
+
static void hs_init(hash_set *hs, size_t cap) {
|
|
24
|
+
hs->cap = cap;
|
|
25
|
+
hs->count = 0;
|
|
26
|
+
hs->keys = calloc(cap, sizeof(char *));
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
static uint32_t hs_hash(const char *key) {
|
|
30
|
+
uint32_t h = 5381;
|
|
31
|
+
for (const char *p = key; *p; p++)
|
|
32
|
+
h = ((h << 5) + h) ^ (uint32_t)(unsigned char)*p;
|
|
33
|
+
return h;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
static int hs_grow(hash_set *hs) {
|
|
37
|
+
size_t new_cap = hs->cap * 2;
|
|
38
|
+
char **new_keys = calloc(new_cap, sizeof(char *));
|
|
39
|
+
if (!new_keys) return -1;
|
|
40
|
+
|
|
41
|
+
for (size_t i = 0; i < hs->cap; i++) {
|
|
42
|
+
if (hs->keys[i]) {
|
|
43
|
+
uint32_t idx = hs_hash(hs->keys[i]) % new_cap;
|
|
44
|
+
while (new_keys[idx]) idx = (idx + 1) % new_cap;
|
|
45
|
+
new_keys[idx] = hs->keys[i];
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
free(hs->keys);
|
|
49
|
+
hs->keys = new_keys;
|
|
50
|
+
hs->cap = new_cap;
|
|
51
|
+
return 0;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/* Returns 1 if key was newly inserted, 0 if already present, -1 on error */
|
|
55
|
+
static int hs_insert(hash_set *hs, const char *key) {
|
|
56
|
+
if (hs->count * 4 >= hs->cap * 3) { /* load factor > 0.75 */
|
|
57
|
+
if (hs_grow(hs) != 0) return -1;
|
|
58
|
+
}
|
|
59
|
+
uint32_t idx = hs_hash(key) % hs->cap;
|
|
60
|
+
while (hs->keys[idx]) {
|
|
61
|
+
if (strcmp(hs->keys[idx], key) == 0) return 0; /* already exists */
|
|
62
|
+
idx = (idx + 1) % hs->cap;
|
|
63
|
+
}
|
|
64
|
+
hs->keys[idx] = strdup(key);
|
|
65
|
+
if (!hs->keys[idx]) return -1;
|
|
66
|
+
hs->count++;
|
|
67
|
+
return 1;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
static void hs_free(hash_set *hs) {
|
|
71
|
+
for (size_t i = 0; i < hs->cap; i++) free(hs->keys[i]);
|
|
72
|
+
free(hs->keys);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/* ---- Unique transform ---- */
|
|
76
|
+
|
|
77
|
+
typedef struct {
|
|
78
|
+
char **key_cols; /* column names to dedup by (NULL = all) */
|
|
79
|
+
size_t n_key_cols;
|
|
80
|
+
hash_set seen;
|
|
81
|
+
} unique_state;
|
|
82
|
+
|
|
83
|
+
static char *build_row_key(const tf_batch *b, size_t row,
|
|
84
|
+
int *col_indices, size_t n_keys) {
|
|
85
|
+
/* Build key by concatenating column values with \x01 separator */
|
|
86
|
+
size_t buf_cap = 256;
|
|
87
|
+
char *buf = malloc(buf_cap);
|
|
88
|
+
if (!buf) return NULL;
|
|
89
|
+
size_t buf_len = 0;
|
|
90
|
+
|
|
91
|
+
for (size_t k = 0; k < n_keys; k++) {
|
|
92
|
+
int c = col_indices[k];
|
|
93
|
+
if (c < 0) continue;
|
|
94
|
+
|
|
95
|
+
if (k > 0) {
|
|
96
|
+
if (buf_len + 1 >= buf_cap) {
|
|
97
|
+
buf_cap *= 2;
|
|
98
|
+
char *tmp = realloc(buf, buf_cap);
|
|
99
|
+
if (!tmp) { free(buf); return NULL; }
|
|
100
|
+
buf = tmp;
|
|
101
|
+
}
|
|
102
|
+
buf[buf_len++] = '\x01';
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
char val_buf[64];
|
|
106
|
+
const char *val = NULL;
|
|
107
|
+
size_t val_len = 0;
|
|
108
|
+
|
|
109
|
+
if (tf_batch_is_null(b, row, c)) {
|
|
110
|
+
val = "\\N";
|
|
111
|
+
val_len = 2;
|
|
112
|
+
} else {
|
|
113
|
+
switch (b->col_types[c]) {
|
|
114
|
+
case TF_TYPE_BOOL:
|
|
115
|
+
val = tf_batch_get_bool(b, row, c) ? "T" : "F";
|
|
116
|
+
val_len = 1;
|
|
117
|
+
break;
|
|
118
|
+
case TF_TYPE_INT64:
|
|
119
|
+
val_len = snprintf(val_buf, sizeof(val_buf), "%lld",
|
|
120
|
+
(long long)tf_batch_get_int64(b, row, c));
|
|
121
|
+
val = val_buf;
|
|
122
|
+
break;
|
|
123
|
+
case TF_TYPE_FLOAT64:
|
|
124
|
+
val_len = snprintf(val_buf, sizeof(val_buf), "%.17g",
|
|
125
|
+
tf_batch_get_float64(b, row, c));
|
|
126
|
+
val = val_buf;
|
|
127
|
+
break;
|
|
128
|
+
case TF_TYPE_STRING:
|
|
129
|
+
val = tf_batch_get_string(b, row, c);
|
|
130
|
+
val_len = strlen(val);
|
|
131
|
+
break;
|
|
132
|
+
case TF_TYPE_DATE:
|
|
133
|
+
val_len = snprintf(val_buf, sizeof(val_buf), "%d",
|
|
134
|
+
(int)tf_batch_get_date(b, row, c));
|
|
135
|
+
val = val_buf;
|
|
136
|
+
break;
|
|
137
|
+
case TF_TYPE_TIMESTAMP:
|
|
138
|
+
val_len = snprintf(val_buf, sizeof(val_buf), "%lld",
|
|
139
|
+
(long long)tf_batch_get_timestamp(b, row, c));
|
|
140
|
+
val = val_buf;
|
|
141
|
+
break;
|
|
142
|
+
default:
|
|
143
|
+
val = "\\N";
|
|
144
|
+
val_len = 2;
|
|
145
|
+
break;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
while (buf_len + val_len >= buf_cap) {
|
|
150
|
+
buf_cap *= 2;
|
|
151
|
+
char *tmp = realloc(buf, buf_cap);
|
|
152
|
+
if (!tmp) { free(buf); return NULL; }
|
|
153
|
+
buf = tmp;
|
|
154
|
+
}
|
|
155
|
+
memcpy(buf + buf_len, val, val_len);
|
|
156
|
+
buf_len += val_len;
|
|
157
|
+
}
|
|
158
|
+
buf[buf_len] = '\0';
|
|
159
|
+
return buf;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
static int unique_process(tf_step *self, tf_batch *in, tf_batch **out,
|
|
163
|
+
tf_side_channels *side) {
|
|
164
|
+
(void)side;
|
|
165
|
+
unique_state *st = self->state;
|
|
166
|
+
*out = NULL;
|
|
167
|
+
|
|
168
|
+
/* Resolve column indices */
|
|
169
|
+
size_t n_keys;
|
|
170
|
+
int *col_indices;
|
|
171
|
+
if (st->n_key_cols > 0) {
|
|
172
|
+
n_keys = st->n_key_cols;
|
|
173
|
+
col_indices = malloc(n_keys * sizeof(int));
|
|
174
|
+
if (!col_indices) return TF_ERROR;
|
|
175
|
+
for (size_t k = 0; k < n_keys; k++) {
|
|
176
|
+
col_indices[k] = tf_batch_col_index(in, st->key_cols[k]);
|
|
177
|
+
}
|
|
178
|
+
} else {
|
|
179
|
+
/* All columns */
|
|
180
|
+
n_keys = in->n_cols;
|
|
181
|
+
col_indices = malloc(n_keys * sizeof(int));
|
|
182
|
+
if (!col_indices) return TF_ERROR;
|
|
183
|
+
for (size_t k = 0; k < n_keys; k++) {
|
|
184
|
+
col_indices[k] = (int)k;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
|
|
189
|
+
if (!ob) { free(col_indices); return TF_ERROR; }
|
|
190
|
+
for (size_t c = 0; c < in->n_cols; c++) {
|
|
191
|
+
tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
size_t out_row = 0;
|
|
195
|
+
for (size_t r = 0; r < in->n_rows; r++) {
|
|
196
|
+
char *key = build_row_key(in, r, col_indices, n_keys);
|
|
197
|
+
if (!key) { free(col_indices); tf_batch_free(ob); return TF_ERROR; }
|
|
198
|
+
|
|
199
|
+
int inserted = hs_insert(&st->seen, key);
|
|
200
|
+
free(key);
|
|
201
|
+
if (inserted < 0) { free(col_indices); tf_batch_free(ob); return TF_ERROR; }
|
|
202
|
+
if (inserted == 0) continue; /* duplicate */
|
|
203
|
+
|
|
204
|
+
/* Copy row */
|
|
205
|
+
tf_batch_ensure_capacity(ob, out_row + 1);
|
|
206
|
+
for (size_t c = 0; c < in->n_cols; c++) {
|
|
207
|
+
if (tf_batch_is_null(in, r, c)) {
|
|
208
|
+
tf_batch_set_null(ob, out_row, c);
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
switch (in->col_types[c]) {
|
|
212
|
+
case TF_TYPE_BOOL:
|
|
213
|
+
tf_batch_set_bool(ob, out_row, c, tf_batch_get_bool(in, r, c));
|
|
214
|
+
break;
|
|
215
|
+
case TF_TYPE_INT64:
|
|
216
|
+
tf_batch_set_int64(ob, out_row, c, tf_batch_get_int64(in, r, c));
|
|
217
|
+
break;
|
|
218
|
+
case TF_TYPE_FLOAT64:
|
|
219
|
+
tf_batch_set_float64(ob, out_row, c, tf_batch_get_float64(in, r, c));
|
|
220
|
+
break;
|
|
221
|
+
case TF_TYPE_STRING:
|
|
222
|
+
tf_batch_set_string(ob, out_row, c, tf_batch_get_string(in, r, c));
|
|
223
|
+
break;
|
|
224
|
+
case TF_TYPE_DATE:
|
|
225
|
+
tf_batch_set_date(ob, out_row, c, tf_batch_get_date(in, r, c));
|
|
226
|
+
break;
|
|
227
|
+
case TF_TYPE_TIMESTAMP:
|
|
228
|
+
tf_batch_set_timestamp(ob, out_row, c, tf_batch_get_timestamp(in, r, c));
|
|
229
|
+
break;
|
|
230
|
+
default:
|
|
231
|
+
tf_batch_set_null(ob, out_row, c);
|
|
232
|
+
break;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
out_row++;
|
|
236
|
+
}
|
|
237
|
+
ob->n_rows = out_row;
|
|
238
|
+
free(col_indices);
|
|
239
|
+
|
|
240
|
+
if (out_row > 0) {
|
|
241
|
+
*out = ob;
|
|
242
|
+
} else {
|
|
243
|
+
tf_batch_free(ob);
|
|
244
|
+
}
|
|
245
|
+
return TF_OK;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
static int unique_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
|
|
249
|
+
(void)self; (void)side;
|
|
250
|
+
*out = NULL;
|
|
251
|
+
return TF_OK;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
static void unique_destroy(tf_step *self) {
|
|
255
|
+
unique_state *st = self->state;
|
|
256
|
+
if (st) {
|
|
257
|
+
for (size_t i = 0; i < st->n_key_cols; i++) free(st->key_cols[i]);
|
|
258
|
+
free(st->key_cols);
|
|
259
|
+
hs_free(&st->seen);
|
|
260
|
+
free(st);
|
|
261
|
+
}
|
|
262
|
+
free(self);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
tf_step *tf_unique_create(const cJSON *args) {
|
|
266
|
+
unique_state *st = calloc(1, sizeof(unique_state));
|
|
267
|
+
if (!st) return NULL;
|
|
268
|
+
hs_init(&st->seen, 256);
|
|
269
|
+
|
|
270
|
+
if (args) {
|
|
271
|
+
cJSON *columns = cJSON_GetObjectItemCaseSensitive(args, "columns");
|
|
272
|
+
if (columns && cJSON_IsArray(columns)) {
|
|
273
|
+
int n = cJSON_GetArraySize(columns);
|
|
274
|
+
if (n > 0) {
|
|
275
|
+
st->key_cols = calloc(n, sizeof(char *));
|
|
276
|
+
st->n_key_cols = n;
|
|
277
|
+
for (int i = 0; i < n; i++) {
|
|
278
|
+
cJSON *item = cJSON_GetArrayItem(columns, i);
|
|
279
|
+
if (cJSON_IsString(item)) {
|
|
280
|
+
st->key_cols[i] = strdup(item->valuestring);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
tf_step *step = malloc(sizeof(tf_step));
|
|
288
|
+
if (!step) {
|
|
289
|
+
for (size_t i = 0; i < st->n_key_cols; i++) free(st->key_cols[i]);
|
|
290
|
+
free(st->key_cols);
|
|
291
|
+
hs_free(&st->seen);
|
|
292
|
+
free(st);
|
|
293
|
+
return NULL;
|
|
294
|
+
}
|
|
295
|
+
step->process = unique_process;
|
|
296
|
+
step->flush = unique_flush;
|
|
297
|
+
step->destroy = unique_destroy;
|
|
298
|
+
step->state = st;
|
|
299
|
+
return step;
|
|
300
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* op_unpivot.c — Wide to long. Adds variable + value columns, multiplies rows.
|
|
3
|
+
*
|
|
4
|
+
* Config: {"columns": ["q1", "q2", "q3"]}
|
|
5
|
+
* Keeps non-listed columns, melts listed columns into variable/value pairs.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
#include "internal.h"
|
|
9
|
+
#include "cJSON.h"
|
|
10
|
+
#include "date_utils.h"
|
|
11
|
+
#include <stdlib.h>
|
|
12
|
+
#include <string.h>
|
|
13
|
+
#include <stdio.h>
|
|
14
|
+
|
|
15
|
+
typedef struct {
|
|
16
|
+
char **cols;
|
|
17
|
+
size_t n_cols;
|
|
18
|
+
} unpivot_state;
|
|
19
|
+
|
|
20
|
+
static int unpivot_process(tf_step *self, tf_batch *in, tf_batch **out,
|
|
21
|
+
tf_side_channels *side) {
|
|
22
|
+
(void)side;
|
|
23
|
+
unpivot_state *st = self->state;
|
|
24
|
+
*out = NULL;
|
|
25
|
+
|
|
26
|
+
/* Determine which columns are value columns and which are id columns */
|
|
27
|
+
int *is_value = calloc(in->n_cols, sizeof(int));
|
|
28
|
+
if (!is_value) return TF_ERROR;
|
|
29
|
+
size_t n_value = 0;
|
|
30
|
+
for (size_t c = 0; c < in->n_cols; c++) {
|
|
31
|
+
for (size_t k = 0; k < st->n_cols; k++) {
|
|
32
|
+
if (strcmp(in->col_names[c], st->cols[k]) == 0) {
|
|
33
|
+
is_value[c] = 1;
|
|
34
|
+
n_value++;
|
|
35
|
+
break;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
size_t n_id = in->n_cols - n_value;
|
|
40
|
+
if (n_value == 0) { free(is_value); return TF_OK; }
|
|
41
|
+
|
|
42
|
+
/* Output: id columns + "variable" + "value" */
|
|
43
|
+
size_t out_n_cols = n_id + 2;
|
|
44
|
+
size_t max_rows = in->n_rows * n_value;
|
|
45
|
+
tf_batch *ob = tf_batch_create(out_n_cols, max_rows > 0 ? max_rows : 16);
|
|
46
|
+
if (!ob) { free(is_value); return TF_ERROR; }
|
|
47
|
+
|
|
48
|
+
size_t oc = 0;
|
|
49
|
+
for (size_t c = 0; c < in->n_cols; c++) {
|
|
50
|
+
if (!is_value[c])
|
|
51
|
+
tf_batch_set_schema(ob, oc++, in->col_names[c], in->col_types[c]);
|
|
52
|
+
}
|
|
53
|
+
tf_batch_set_schema(ob, oc, "variable", TF_TYPE_STRING);
|
|
54
|
+
tf_batch_set_schema(ob, oc + 1, "value", TF_TYPE_STRING);
|
|
55
|
+
|
|
56
|
+
size_t out_row = 0;
|
|
57
|
+
for (size_t r = 0; r < in->n_rows; r++) {
|
|
58
|
+
for (size_t c = 0; c < in->n_cols; c++) {
|
|
59
|
+
if (!is_value[c]) continue;
|
|
60
|
+
|
|
61
|
+
tf_batch_ensure_capacity(ob, out_row + 1);
|
|
62
|
+
|
|
63
|
+
/* Copy id columns */
|
|
64
|
+
size_t oidx = 0;
|
|
65
|
+
for (size_t ic = 0; ic < in->n_cols; ic++) {
|
|
66
|
+
if (is_value[ic]) continue;
|
|
67
|
+
if (tf_batch_is_null(in, r, ic))
|
|
68
|
+
tf_batch_set_null(ob, out_row, oidx);
|
|
69
|
+
else {
|
|
70
|
+
switch (in->col_types[ic]) {
|
|
71
|
+
case TF_TYPE_BOOL: tf_batch_set_bool(ob, out_row, oidx, tf_batch_get_bool(in, r, ic)); break;
|
|
72
|
+
case TF_TYPE_INT64: tf_batch_set_int64(ob, out_row, oidx, tf_batch_get_int64(in, r, ic)); break;
|
|
73
|
+
case TF_TYPE_FLOAT64: tf_batch_set_float64(ob, out_row, oidx, tf_batch_get_float64(in, r, ic)); break;
|
|
74
|
+
case TF_TYPE_STRING: tf_batch_set_string(ob, out_row, oidx, tf_batch_get_string(in, r, ic)); break;
|
|
75
|
+
case TF_TYPE_DATE: tf_batch_set_date(ob, out_row, oidx, tf_batch_get_date(in, r, ic)); break;
|
|
76
|
+
case TF_TYPE_TIMESTAMP: tf_batch_set_timestamp(ob, out_row, oidx, tf_batch_get_timestamp(in, r, ic)); break;
|
|
77
|
+
default: tf_batch_set_null(ob, out_row, oidx); break;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
oidx++;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/* Set variable name */
|
|
84
|
+
tf_batch_set_string(ob, out_row, n_id, in->col_names[c]);
|
|
85
|
+
|
|
86
|
+
/* Set value (convert to string) */
|
|
87
|
+
if (tf_batch_is_null(in, r, c)) {
|
|
88
|
+
tf_batch_set_null(ob, out_row, n_id + 1);
|
|
89
|
+
} else {
|
|
90
|
+
char buf[64];
|
|
91
|
+
switch (in->col_types[c]) {
|
|
92
|
+
case TF_TYPE_STRING: tf_batch_set_string(ob, out_row, n_id + 1, tf_batch_get_string(in, r, c)); break;
|
|
93
|
+
case TF_TYPE_INT64:
|
|
94
|
+
snprintf(buf, sizeof(buf), "%lld", (long long)tf_batch_get_int64(in, r, c));
|
|
95
|
+
tf_batch_set_string(ob, out_row, n_id + 1, buf); break;
|
|
96
|
+
case TF_TYPE_FLOAT64:
|
|
97
|
+
snprintf(buf, sizeof(buf), "%g", tf_batch_get_float64(in, r, c));
|
|
98
|
+
tf_batch_set_string(ob, out_row, n_id + 1, buf); break;
|
|
99
|
+
case TF_TYPE_BOOL:
|
|
100
|
+
tf_batch_set_string(ob, out_row, n_id + 1, tf_batch_get_bool(in, r, c) ? "true" : "false"); break;
|
|
101
|
+
case TF_TYPE_DATE:
|
|
102
|
+
tf_date_format(tf_batch_get_date(in, r, c), buf, sizeof(buf));
|
|
103
|
+
tf_batch_set_string(ob, out_row, n_id + 1, buf); break;
|
|
104
|
+
case TF_TYPE_TIMESTAMP:
|
|
105
|
+
tf_timestamp_format(tf_batch_get_timestamp(in, r, c), buf, sizeof(buf));
|
|
106
|
+
tf_batch_set_string(ob, out_row, n_id + 1, buf); break;
|
|
107
|
+
default:
|
|
108
|
+
tf_batch_set_null(ob, out_row, n_id + 1); break;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
ob->n_rows = ++out_row;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
free(is_value);
|
|
117
|
+
if (out_row > 0) *out = ob;
|
|
118
|
+
else tf_batch_free(ob);
|
|
119
|
+
return TF_OK;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
static int unpivot_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
|
|
123
|
+
(void)self; (void)side; *out = NULL; return TF_OK;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
static void unpivot_destroy(tf_step *self) {
|
|
127
|
+
unpivot_state *st = self->state;
|
|
128
|
+
if (st) {
|
|
129
|
+
for (size_t i = 0; i < st->n_cols; i++) free(st->cols[i]);
|
|
130
|
+
free(st->cols); free(st);
|
|
131
|
+
}
|
|
132
|
+
free(self);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
tf_step *tf_unpivot_create(const cJSON *args) {
|
|
136
|
+
if (!args) return NULL;
|
|
137
|
+
cJSON *columns = cJSON_GetObjectItemCaseSensitive(args, "columns");
|
|
138
|
+
if (!columns || !cJSON_IsArray(columns)) return NULL;
|
|
139
|
+
|
|
140
|
+
int n = cJSON_GetArraySize(columns);
|
|
141
|
+
if (n <= 0) return NULL;
|
|
142
|
+
|
|
143
|
+
unpivot_state *st = calloc(1, sizeof(unpivot_state));
|
|
144
|
+
if (!st) return NULL;
|
|
145
|
+
st->cols = calloc(n, sizeof(char *));
|
|
146
|
+
st->n_cols = n;
|
|
147
|
+
for (int i = 0; i < n; i++) {
|
|
148
|
+
cJSON *item = cJSON_GetArrayItem(columns, i);
|
|
149
|
+
if (cJSON_IsString(item)) st->cols[i] = strdup(item->valuestring);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
tf_step *step = malloc(sizeof(tf_step));
|
|
153
|
+
if (!step) { unpivot_destroy(&(tf_step){.state = st}); return NULL; }
|
|
154
|
+
step->process = unpivot_process;
|
|
155
|
+
step->flush = unpivot_flush;
|
|
156
|
+
step->destroy = unpivot_destroy;
|
|
157
|
+
step->state = st;
|
|
158
|
+
return step;
|
|
159
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* op_validate.c — Like filter but adds _valid bool column, keeps all rows.
|
|
3
|
+
*
|
|
4
|
+
* Config: {"expr": "col('age') > 0"}
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
#include "internal.h"
|
|
8
|
+
#include "cJSON.h"
|
|
9
|
+
#include <stdlib.h>
|
|
10
|
+
#include <string.h>
|
|
11
|
+
|
|
12
|
+
typedef struct {
|
|
13
|
+
tf_expr *expr;
|
|
14
|
+
} validate_state;
|
|
15
|
+
|
|
16
|
+
static int validate_process(tf_step *self, tf_batch *in, tf_batch **out,
|
|
17
|
+
tf_side_channels *side) {
|
|
18
|
+
(void)side;
|
|
19
|
+
validate_state *st = self->state;
|
|
20
|
+
*out = NULL;
|
|
21
|
+
|
|
22
|
+
tf_batch *ob = tf_batch_create(in->n_cols + 1, in->n_rows);
|
|
23
|
+
if (!ob) return TF_ERROR;
|
|
24
|
+
for (size_t c = 0; c < in->n_cols; c++)
|
|
25
|
+
tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
|
|
26
|
+
tf_batch_set_schema(ob, in->n_cols, "_valid", TF_TYPE_BOOL);
|
|
27
|
+
|
|
28
|
+
for (size_t r = 0; r < in->n_rows; r++) {
|
|
29
|
+
tf_batch_copy_row(ob, r, in, r);
|
|
30
|
+
bool valid = false;
|
|
31
|
+
tf_expr_eval(st->expr, in, r, &valid);
|
|
32
|
+
tf_batch_set_bool(ob, r, in->n_cols, valid);
|
|
33
|
+
ob->n_rows = r + 1;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
*out = ob;
|
|
37
|
+
return TF_OK;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
static int validate_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
|
|
41
|
+
(void)self; (void)side;
|
|
42
|
+
*out = NULL;
|
|
43
|
+
return TF_OK;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
static void validate_destroy(tf_step *self) {
|
|
47
|
+
validate_state *st = self->state;
|
|
48
|
+
if (st) { tf_expr_free(st->expr); free(st); }
|
|
49
|
+
free(self);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
tf_step *tf_validate_create(const cJSON *args) {
|
|
53
|
+
if (!args) return NULL;
|
|
54
|
+
cJSON *expr_json = cJSON_GetObjectItemCaseSensitive(args, "expr");
|
|
55
|
+
if (!cJSON_IsString(expr_json)) return NULL;
|
|
56
|
+
|
|
57
|
+
tf_expr *expr = tf_expr_parse(expr_json->valuestring);
|
|
58
|
+
if (!expr) return NULL;
|
|
59
|
+
|
|
60
|
+
validate_state *st = calloc(1, sizeof(validate_state));
|
|
61
|
+
if (!st) { tf_expr_free(expr); return NULL; }
|
|
62
|
+
st->expr = expr;
|
|
63
|
+
|
|
64
|
+
tf_step *step = malloc(sizeof(tf_step));
|
|
65
|
+
if (!step) { tf_expr_free(expr); free(st); return NULL; }
|
|
66
|
+
step->process = validate_process;
|
|
67
|
+
step->flush = validate_flush;
|
|
68
|
+
step->destroy = validate_destroy;
|
|
69
|
+
step->state = st;
|
|
70
|
+
return step;
|
|
71
|
+
}
|
package/csrc/op_window.c
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* op_window.c — Sliding window aggregations.
|
|
3
|
+
*
|
|
4
|
+
* Config: {"column": "price", "size": 3, "func": "avg", "result": "price_avg3"}
|
|
5
|
+
* Supported funcs: avg, sum, min, max, count
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
#include "internal.h"
|
|
9
|
+
#include "cJSON.h"
|
|
10
|
+
#include <stdlib.h>
|
|
11
|
+
#include <string.h>
|
|
12
|
+
#include <stdio.h>
|
|
13
|
+
#include <math.h>
|
|
14
|
+
|
|
15
|
+
typedef enum {
|
|
16
|
+
WIN_AVG, WIN_SUM, WIN_MIN, WIN_MAX, WIN_COUNT,
|
|
17
|
+
} win_func;
|
|
18
|
+
|
|
19
|
+
typedef struct {
|
|
20
|
+
char *column;
|
|
21
|
+
char *result;
|
|
22
|
+
win_func func;
|
|
23
|
+
size_t size;
|
|
24
|
+
double *ring; /* circular buffer */
|
|
25
|
+
size_t head; /* next write position */
|
|
26
|
+
size_t count; /* items in buffer */
|
|
27
|
+
} window_state;
|
|
28
|
+
|
|
29
|
+
static win_func parse_win_func(const char *s) {
|
|
30
|
+
if (strcmp(s, "avg") == 0) return WIN_AVG;
|
|
31
|
+
if (strcmp(s, "sum") == 0) return WIN_SUM;
|
|
32
|
+
if (strcmp(s, "min") == 0) return WIN_MIN;
|
|
33
|
+
if (strcmp(s, "max") == 0) return WIN_MAX;
|
|
34
|
+
if (strcmp(s, "count") == 0) return WIN_COUNT;
|
|
35
|
+
return WIN_AVG;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
static int window_process(tf_step *self, tf_batch *in, tf_batch **out,
|
|
39
|
+
tf_side_channels *side) {
|
|
40
|
+
(void)side;
|
|
41
|
+
window_state *st = self->state;
|
|
42
|
+
*out = NULL;
|
|
43
|
+
|
|
44
|
+
tf_batch *ob = tf_batch_create(in->n_cols + 1, in->n_rows);
|
|
45
|
+
if (!ob) return TF_ERROR;
|
|
46
|
+
for (size_t c = 0; c < in->n_cols; c++)
|
|
47
|
+
tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
|
|
48
|
+
tf_batch_set_schema(ob, in->n_cols, st->result, TF_TYPE_FLOAT64);
|
|
49
|
+
|
|
50
|
+
int ci = tf_batch_col_index(in, st->column);
|
|
51
|
+
|
|
52
|
+
for (size_t r = 0; r < in->n_rows; r++) {
|
|
53
|
+
tf_batch_copy_row(ob, r, in, r);
|
|
54
|
+
|
|
55
|
+
if (ci < 0 || tf_batch_is_null(in, r, ci)) {
|
|
56
|
+
tf_batch_set_null(ob, r, in->n_cols);
|
|
57
|
+
ob->n_rows = r + 1;
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
double val = 0;
|
|
62
|
+
if (in->col_types[ci] == TF_TYPE_INT64) val = (double)tf_batch_get_int64(in, r, ci);
|
|
63
|
+
else if (in->col_types[ci] == TF_TYPE_FLOAT64) val = tf_batch_get_float64(in, r, ci);
|
|
64
|
+
|
|
65
|
+
/* Add to ring buffer */
|
|
66
|
+
st->ring[st->head] = val;
|
|
67
|
+
st->head = (st->head + 1) % st->size;
|
|
68
|
+
if (st->count < st->size) st->count++;
|
|
69
|
+
|
|
70
|
+
/* Compute window aggregate */
|
|
71
|
+
double result = 0;
|
|
72
|
+
switch (st->func) {
|
|
73
|
+
case WIN_SUM:
|
|
74
|
+
case WIN_AVG: {
|
|
75
|
+
double sum = 0;
|
|
76
|
+
for (size_t i = 0; i < st->count; i++) sum += st->ring[i];
|
|
77
|
+
result = (st->func == WIN_AVG) ? sum / st->count : sum;
|
|
78
|
+
break;
|
|
79
|
+
}
|
|
80
|
+
case WIN_MIN: {
|
|
81
|
+
result = st->ring[0];
|
|
82
|
+
for (size_t i = 1; i < st->count; i++)
|
|
83
|
+
if (st->ring[i] < result) result = st->ring[i];
|
|
84
|
+
break;
|
|
85
|
+
}
|
|
86
|
+
case WIN_MAX: {
|
|
87
|
+
result = st->ring[0];
|
|
88
|
+
for (size_t i = 1; i < st->count; i++)
|
|
89
|
+
if (st->ring[i] > result) result = st->ring[i];
|
|
90
|
+
break;
|
|
91
|
+
}
|
|
92
|
+
case WIN_COUNT:
|
|
93
|
+
result = (double)st->count;
|
|
94
|
+
break;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
tf_batch_set_float64(ob, r, in->n_cols, result);
|
|
98
|
+
ob->n_rows = r + 1;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
*out = ob;
|
|
102
|
+
return TF_OK;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
static int window_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
|
|
106
|
+
(void)self; (void)side; *out = NULL; return TF_OK;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
static void window_destroy(tf_step *self) {
|
|
110
|
+
window_state *st = self->state;
|
|
111
|
+
if (st) { free(st->column); free(st->result); free(st->ring); free(st); }
|
|
112
|
+
free(self);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
tf_step *tf_window_create(const cJSON *args) {
|
|
116
|
+
if (!args) return NULL;
|
|
117
|
+
cJSON *col_j = cJSON_GetObjectItemCaseSensitive(args, "column");
|
|
118
|
+
cJSON *size_j = cJSON_GetObjectItemCaseSensitive(args, "size");
|
|
119
|
+
cJSON *func_j = cJSON_GetObjectItemCaseSensitive(args, "func");
|
|
120
|
+
if (!cJSON_IsString(col_j) || !cJSON_IsNumber(size_j) || !cJSON_IsString(func_j))
|
|
121
|
+
return NULL;
|
|
122
|
+
|
|
123
|
+
size_t win_size = (size_t)size_j->valueint;
|
|
124
|
+
if (win_size == 0) win_size = 1;
|
|
125
|
+
|
|
126
|
+
window_state *st = calloc(1, sizeof(window_state));
|
|
127
|
+
if (!st) return NULL;
|
|
128
|
+
st->column = strdup(col_j->valuestring);
|
|
129
|
+
st->func = parse_win_func(func_j->valuestring);
|
|
130
|
+
st->size = win_size;
|
|
131
|
+
st->ring = calloc(win_size, sizeof(double));
|
|
132
|
+
|
|
133
|
+
cJSON *res_j = cJSON_GetObjectItemCaseSensitive(args, "result");
|
|
134
|
+
if (cJSON_IsString(res_j)) {
|
|
135
|
+
st->result = strdup(res_j->valuestring);
|
|
136
|
+
} else {
|
|
137
|
+
char buf[256];
|
|
138
|
+
snprintf(buf, sizeof(buf), "%s_%s%zu", st->column,
|
|
139
|
+
func_j->valuestring, win_size);
|
|
140
|
+
st->result = strdup(buf);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
tf_step *step = malloc(sizeof(tf_step));
|
|
144
|
+
if (!step) { free(st->column); free(st->result); free(st->ring); free(st); return NULL; }
|
|
145
|
+
step->process = window_process;
|
|
146
|
+
step->flush = window_flush;
|
|
147
|
+
step->destroy = window_destroy;
|
|
148
|
+
step->state = st;
|
|
149
|
+
return step;
|
|
150
|
+
}
|