tranfi 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -0
- package/NOTICE +8 -0
- package/README.md +272 -40
- package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
- package/app/index.html +1 -1
- package/binding.gyp +55 -3
- package/csrc/arena.c +7 -5
- package/csrc/batch.c +818 -71
- package/csrc/buffer.c +84 -8
- package/csrc/cJSON.c +262 -19
- package/csrc/cJSON.h +17 -1
- package/csrc/codec_csv.c +1074 -181
- package/csrc/codec_jsonl.c +830 -118
- package/csrc/codec_table.c +108 -78
- package/csrc/codec_text.c +286 -68
- package/csrc/compiler.c +31 -3
- package/csrc/config.h +21 -0
- package/csrc/dsl.c +4722 -485
- package/csrc/expr.c +363 -55
- package/csrc/expr.h +2 -0
- package/csrc/internal.h +316 -27
- package/csrc/ir.c +65 -18
- package/csrc/ir.h +41 -0
- package/csrc/ir_schema.c +20 -5
- package/csrc/ir_serialize.c +68 -6
- package/csrc/ir_sql.c +796 -185
- package/csrc/ir_validate.c +462 -6
- package/csrc/json_path.c +210 -0
- package/csrc/main.c +879 -30
- package/csrc/memory_estimate.c +477 -0
- package/csrc/op_acf.c +171 -21
- package/csrc/op_across.c +477 -0
- package/csrc/op_anomaly.c +167 -32
- package/csrc/op_assert.c +761 -0
- package/csrc/op_bin.c +168 -29
- package/csrc/op_cast.c +383 -55
- package/csrc/op_clip.c +30 -19
- package/csrc/op_date_trunc.c +208 -34
- package/csrc/op_datetime.c +259 -77
- package/csrc/op_derive.c +65 -97
- package/csrc/op_diff.c +146 -30
- package/csrc/op_ewma.c +149 -30
- package/csrc/op_explode.c +124 -26
- package/csrc/op_fill_down.c +125 -53
- package/csrc/op_fill_null.c +176 -31
- package/csrc/op_filter.c +89 -40
- package/csrc/op_frequency.c +571 -43
- package/csrc/op_grep.c +36 -18
- package/csrc/op_group_agg.c +1790 -119
- package/csrc/op_hash.c +48 -15
- package/csrc/op_head.c +21 -86
- package/csrc/op_interpolate.c +268 -62
- package/csrc/op_join.c +2700 -182
- package/csrc/op_json_extract.c +227 -0
- package/csrc/op_json_filter.c +384 -0
- package/csrc/op_json_flatten.c +293 -0
- package/csrc/op_json_schema.c +503 -0
- package/csrc/op_label_encode.c +328 -53
- package/csrc/op_lag.c +181 -0
- package/csrc/op_lead.c +141 -89
- package/csrc/op_normalize.c +363 -79
- package/csrc/op_onehot.c +345 -73
- package/csrc/op_pivot.c +1546 -162
- package/csrc/op_quarantine.c +189 -0
- package/csrc/op_registry.c +2062 -166
- package/csrc/op_rename.c +41 -50
- package/csrc/op_replace.c +270 -118
- package/csrc/op_rleid.c +297 -0
- package/csrc/op_rowid.c +559 -0
- package/csrc/op_sample.c +80 -23
- package/csrc/op_schema.c +1341 -0
- package/csrc/op_schema_infer.c +252 -0
- package/csrc/op_select.c +265 -65
- package/csrc/op_set.c +3449 -0
- package/csrc/op_skip.c +30 -87
- package/csrc/op_sort.c +670 -124
- package/csrc/op_source_name.c +120 -0
- package/csrc/op_split.c +65 -28
- package/csrc/op_split_data.c +41 -9
- package/csrc/op_stack.c +178 -222
- package/csrc/op_stats.c +206 -110
- package/csrc/op_step.c +217 -55
- package/csrc/op_tail.c +21 -12
- package/csrc/op_tee.c +338 -0
- package/csrc/op_top.c +260 -53
- package/csrc/op_trim.c +48 -19
- package/csrc/op_unique.c +1193 -150
- package/csrc/op_unpivot.c +100 -66
- package/csrc/op_validate.c +601 -24
- package/csrc/op_window.c +492 -51
- package/csrc/path_policy.c +85 -0
- package/csrc/pipeline.c +872 -99
- package/csrc/recipes.c +3 -1
- package/csrc/report.c +73 -30
- package/csrc/selector.c +1097 -0
- package/csrc/size_utils.c +348 -0
- package/csrc/spill.c +317 -0
- package/csrc/spill.h +21 -0
- package/csrc/tranfi.h +169 -1
- package/csrc/transform.h +209 -0
- package/csrc/transform_api.c +2237 -0
- package/csrc/transform_categorical.c +923 -0
- package/csrc/transform_internal.h +472 -0
- package/csrc/transform_json.c +3812 -0
- package/csrc/transform_numeric.c +1966 -0
- package/csrc/transform_sha256.c +154 -0
- package/csrc/transform_wasm.h +162 -0
- package/csrc/transform_wasm_api.c +1373 -0
- package/csrc/wasm_api.c +70 -9
- package/napi_api.c +219 -11
- package/napi_transform.c +1648 -0
- package/napi_transform.h +8 -0
- package/package.json +27 -11
- package/scripts/install-native.js +76 -0
- package/scripts/prepack.js +64 -0
- package/scripts/sync-csrc.js +23 -0
- package/src/cli.js +8 -11
- package/src/engines/duckdb.js +45 -12
- package/src/index.js +661 -42
- package/src/memory_policy.js +411 -0
- package/src/native.js +1 -5
- package/src/pipeline.js +454 -31
- package/src/recipe_json.js +80 -0
- package/src/server.js +10 -8
- package/src/transform.js +403 -0
- package/src/transform_error.js +10 -0
- package/src/wasm.js +6 -4
- package/wasm/index.js +498 -10
- package/wasm/tranfi_core.js +0 -0
- package/wasm/transform.js +1156 -0
- package/wasm/worker.js +786 -0
- package/csrc/plan.c +0 -206
package/csrc/op_window.c
CHANGED
|
@@ -11,28 +11,142 @@
|
|
|
11
11
|
#include <string.h>
|
|
12
12
|
#include <stdio.h>
|
|
13
13
|
#include <math.h>
|
|
14
|
+
#include <ctype.h>
|
|
14
15
|
|
|
15
16
|
typedef enum {
|
|
16
17
|
WIN_AVG, WIN_SUM, WIN_MIN, WIN_MAX, WIN_COUNT,
|
|
17
18
|
} win_func;
|
|
18
19
|
|
|
20
|
+
typedef enum {
|
|
21
|
+
WIN_MISSING_ERROR,
|
|
22
|
+
WIN_MISSING_NULL,
|
|
23
|
+
WIN_MISSING_IGNORE,
|
|
24
|
+
} window_missing_policy;
|
|
25
|
+
|
|
26
|
+
typedef enum {
|
|
27
|
+
WIN_TYPE_FAIL,
|
|
28
|
+
WIN_TYPE_NULL,
|
|
29
|
+
} window_type_policy;
|
|
30
|
+
|
|
19
31
|
typedef struct {
|
|
20
32
|
char *column;
|
|
21
33
|
char *result;
|
|
34
|
+
char *op_label;
|
|
22
35
|
win_func func;
|
|
23
36
|
size_t size;
|
|
24
37
|
double *ring; /* circular buffer */
|
|
25
38
|
size_t head; /* next write position */
|
|
26
39
|
size_t count; /* items in buffer */
|
|
40
|
+
window_missing_policy missing;
|
|
41
|
+
window_type_policy on_type_error;
|
|
27
42
|
} window_state;
|
|
28
43
|
|
|
29
|
-
static
|
|
30
|
-
if (strcmp(s, "avg") == 0)
|
|
31
|
-
if (strcmp(s, "sum") == 0)
|
|
32
|
-
if (strcmp(s, "min") == 0)
|
|
33
|
-
if (strcmp(s, "max") == 0)
|
|
34
|
-
if (strcmp(s, "count") == 0)
|
|
35
|
-
return
|
|
44
|
+
static int parse_win_func(const char *s, win_func *out) {
|
|
45
|
+
if (strcmp(s, "avg") == 0) { *out = WIN_AVG; return TF_OK; }
|
|
46
|
+
if (strcmp(s, "sum") == 0) { *out = WIN_SUM; return TF_OK; }
|
|
47
|
+
if (strcmp(s, "min") == 0) { *out = WIN_MIN; return TF_OK; }
|
|
48
|
+
if (strcmp(s, "max") == 0) { *out = WIN_MAX; return TF_OK; }
|
|
49
|
+
if (strcmp(s, "count") == 0) { *out = WIN_COUNT; return TF_OK; }
|
|
50
|
+
return TF_ERROR;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
static int window_is_numeric_type(tf_type type) {
|
|
54
|
+
return type == TF_TYPE_INT64 || type == TF_TYPE_FLOAT64;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
static double window_get_numeric(const tf_batch *b, size_t r, int ci) {
|
|
58
|
+
if (b->col_types[ci] == TF_TYPE_INT64) return (double)tf_batch_get_int64(b, r, ci);
|
|
59
|
+
return tf_batch_get_float64(b, r, ci);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
static const char *window_label(const window_state *st) {
|
|
63
|
+
return (st && st->op_label) ? st->op_label : "window";
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
static char *window_default_result_name(const char *column, const char *func, size_t size) {
|
|
67
|
+
char size_buf[32];
|
|
68
|
+
int n = snprintf(size_buf, sizeof(size_buf), "%zu", size);
|
|
69
|
+
if (n < 0 || (size_t)n >= sizeof(size_buf)) return NULL;
|
|
70
|
+
|
|
71
|
+
char *func_suffix = tf_string_append_suffix_checked("_", func);
|
|
72
|
+
if (!func_suffix) return NULL;
|
|
73
|
+
char *full_suffix = tf_string_append_suffix_checked(func_suffix, size_buf);
|
|
74
|
+
free(func_suffix);
|
|
75
|
+
if (!full_suffix) return NULL;
|
|
76
|
+
|
|
77
|
+
char *result = tf_string_append_suffix_checked(column, full_suffix);
|
|
78
|
+
free(full_suffix);
|
|
79
|
+
return result;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
static void window_set_col_error(const window_state *st, const char *suffix) {
|
|
83
|
+
char msg[512];
|
|
84
|
+
snprintf(msg, sizeof(msg), "%s: column '%s' %s", window_label(st), st && st->column ? st->column : "", suffix);
|
|
85
|
+
tf_set_last_error(msg);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
static int window_parse_missing_policy(const cJSON *args, const char *label, window_missing_policy *out) {
|
|
89
|
+
*out = WIN_MISSING_ERROR;
|
|
90
|
+
const cJSON *j = cJSON_GetObjectItemCaseSensitive(args, "missing");
|
|
91
|
+
if (!j) return TF_OK;
|
|
92
|
+
if (!cJSON_IsString(j)) {
|
|
93
|
+
char msg[128];
|
|
94
|
+
snprintf(msg, sizeof(msg), "%s: missing must be error, null, or ignore", label);
|
|
95
|
+
tf_set_last_error(msg);
|
|
96
|
+
return TF_ERROR;
|
|
97
|
+
}
|
|
98
|
+
if (strcmp(j->valuestring, "error") == 0) *out = WIN_MISSING_ERROR;
|
|
99
|
+
else if (strcmp(j->valuestring, "null") == 0) *out = WIN_MISSING_NULL;
|
|
100
|
+
else if (strcmp(j->valuestring, "ignore") == 0) *out = WIN_MISSING_IGNORE;
|
|
101
|
+
else {
|
|
102
|
+
char msg[128];
|
|
103
|
+
snprintf(msg, sizeof(msg), "%s: missing must be error, null, or ignore", label);
|
|
104
|
+
tf_set_last_error(msg);
|
|
105
|
+
return TF_ERROR;
|
|
106
|
+
}
|
|
107
|
+
return TF_OK;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
static int window_parse_type_policy(const cJSON *args, const char *label, window_type_policy *out) {
|
|
111
|
+
*out = WIN_TYPE_FAIL;
|
|
112
|
+
const cJSON *j = cJSON_GetObjectItemCaseSensitive(args, "on_type_error");
|
|
113
|
+
if (!j) return TF_OK;
|
|
114
|
+
if (!cJSON_IsString(j)) {
|
|
115
|
+
char msg[128];
|
|
116
|
+
snprintf(msg, sizeof(msg), "%s: on_type_error must be fail or null", label);
|
|
117
|
+
tf_set_last_error(msg);
|
|
118
|
+
return TF_ERROR;
|
|
119
|
+
}
|
|
120
|
+
if (strcmp(j->valuestring, "fail") == 0) *out = WIN_TYPE_FAIL;
|
|
121
|
+
else if (strcmp(j->valuestring, "null") == 0) *out = WIN_TYPE_NULL;
|
|
122
|
+
else {
|
|
123
|
+
char msg[128];
|
|
124
|
+
snprintf(msg, sizeof(msg), "%s: on_type_error must be fail or null", label);
|
|
125
|
+
tf_set_last_error(msg);
|
|
126
|
+
return TF_ERROR;
|
|
127
|
+
}
|
|
128
|
+
return TF_OK;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
static int window_passthrough(tf_batch *in, tf_batch **out) {
|
|
132
|
+
tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
|
|
133
|
+
if (!ob) return TF_ERROR;
|
|
134
|
+
if (tf_batch_clone_schema(ob, in) != TF_OK) {
|
|
135
|
+
tf_batch_free(ob);
|
|
136
|
+
return TF_ERROR;
|
|
137
|
+
}
|
|
138
|
+
for (size_t r = 0; r < in->n_rows; r++) {
|
|
139
|
+
if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
|
|
140
|
+
tf_batch_free(ob);
|
|
141
|
+
return TF_ERROR;
|
|
142
|
+
}
|
|
143
|
+
if (tf_batch_expose_row(ob, r) != TF_OK) {
|
|
144
|
+
tf_batch_free(ob);
|
|
145
|
+
return TF_ERROR;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
*out = ob;
|
|
149
|
+
return TF_OK;
|
|
36
150
|
}
|
|
37
151
|
|
|
38
152
|
static int window_process(tf_step *self, tf_batch *in, tf_batch **out,
|
|
@@ -41,31 +155,47 @@ static int window_process(tf_step *self, tf_batch *in, tf_batch **out,
|
|
|
41
155
|
window_state *st = self->state;
|
|
42
156
|
*out = NULL;
|
|
43
157
|
|
|
158
|
+
int ci = tf_batch_col_index(in, st->column);
|
|
159
|
+
int force_null = 0;
|
|
160
|
+
if (ci < 0) {
|
|
161
|
+
if (st->missing == WIN_MISSING_ERROR) {
|
|
162
|
+
window_set_col_error(st, "not found");
|
|
163
|
+
return TF_ERROR;
|
|
164
|
+
}
|
|
165
|
+
if (st->missing == WIN_MISSING_IGNORE) return window_passthrough(in, out);
|
|
166
|
+
force_null = 1;
|
|
167
|
+
} else if (st->func != WIN_COUNT && !window_is_numeric_type(in->col_types[ci])) {
|
|
168
|
+
if (st->on_type_error == WIN_TYPE_FAIL) {
|
|
169
|
+
window_set_col_error(st, "must be numeric");
|
|
170
|
+
return TF_ERROR;
|
|
171
|
+
}
|
|
172
|
+
force_null = 1;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const char *extra_names[1] = {st->result};
|
|
176
|
+
tf_type extra_types[1] = {TF_TYPE_FLOAT64};
|
|
44
177
|
tf_batch *ob = tf_batch_create(in->n_cols + 1, in->n_rows);
|
|
45
178
|
if (!ob) return TF_ERROR;
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
int ci = tf_batch_col_index(in, st->column);
|
|
179
|
+
if (tf_batch_clone_with_extra_cols(ob, in, extra_names, extra_types, 1) != TF_OK) {
|
|
180
|
+
tf_batch_free(ob);
|
|
181
|
+
return TF_ERROR;
|
|
182
|
+
}
|
|
51
183
|
|
|
52
184
|
for (size_t r = 0; r < in->n_rows; r++) {
|
|
53
|
-
tf_batch_copy_row(ob, r, in, r)
|
|
185
|
+
if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
|
|
186
|
+
tf_batch_free(ob);
|
|
187
|
+
return TF_ERROR;
|
|
188
|
+
}
|
|
54
189
|
|
|
55
|
-
if (
|
|
56
|
-
tf_batch_set_null(ob, r, in->n_cols);
|
|
57
|
-
ob
|
|
190
|
+
if (force_null || tf_batch_is_null(in, r, ci)) {
|
|
191
|
+
if (tf_batch_set_null(ob, r, in->n_cols) != TF_OK) { tf_batch_free(ob); return TF_ERROR; }
|
|
192
|
+
if (tf_batch_expose_row(ob, r) != TF_OK) { tf_batch_free(ob); return TF_ERROR; }
|
|
58
193
|
continue;
|
|
59
194
|
}
|
|
60
195
|
|
|
61
|
-
double val = 0;
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
/* Add to ring buffer */
|
|
66
|
-
st->ring[st->head] = val;
|
|
67
|
-
st->head = (st->head + 1) % st->size;
|
|
68
|
-
if (st->count < st->size) st->count++;
|
|
196
|
+
double val = st->func == WIN_COUNT ? 0.0 : window_get_numeric(in, r, ci);
|
|
197
|
+
size_t next_head = (st->head + 1) % st->size;
|
|
198
|
+
size_t next_count = st->count < st->size ? st->count + 1 : st->count;
|
|
69
199
|
|
|
70
200
|
/* Compute window aggregate */
|
|
71
201
|
double result = 0;
|
|
@@ -73,29 +203,43 @@ static int window_process(tf_step *self, tf_batch *in, tf_batch **out,
|
|
|
73
203
|
case WIN_SUM:
|
|
74
204
|
case WIN_AVG: {
|
|
75
205
|
double sum = 0;
|
|
76
|
-
for (size_t i = 0; i <
|
|
77
|
-
|
|
206
|
+
for (size_t i = 0; i < next_count; i++)
|
|
207
|
+
sum += i == st->head ? val : st->ring[i];
|
|
208
|
+
result = (st->func == WIN_AVG) ? sum / next_count : sum;
|
|
78
209
|
break;
|
|
79
210
|
}
|
|
80
211
|
case WIN_MIN: {
|
|
81
|
-
result = st->ring[0];
|
|
82
|
-
for (size_t i = 1; i <
|
|
83
|
-
|
|
212
|
+
result = (st->head == 0) ? val : st->ring[0];
|
|
213
|
+
for (size_t i = 1; i < next_count; i++) {
|
|
214
|
+
double item = i == st->head ? val : st->ring[i];
|
|
215
|
+
if (item < result) result = item;
|
|
216
|
+
}
|
|
84
217
|
break;
|
|
85
218
|
}
|
|
86
219
|
case WIN_MAX: {
|
|
87
|
-
result = st->ring[0];
|
|
88
|
-
for (size_t i = 1; i <
|
|
89
|
-
|
|
220
|
+
result = (st->head == 0) ? val : st->ring[0];
|
|
221
|
+
for (size_t i = 1; i < next_count; i++) {
|
|
222
|
+
double item = i == st->head ? val : st->ring[i];
|
|
223
|
+
if (item > result) result = item;
|
|
224
|
+
}
|
|
90
225
|
break;
|
|
91
226
|
}
|
|
92
227
|
case WIN_COUNT:
|
|
93
|
-
result = (double)
|
|
228
|
+
result = (double)next_count;
|
|
94
229
|
break;
|
|
95
230
|
}
|
|
96
231
|
|
|
97
|
-
tf_batch_set_float64(ob, r, in->n_cols, result)
|
|
98
|
-
|
|
232
|
+
if (tf_batch_set_float64(ob, r, in->n_cols, result) != TF_OK) {
|
|
233
|
+
tf_batch_free(ob);
|
|
234
|
+
return TF_ERROR;
|
|
235
|
+
}
|
|
236
|
+
if (tf_batch_expose_row(ob, r) != TF_OK) {
|
|
237
|
+
tf_batch_free(ob);
|
|
238
|
+
return TF_ERROR;
|
|
239
|
+
}
|
|
240
|
+
st->ring[st->head] = val;
|
|
241
|
+
st->head = next_head;
|
|
242
|
+
st->count = next_count;
|
|
99
243
|
}
|
|
100
244
|
|
|
101
245
|
*out = ob;
|
|
@@ -108,40 +252,337 @@ static int window_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
|
|
|
108
252
|
|
|
109
253
|
static void window_destroy(tf_step *self) {
|
|
110
254
|
window_state *st = self->state;
|
|
111
|
-
if (st) { free(st->column); free(st->result); free(st->ring); free(st); }
|
|
255
|
+
if (st) { free(st->column); free(st->result); free(st->op_label); free(st->ring); free(st); }
|
|
256
|
+
free(self);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
static tf_step *tf_window_create_alias(const cJSON *args, const char *func, const char *label) {
|
|
261
|
+
if (!args || !func || !label) return NULL;
|
|
262
|
+
cJSON *copy = cJSON_Duplicate(args, 1);
|
|
263
|
+
if (!copy) return NULL;
|
|
264
|
+
cJSON_DeleteItemFromObjectCaseSensitive(copy, "func");
|
|
265
|
+
cJSON_DeleteItemFromObjectCaseSensitive(copy, "__op_label");
|
|
266
|
+
if (tf_json_add_string(copy, "__op_label", label) != TF_OK ||
|
|
267
|
+
tf_json_add_string(copy, "func", func) != TF_OK) {
|
|
268
|
+
cJSON_Delete(copy);
|
|
269
|
+
return NULL;
|
|
270
|
+
}
|
|
271
|
+
tf_step *step = tf_window_create(copy);
|
|
272
|
+
cJSON_Delete(copy);
|
|
273
|
+
return step;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
tf_step *tf_rolling_sum_create(const cJSON *args) {
|
|
277
|
+
return tf_window_create_alias(args, "sum", "rolling-sum");
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
tf_step *tf_rolling_mean_create(const cJSON *args) {
|
|
281
|
+
return tf_window_create_alias(args, "avg", "rolling-mean");
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
tf_step *tf_rolling_min_create(const cJSON *args) {
|
|
285
|
+
return tf_window_create_alias(args, "min", "rolling-min");
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
tf_step *tf_rolling_max_create(const cJSON *args) {
|
|
289
|
+
return tf_window_create_alias(args, "max", "rolling-max");
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
typedef enum {
|
|
293
|
+
BOOL_ROLL_ANY,
|
|
294
|
+
BOOL_ROLL_ALL,
|
|
295
|
+
} bool_roll_func;
|
|
296
|
+
|
|
297
|
+
typedef enum {
|
|
298
|
+
BOOL_NULLS_IGNORE,
|
|
299
|
+
BOOL_NULLS_FALSE,
|
|
300
|
+
BOOL_NULLS_TRUE,
|
|
301
|
+
BOOL_NULLS_PROPAGATE,
|
|
302
|
+
} bool_null_policy;
|
|
303
|
+
|
|
304
|
+
typedef struct {
|
|
305
|
+
char *column;
|
|
306
|
+
char *result;
|
|
307
|
+
bool_roll_func func;
|
|
308
|
+
bool_null_policy nulls;
|
|
309
|
+
size_t size;
|
|
310
|
+
uint8_t *values;
|
|
311
|
+
uint8_t *is_null;
|
|
312
|
+
size_t head;
|
|
313
|
+
size_t count;
|
|
314
|
+
} bool_window_state;
|
|
315
|
+
|
|
316
|
+
static int streq_ci(const char *a, const char *b) {
|
|
317
|
+
if (!a || !b) return 0;
|
|
318
|
+
while (*a && *b) {
|
|
319
|
+
unsigned char ca = (unsigned char)*a;
|
|
320
|
+
unsigned char cb = (unsigned char)*b;
|
|
321
|
+
if (tolower(ca) != tolower(cb)) return 0;
|
|
322
|
+
a++;
|
|
323
|
+
b++;
|
|
324
|
+
}
|
|
325
|
+
return *a == '\0' && *b == '\0';
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
static int parse_bool_null_policy(const cJSON *args, bool_null_policy *out) {
|
|
329
|
+
*out = BOOL_NULLS_IGNORE;
|
|
330
|
+
cJSON *nulls_j = cJSON_GetObjectItemCaseSensitive(args, "nulls");
|
|
331
|
+
if (!nulls_j) return TF_OK;
|
|
332
|
+
if (!cJSON_IsString(nulls_j)) {
|
|
333
|
+
tf_set_last_error("rolling-any/all: nulls must be ignore, false, true, or propagate");
|
|
334
|
+
return TF_ERROR;
|
|
335
|
+
}
|
|
336
|
+
const char *s = nulls_j->valuestring;
|
|
337
|
+
if (strcmp(s, "ignore") == 0) *out = BOOL_NULLS_IGNORE;
|
|
338
|
+
else if (strcmp(s, "false") == 0) *out = BOOL_NULLS_FALSE;
|
|
339
|
+
else if (strcmp(s, "true") == 0) *out = BOOL_NULLS_TRUE;
|
|
340
|
+
else if (strcmp(s, "propagate") == 0) *out = BOOL_NULLS_PROPAGATE;
|
|
341
|
+
else {
|
|
342
|
+
tf_set_last_error("rolling-any/all: nulls must be ignore, false, true, or propagate");
|
|
343
|
+
return TF_ERROR;
|
|
344
|
+
}
|
|
345
|
+
return TF_OK;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
static int cell_to_bool(const tf_batch *b, size_t row, int col, uint8_t *out) {
|
|
349
|
+
if (col < 0 || tf_batch_is_null(b, row, (size_t)col)) return 0;
|
|
350
|
+
switch (b->col_types[col]) {
|
|
351
|
+
case TF_TYPE_BOOL:
|
|
352
|
+
*out = tf_batch_get_bool(b, row, (size_t)col) ? 1 : 0;
|
|
353
|
+
return 1;
|
|
354
|
+
case TF_TYPE_INT64:
|
|
355
|
+
*out = tf_batch_get_int64(b, row, (size_t)col) != 0 ? 1 : 0;
|
|
356
|
+
return 1;
|
|
357
|
+
case TF_TYPE_FLOAT64:
|
|
358
|
+
*out = tf_batch_get_float64(b, row, (size_t)col) != 0.0 ? 1 : 0;
|
|
359
|
+
return 1;
|
|
360
|
+
case TF_TYPE_STRING: {
|
|
361
|
+
const char *s = tf_batch_get_string(b, row, (size_t)col);
|
|
362
|
+
if (!s) return 0;
|
|
363
|
+
if (streq_ci(s, "true") || streq_ci(s, "t") || streq_ci(s, "yes") ||
|
|
364
|
+
streq_ci(s, "y") || strcmp(s, "1") == 0) {
|
|
365
|
+
*out = 1;
|
|
366
|
+
return 1;
|
|
367
|
+
}
|
|
368
|
+
if (streq_ci(s, "false") || streq_ci(s, "f") || streq_ci(s, "no") ||
|
|
369
|
+
streq_ci(s, "n") || strcmp(s, "0") == 0 || strcmp(s, "") == 0) {
|
|
370
|
+
*out = 0;
|
|
371
|
+
return 1;
|
|
372
|
+
}
|
|
373
|
+
return 0;
|
|
374
|
+
}
|
|
375
|
+
default:
|
|
376
|
+
return 0;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
static int bool_window_process(tf_step *self, tf_batch *in, tf_batch **out,
|
|
381
|
+
tf_side_channels *side) {
|
|
382
|
+
(void)side;
|
|
383
|
+
bool_window_state *st = self->state;
|
|
384
|
+
*out = NULL;
|
|
385
|
+
|
|
386
|
+
const char *extra_names[1] = {st->result};
|
|
387
|
+
tf_type extra_types[1] = {TF_TYPE_BOOL};
|
|
388
|
+
tf_batch *ob = tf_batch_create(in->n_cols + 1, in->n_rows);
|
|
389
|
+
if (!ob) return TF_ERROR;
|
|
390
|
+
if (tf_batch_clone_with_extra_cols(ob, in, extra_names, extra_types, 1) != TF_OK) {
|
|
391
|
+
tf_batch_free(ob);
|
|
392
|
+
return TF_ERROR;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
int ci = tf_batch_col_index(in, st->column);
|
|
396
|
+
|
|
397
|
+
for (size_t r = 0; r < in->n_rows; r++) {
|
|
398
|
+
if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
|
|
399
|
+
tf_batch_free(ob);
|
|
400
|
+
return TF_ERROR;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
uint8_t value = 0;
|
|
404
|
+
uint8_t is_null = cell_to_bool(in, r, ci, &value) ? 0 : 1;
|
|
405
|
+
size_t next_head = (st->head + 1) % st->size;
|
|
406
|
+
size_t next_count = st->count < st->size ? st->count + 1 : st->count;
|
|
407
|
+
|
|
408
|
+
size_t non_null = 0;
|
|
409
|
+
int seen_null = 0;
|
|
410
|
+
int any_true = 0;
|
|
411
|
+
int any_false = 0;
|
|
412
|
+
for (size_t i = 0; i < next_count; i++) {
|
|
413
|
+
uint8_t item_null = i == st->head ? is_null : st->is_null[i];
|
|
414
|
+
uint8_t item_value = i == st->head ? value : st->values[i];
|
|
415
|
+
if (item_null) {
|
|
416
|
+
seen_null = 1;
|
|
417
|
+
continue;
|
|
418
|
+
}
|
|
419
|
+
non_null++;
|
|
420
|
+
if (item_value) any_true = 1;
|
|
421
|
+
else any_false = 1;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
int out_null = 0;
|
|
425
|
+
int out_value = 0;
|
|
426
|
+
switch (st->nulls) {
|
|
427
|
+
case BOOL_NULLS_IGNORE:
|
|
428
|
+
if (non_null == 0) out_null = 1;
|
|
429
|
+
else out_value = (st->func == BOOL_ROLL_ANY) ? any_true : !any_false;
|
|
430
|
+
break;
|
|
431
|
+
case BOOL_NULLS_FALSE:
|
|
432
|
+
out_value = (st->func == BOOL_ROLL_ANY) ? any_true : (!seen_null && !any_false);
|
|
433
|
+
break;
|
|
434
|
+
case BOOL_NULLS_TRUE:
|
|
435
|
+
out_value = (st->func == BOOL_ROLL_ANY) ? (seen_null || any_true) : !any_false;
|
|
436
|
+
break;
|
|
437
|
+
case BOOL_NULLS_PROPAGATE:
|
|
438
|
+
if (seen_null) out_null = 1;
|
|
439
|
+
else out_value = (st->func == BOOL_ROLL_ANY) ? any_true : !any_false;
|
|
440
|
+
break;
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
int write_rc = out_null ? tf_batch_set_null(ob, r, in->n_cols)
|
|
444
|
+
: tf_batch_set_bool(ob, r, in->n_cols, out_value != 0);
|
|
445
|
+
if (write_rc != TF_OK) {
|
|
446
|
+
tf_batch_free(ob);
|
|
447
|
+
return TF_ERROR;
|
|
448
|
+
}
|
|
449
|
+
if (tf_batch_expose_row(ob, r) != TF_OK) {
|
|
450
|
+
tf_batch_free(ob);
|
|
451
|
+
return TF_ERROR;
|
|
452
|
+
}
|
|
453
|
+
st->values[st->head] = value;
|
|
454
|
+
st->is_null[st->head] = is_null;
|
|
455
|
+
st->head = next_head;
|
|
456
|
+
st->count = next_count;
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
*out = ob;
|
|
460
|
+
return TF_OK;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
static int bool_window_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
|
|
464
|
+
(void)self; (void)side; *out = NULL; return TF_OK;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
static void bool_window_destroy(tf_step *self) {
|
|
468
|
+
bool_window_state *st = self->state;
|
|
469
|
+
if (st) {
|
|
470
|
+
free(st->column);
|
|
471
|
+
free(st->result);
|
|
472
|
+
free(st->values);
|
|
473
|
+
free(st->is_null);
|
|
474
|
+
free(st);
|
|
475
|
+
}
|
|
112
476
|
free(self);
|
|
113
477
|
}
|
|
114
478
|
|
|
479
|
+
static tf_step *tf_rolling_bool_create(const cJSON *args, bool_roll_func func) {
|
|
480
|
+
if (!args) return NULL;
|
|
481
|
+
cJSON *col_j = cJSON_GetObjectItemCaseSensitive(args, "column");
|
|
482
|
+
size_t win_size = 0;
|
|
483
|
+
int has_size = tf_json_get_size_arg(args, "size",
|
|
484
|
+
1, TF_MAX_WINDOW_SIZE,
|
|
485
|
+
&win_size, "rolling-any/all");
|
|
486
|
+
if (has_size < 0) return NULL;
|
|
487
|
+
if (!cJSON_IsString(col_j) || has_size == 0) {
|
|
488
|
+
tf_set_last_error("rolling-any/all: requires column and positive size");
|
|
489
|
+
return NULL;
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
bool_null_policy nulls;
|
|
493
|
+
if (parse_bool_null_policy(args, &nulls) != TF_OK) return NULL;
|
|
494
|
+
|
|
495
|
+
bool_window_state *st = tf_callocarray_checked(1, sizeof(bool_window_state));
|
|
496
|
+
if (!st) return NULL;
|
|
497
|
+
st->column = tf_strdup_checked(col_j->valuestring);
|
|
498
|
+
st->func = func;
|
|
499
|
+
st->nulls = nulls;
|
|
500
|
+
st->size = win_size;
|
|
501
|
+
st->values = tf_callocarray_checked(win_size, sizeof(uint8_t));
|
|
502
|
+
st->is_null = tf_callocarray_checked(win_size, sizeof(uint8_t));
|
|
503
|
+
|
|
504
|
+
cJSON *res_j = cJSON_GetObjectItemCaseSensitive(args, "result");
|
|
505
|
+
if (cJSON_IsString(res_j)) {
|
|
506
|
+
st->result = tf_strdup_checked(res_j->valuestring);
|
|
507
|
+
} else {
|
|
508
|
+
st->result = window_default_result_name(st->column,
|
|
509
|
+
func == BOOL_ROLL_ANY ? "any" : "all",
|
|
510
|
+
win_size);
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
if (!st->column || !st->result || !st->values || !st->is_null) {
|
|
514
|
+
free(st->column); free(st->result); free(st->values); free(st->is_null); free(st);
|
|
515
|
+
return NULL;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
tf_step *step = tf_callocarray_checked(1, sizeof(tf_step));
|
|
519
|
+
if (!step) { free(st->column); free(st->result); free(st->values); free(st->is_null); free(st); return NULL; }
|
|
520
|
+
step->process = bool_window_process;
|
|
521
|
+
step->flush = bool_window_flush;
|
|
522
|
+
step->destroy = bool_window_destroy;
|
|
523
|
+
step->state = st;
|
|
524
|
+
return step;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
tf_step *tf_rolling_any_create(const cJSON *args) {
|
|
528
|
+
return tf_rolling_bool_create(args, BOOL_ROLL_ANY);
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
tf_step *tf_rolling_all_create(const cJSON *args) {
|
|
532
|
+
return tf_rolling_bool_create(args, BOOL_ROLL_ALL);
|
|
533
|
+
}
|
|
534
|
+
|
|
115
535
|
tf_step *tf_window_create(const cJSON *args) {
|
|
116
536
|
if (!args) return NULL;
|
|
117
537
|
cJSON *col_j = cJSON_GetObjectItemCaseSensitive(args, "column");
|
|
118
|
-
cJSON *size_j = cJSON_GetObjectItemCaseSensitive(args, "size");
|
|
119
538
|
cJSON *func_j = cJSON_GetObjectItemCaseSensitive(args, "func");
|
|
120
|
-
|
|
539
|
+
cJSON *label_j = cJSON_GetObjectItemCaseSensitive(args, "__op_label");
|
|
540
|
+
const char *label = cJSON_IsString(label_j) ? label_j->valuestring : "window";
|
|
541
|
+
size_t win_size = 0;
|
|
542
|
+
int has_size = tf_json_get_size_arg(args, "size",
|
|
543
|
+
1, TF_MAX_WINDOW_SIZE,
|
|
544
|
+
&win_size, label);
|
|
545
|
+
if (has_size < 0) return NULL;
|
|
546
|
+
if (!cJSON_IsString(col_j) || !col_j->valuestring[0] || has_size == 0 || !cJSON_IsString(func_j)) {
|
|
547
|
+
char msg[128];
|
|
548
|
+
snprintf(msg, sizeof(msg), "%s: column, size, and func are required", label);
|
|
549
|
+
tf_set_last_error(msg);
|
|
121
550
|
return NULL;
|
|
551
|
+
}
|
|
122
552
|
|
|
123
|
-
|
|
124
|
-
if (
|
|
553
|
+
win_func func;
|
|
554
|
+
if (parse_win_func(func_j->valuestring, &func) != TF_OK) {
|
|
555
|
+
char msg[128];
|
|
556
|
+
snprintf(msg, sizeof(msg), "%s: func must be avg, sum, min, max, or count", label);
|
|
557
|
+
tf_set_last_error(msg);
|
|
558
|
+
return NULL;
|
|
559
|
+
}
|
|
125
560
|
|
|
126
|
-
window_state *st =
|
|
561
|
+
window_state *st = tf_callocarray_checked(1, sizeof(window_state));
|
|
127
562
|
if (!st) return NULL;
|
|
128
|
-
st->column =
|
|
129
|
-
st->
|
|
563
|
+
st->column = tf_strdup_checked(col_j->valuestring);
|
|
564
|
+
st->op_label = tf_strdup_checked(label);
|
|
565
|
+
st->func = func;
|
|
130
566
|
st->size = win_size;
|
|
131
|
-
st->ring =
|
|
567
|
+
st->ring = tf_callocarray_checked(win_size, sizeof(double));
|
|
568
|
+
if (!st->column || !st->op_label || !st->ring) {
|
|
569
|
+
free(st->column); free(st->op_label); free(st->ring); free(st); return NULL;
|
|
570
|
+
}
|
|
571
|
+
if (window_parse_missing_policy(args, label, &st->missing) != TF_OK ||
|
|
572
|
+
window_parse_type_policy(args, label, &st->on_type_error) != TF_OK) {
|
|
573
|
+
free(st->column); free(st->op_label); free(st->ring); free(st); return NULL;
|
|
574
|
+
}
|
|
132
575
|
|
|
133
576
|
cJSON *res_j = cJSON_GetObjectItemCaseSensitive(args, "result");
|
|
134
577
|
if (cJSON_IsString(res_j)) {
|
|
135
|
-
st->result =
|
|
578
|
+
st->result = tf_strdup_checked(res_j->valuestring);
|
|
136
579
|
} else {
|
|
137
|
-
|
|
138
|
-
snprintf(buf, sizeof(buf), "%s_%s%zu", st->column,
|
|
139
|
-
func_j->valuestring, win_size);
|
|
140
|
-
st->result = strdup(buf);
|
|
580
|
+
st->result = window_default_result_name(st->column, func_j->valuestring, win_size);
|
|
141
581
|
}
|
|
582
|
+
if (!st->result) { free(st->column); free(st->op_label); free(st->ring); free(st); return NULL; }
|
|
142
583
|
|
|
143
|
-
tf_step *step =
|
|
144
|
-
if (!step) { free(st->column); free(st->result); free(st->ring); free(st); return NULL; }
|
|
584
|
+
tf_step *step = tf_callocarray_checked(1, sizeof(tf_step));
|
|
585
|
+
if (!step) { free(st->column); free(st->result); free(st->op_label); free(st->ring); free(st); return NULL; }
|
|
145
586
|
step->process = window_process;
|
|
146
587
|
step->flush = window_flush;
|
|
147
588
|
step->destroy = window_destroy;
|