tranfi 0.0.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -21
- package/NOTICE +8 -0
- package/README.md +627 -0
- package/app/assets/index-6quYZ5Ap.css +5 -0
- package/app/assets/index-BIAIKnrp.js +160 -0
- package/app/assets/materialdesignicons-webfont-B7mPwVP_.ttf +0 -0
- package/app/assets/materialdesignicons-webfont-CSr8KVlo.eot +0 -0
- package/app/assets/materialdesignicons-webfont-Dp5v-WZN.woff2 +0 -0
- package/app/assets/materialdesignicons-webfont-PXm3-2wK.woff +0 -0
- package/app/index.html +13 -0
- package/binding.gyp +121 -0
- package/csrc/arena.c +93 -0
- package/csrc/batch.c +976 -0
- package/csrc/buffer.c +154 -0
- package/csrc/cJSON.c +3386 -0
- package/csrc/cJSON.h +316 -0
- package/csrc/codec_csv.c +1951 -0
- package/csrc/codec_jsonl.c +1086 -0
- package/csrc/codec_table.c +248 -0
- package/csrc/codec_text.c +447 -0
- package/csrc/compiler.c +130 -0
- package/csrc/config.h +21 -0
- package/csrc/date_utils.h +94 -0
- package/csrc/dsl.c +5417 -0
- package/csrc/dsl.h +22 -0
- package/csrc/expr.c +1553 -0
- package/csrc/expr.h +58 -0
- package/csrc/internal.h +539 -0
- package/csrc/ir.c +166 -0
- package/csrc/ir.h +208 -0
- package/csrc/ir_schema.c +75 -0
- package/csrc/ir_serialize.c +166 -0
- package/csrc/ir_sql.c +1822 -0
- package/csrc/ir_validate.c +576 -0
- package/csrc/json_path.c +210 -0
- package/csrc/main.c +1241 -0
- package/csrc/memory_estimate.c +477 -0
- package/csrc/op_acf.c +283 -0
- package/csrc/op_across.c +477 -0
- package/csrc/op_anomaly.c +255 -0
- package/csrc/op_assert.c +761 -0
- package/csrc/op_bin.c +248 -0
- package/csrc/op_cast.c +523 -0
- package/csrc/op_clip.c +99 -0
- package/csrc/op_date_trunc.c +355 -0
- package/csrc/op_datetime.c +394 -0
- package/csrc/op_derive.c +216 -0
- package/csrc/op_diff.c +250 -0
- package/csrc/op_ewma.c +222 -0
- package/csrc/op_explode.c +206 -0
- package/csrc/op_fill_down.c +235 -0
- package/csrc/op_fill_null.c +268 -0
- package/csrc/op_filter.c +181 -0
- package/csrc/op_frequency.c +721 -0
- package/csrc/op_grep.c +181 -0
- package/csrc/op_group_agg.c +1956 -0
- package/csrc/op_hash.c +159 -0
- package/csrc/op_head.c +84 -0
- package/csrc/op_interpolate.c +445 -0
- package/csrc/op_join.c +2902 -0
- package/csrc/op_json_extract.c +227 -0
- package/csrc/op_json_filter.c +384 -0
- package/csrc/op_json_flatten.c +293 -0
- package/csrc/op_json_schema.c +503 -0
- package/csrc/op_label_encode.c +419 -0
- package/csrc/op_lag.c +181 -0
- package/csrc/op_lead.c +242 -0
- package/csrc/op_normalize.c +510 -0
- package/csrc/op_onehot.c +457 -0
- package/csrc/op_pivot.c +1754 -0
- package/csrc/op_quarantine.c +189 -0
- package/csrc/op_registry.c +3044 -0
- package/csrc/op_rename.c +129 -0
- package/csrc/op_replace.c +354 -0
- package/csrc/op_rleid.c +297 -0
- package/csrc/op_rowid.c +559 -0
- package/csrc/op_sample.c +158 -0
- package/csrc/op_schema.c +1341 -0
- package/csrc/op_schema_infer.c +252 -0
- package/csrc/op_select.c +340 -0
- package/csrc/op_set.c +3449 -0
- package/csrc/op_skip.c +95 -0
- package/csrc/op_sort.c +819 -0
- package/csrc/op_source_name.c +120 -0
- package/csrc/op_split.c +151 -0
- package/csrc/op_split_data.c +119 -0
- package/csrc/op_stack.c +271 -0
- package/csrc/op_stats.c +875 -0
- package/csrc/op_step.c +333 -0
- package/csrc/op_tail.c +105 -0
- package/csrc/op_tee.c +338 -0
- package/csrc/op_top.c +357 -0
- package/csrc/op_trim.c +138 -0
- package/csrc/op_unique.c +1343 -0
- package/csrc/op_unpivot.c +193 -0
- package/csrc/op_validate.c +648 -0
- package/csrc/op_window.c +591 -0
- package/csrc/path_policy.c +85 -0
- package/csrc/pipeline.c +1088 -0
- package/csrc/recipes.c +104 -0
- package/csrc/recipes.h +27 -0
- package/csrc/report.c +506 -0
- package/csrc/report.h +22 -0
- package/csrc/selector.c +1097 -0
- package/csrc/size_utils.c +348 -0
- package/csrc/spill.c +317 -0
- package/csrc/spill.h +21 -0
- package/csrc/tranfi.h +291 -0
- package/csrc/transform.h +209 -0
- package/csrc/transform_api.c +2237 -0
- package/csrc/transform_categorical.c +923 -0
- package/csrc/transform_internal.h +472 -0
- package/csrc/transform_json.c +3812 -0
- package/csrc/transform_numeric.c +1966 -0
- package/csrc/transform_sha256.c +154 -0
- package/csrc/transform_wasm.h +162 -0
- package/csrc/transform_wasm_api.c +1373 -0
- package/csrc/wasm_api.c +218 -0
- package/napi_api.c +534 -0
- package/napi_transform.c +1648 -0
- package/napi_transform.h +8 -0
- package/package.json +64 -59
- package/scripts/install-native.js +76 -0
- package/scripts/prepack.js +64 -0
- package/scripts/sync-csrc.js +23 -0
- package/src/cli.js +190 -0
- package/src/engines/duckdb.js +142 -0
- package/src/index.js +925 -0
- package/src/memory_policy.js +411 -0
- package/src/native.js +18 -0
- package/src/pipeline.js +709 -0
- package/src/recipe_json.js +80 -0
- package/src/server.js +279 -0
- package/src/transform.js +403 -0
- package/src/transform_error.js +10 -0
- package/src/wasm.js +21 -0
- package/wasm/index.js +732 -0
- package/wasm/package.json +1 -0
- package/wasm/tranfi_core.js +0 -0
- package/wasm/transform.js +1156 -0
- package/wasm/worker.js +786 -0
- package/dist/bundle.js +0 -1
- package/index.html +0 -18
- package/src/app.css +0 -169
- package/src/app.js +0 -203
- package/src/app.vue +0 -250
- package/src/bulma-input.vue +0 -110
- package/src/common-inputs.js +0 -28
- package/src/main.js +0 -20
- package/src/transforms.js +0 -166
- package/webpack.config.js +0 -108
package/csrc/op_window.c
ADDED
|
@@ -0,0 +1,591 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* op_window.c — Sliding window aggregations.
|
|
3
|
+
*
|
|
4
|
+
* Config: {"column": "price", "size": 3, "func": "avg", "result": "price_avg3"}
|
|
5
|
+
* Supported funcs: avg, sum, min, max, count
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
#include "internal.h"
|
|
9
|
+
#include "cJSON.h"
|
|
10
|
+
#include <stdlib.h>
|
|
11
|
+
#include <string.h>
|
|
12
|
+
#include <stdio.h>
|
|
13
|
+
#include <math.h>
|
|
14
|
+
#include <ctype.h>
|
|
15
|
+
|
|
16
|
+
typedef enum {
|
|
17
|
+
WIN_AVG, WIN_SUM, WIN_MIN, WIN_MAX, WIN_COUNT,
|
|
18
|
+
} win_func;
|
|
19
|
+
|
|
20
|
+
typedef enum {
|
|
21
|
+
WIN_MISSING_ERROR,
|
|
22
|
+
WIN_MISSING_NULL,
|
|
23
|
+
WIN_MISSING_IGNORE,
|
|
24
|
+
} window_missing_policy;
|
|
25
|
+
|
|
26
|
+
typedef enum {
|
|
27
|
+
WIN_TYPE_FAIL,
|
|
28
|
+
WIN_TYPE_NULL,
|
|
29
|
+
} window_type_policy;
|
|
30
|
+
|
|
31
|
+
typedef struct {
|
|
32
|
+
char *column;
|
|
33
|
+
char *result;
|
|
34
|
+
char *op_label;
|
|
35
|
+
win_func func;
|
|
36
|
+
size_t size;
|
|
37
|
+
double *ring; /* circular buffer */
|
|
38
|
+
size_t head; /* next write position */
|
|
39
|
+
size_t count; /* items in buffer */
|
|
40
|
+
window_missing_policy missing;
|
|
41
|
+
window_type_policy on_type_error;
|
|
42
|
+
} window_state;
|
|
43
|
+
|
|
44
|
+
static int parse_win_func(const char *s, win_func *out) {
|
|
45
|
+
if (strcmp(s, "avg") == 0) { *out = WIN_AVG; return TF_OK; }
|
|
46
|
+
if (strcmp(s, "sum") == 0) { *out = WIN_SUM; return TF_OK; }
|
|
47
|
+
if (strcmp(s, "min") == 0) { *out = WIN_MIN; return TF_OK; }
|
|
48
|
+
if (strcmp(s, "max") == 0) { *out = WIN_MAX; return TF_OK; }
|
|
49
|
+
if (strcmp(s, "count") == 0) { *out = WIN_COUNT; return TF_OK; }
|
|
50
|
+
return TF_ERROR;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
static int window_is_numeric_type(tf_type type) {
|
|
54
|
+
return type == TF_TYPE_INT64 || type == TF_TYPE_FLOAT64;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
static double window_get_numeric(const tf_batch *b, size_t r, int ci) {
|
|
58
|
+
if (b->col_types[ci] == TF_TYPE_INT64) return (double)tf_batch_get_int64(b, r, ci);
|
|
59
|
+
return tf_batch_get_float64(b, r, ci);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
static const char *window_label(const window_state *st) {
|
|
63
|
+
return (st && st->op_label) ? st->op_label : "window";
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
static char *window_default_result_name(const char *column, const char *func, size_t size) {
|
|
67
|
+
char size_buf[32];
|
|
68
|
+
int n = snprintf(size_buf, sizeof(size_buf), "%zu", size);
|
|
69
|
+
if (n < 0 || (size_t)n >= sizeof(size_buf)) return NULL;
|
|
70
|
+
|
|
71
|
+
char *func_suffix = tf_string_append_suffix_checked("_", func);
|
|
72
|
+
if (!func_suffix) return NULL;
|
|
73
|
+
char *full_suffix = tf_string_append_suffix_checked(func_suffix, size_buf);
|
|
74
|
+
free(func_suffix);
|
|
75
|
+
if (!full_suffix) return NULL;
|
|
76
|
+
|
|
77
|
+
char *result = tf_string_append_suffix_checked(column, full_suffix);
|
|
78
|
+
free(full_suffix);
|
|
79
|
+
return result;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
static void window_set_col_error(const window_state *st, const char *suffix) {
|
|
83
|
+
char msg[512];
|
|
84
|
+
snprintf(msg, sizeof(msg), "%s: column '%s' %s", window_label(st), st && st->column ? st->column : "", suffix);
|
|
85
|
+
tf_set_last_error(msg);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
static int window_parse_missing_policy(const cJSON *args, const char *label, window_missing_policy *out) {
|
|
89
|
+
*out = WIN_MISSING_ERROR;
|
|
90
|
+
const cJSON *j = cJSON_GetObjectItemCaseSensitive(args, "missing");
|
|
91
|
+
if (!j) return TF_OK;
|
|
92
|
+
if (!cJSON_IsString(j)) {
|
|
93
|
+
char msg[128];
|
|
94
|
+
snprintf(msg, sizeof(msg), "%s: missing must be error, null, or ignore", label);
|
|
95
|
+
tf_set_last_error(msg);
|
|
96
|
+
return TF_ERROR;
|
|
97
|
+
}
|
|
98
|
+
if (strcmp(j->valuestring, "error") == 0) *out = WIN_MISSING_ERROR;
|
|
99
|
+
else if (strcmp(j->valuestring, "null") == 0) *out = WIN_MISSING_NULL;
|
|
100
|
+
else if (strcmp(j->valuestring, "ignore") == 0) *out = WIN_MISSING_IGNORE;
|
|
101
|
+
else {
|
|
102
|
+
char msg[128];
|
|
103
|
+
snprintf(msg, sizeof(msg), "%s: missing must be error, null, or ignore", label);
|
|
104
|
+
tf_set_last_error(msg);
|
|
105
|
+
return TF_ERROR;
|
|
106
|
+
}
|
|
107
|
+
return TF_OK;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
static int window_parse_type_policy(const cJSON *args, const char *label, window_type_policy *out) {
|
|
111
|
+
*out = WIN_TYPE_FAIL;
|
|
112
|
+
const cJSON *j = cJSON_GetObjectItemCaseSensitive(args, "on_type_error");
|
|
113
|
+
if (!j) return TF_OK;
|
|
114
|
+
if (!cJSON_IsString(j)) {
|
|
115
|
+
char msg[128];
|
|
116
|
+
snprintf(msg, sizeof(msg), "%s: on_type_error must be fail or null", label);
|
|
117
|
+
tf_set_last_error(msg);
|
|
118
|
+
return TF_ERROR;
|
|
119
|
+
}
|
|
120
|
+
if (strcmp(j->valuestring, "fail") == 0) *out = WIN_TYPE_FAIL;
|
|
121
|
+
else if (strcmp(j->valuestring, "null") == 0) *out = WIN_TYPE_NULL;
|
|
122
|
+
else {
|
|
123
|
+
char msg[128];
|
|
124
|
+
snprintf(msg, sizeof(msg), "%s: on_type_error must be fail or null", label);
|
|
125
|
+
tf_set_last_error(msg);
|
|
126
|
+
return TF_ERROR;
|
|
127
|
+
}
|
|
128
|
+
return TF_OK;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
static int window_passthrough(tf_batch *in, tf_batch **out) {
|
|
132
|
+
tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
|
|
133
|
+
if (!ob) return TF_ERROR;
|
|
134
|
+
if (tf_batch_clone_schema(ob, in) != TF_OK) {
|
|
135
|
+
tf_batch_free(ob);
|
|
136
|
+
return TF_ERROR;
|
|
137
|
+
}
|
|
138
|
+
for (size_t r = 0; r < in->n_rows; r++) {
|
|
139
|
+
if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
|
|
140
|
+
tf_batch_free(ob);
|
|
141
|
+
return TF_ERROR;
|
|
142
|
+
}
|
|
143
|
+
if (tf_batch_expose_row(ob, r) != TF_OK) {
|
|
144
|
+
tf_batch_free(ob);
|
|
145
|
+
return TF_ERROR;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
*out = ob;
|
|
149
|
+
return TF_OK;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
static int window_process(tf_step *self, tf_batch *in, tf_batch **out,
|
|
153
|
+
tf_side_channels *side) {
|
|
154
|
+
(void)side;
|
|
155
|
+
window_state *st = self->state;
|
|
156
|
+
*out = NULL;
|
|
157
|
+
|
|
158
|
+
int ci = tf_batch_col_index(in, st->column);
|
|
159
|
+
int force_null = 0;
|
|
160
|
+
if (ci < 0) {
|
|
161
|
+
if (st->missing == WIN_MISSING_ERROR) {
|
|
162
|
+
window_set_col_error(st, "not found");
|
|
163
|
+
return TF_ERROR;
|
|
164
|
+
}
|
|
165
|
+
if (st->missing == WIN_MISSING_IGNORE) return window_passthrough(in, out);
|
|
166
|
+
force_null = 1;
|
|
167
|
+
} else if (st->func != WIN_COUNT && !window_is_numeric_type(in->col_types[ci])) {
|
|
168
|
+
if (st->on_type_error == WIN_TYPE_FAIL) {
|
|
169
|
+
window_set_col_error(st, "must be numeric");
|
|
170
|
+
return TF_ERROR;
|
|
171
|
+
}
|
|
172
|
+
force_null = 1;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const char *extra_names[1] = {st->result};
|
|
176
|
+
tf_type extra_types[1] = {TF_TYPE_FLOAT64};
|
|
177
|
+
tf_batch *ob = tf_batch_create(in->n_cols + 1, in->n_rows);
|
|
178
|
+
if (!ob) return TF_ERROR;
|
|
179
|
+
if (tf_batch_clone_with_extra_cols(ob, in, extra_names, extra_types, 1) != TF_OK) {
|
|
180
|
+
tf_batch_free(ob);
|
|
181
|
+
return TF_ERROR;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
for (size_t r = 0; r < in->n_rows; r++) {
|
|
185
|
+
if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
|
|
186
|
+
tf_batch_free(ob);
|
|
187
|
+
return TF_ERROR;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
if (force_null || tf_batch_is_null(in, r, ci)) {
|
|
191
|
+
if (tf_batch_set_null(ob, r, in->n_cols) != TF_OK) { tf_batch_free(ob); return TF_ERROR; }
|
|
192
|
+
if (tf_batch_expose_row(ob, r) != TF_OK) { tf_batch_free(ob); return TF_ERROR; }
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
double val = st->func == WIN_COUNT ? 0.0 : window_get_numeric(in, r, ci);
|
|
197
|
+
size_t next_head = (st->head + 1) % st->size;
|
|
198
|
+
size_t next_count = st->count < st->size ? st->count + 1 : st->count;
|
|
199
|
+
|
|
200
|
+
/* Compute window aggregate */
|
|
201
|
+
double result = 0;
|
|
202
|
+
switch (st->func) {
|
|
203
|
+
case WIN_SUM:
|
|
204
|
+
case WIN_AVG: {
|
|
205
|
+
double sum = 0;
|
|
206
|
+
for (size_t i = 0; i < next_count; i++)
|
|
207
|
+
sum += i == st->head ? val : st->ring[i];
|
|
208
|
+
result = (st->func == WIN_AVG) ? sum / next_count : sum;
|
|
209
|
+
break;
|
|
210
|
+
}
|
|
211
|
+
case WIN_MIN: {
|
|
212
|
+
result = (st->head == 0) ? val : st->ring[0];
|
|
213
|
+
for (size_t i = 1; i < next_count; i++) {
|
|
214
|
+
double item = i == st->head ? val : st->ring[i];
|
|
215
|
+
if (item < result) result = item;
|
|
216
|
+
}
|
|
217
|
+
break;
|
|
218
|
+
}
|
|
219
|
+
case WIN_MAX: {
|
|
220
|
+
result = (st->head == 0) ? val : st->ring[0];
|
|
221
|
+
for (size_t i = 1; i < next_count; i++) {
|
|
222
|
+
double item = i == st->head ? val : st->ring[i];
|
|
223
|
+
if (item > result) result = item;
|
|
224
|
+
}
|
|
225
|
+
break;
|
|
226
|
+
}
|
|
227
|
+
case WIN_COUNT:
|
|
228
|
+
result = (double)next_count;
|
|
229
|
+
break;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
if (tf_batch_set_float64(ob, r, in->n_cols, result) != TF_OK) {
|
|
233
|
+
tf_batch_free(ob);
|
|
234
|
+
return TF_ERROR;
|
|
235
|
+
}
|
|
236
|
+
if (tf_batch_expose_row(ob, r) != TF_OK) {
|
|
237
|
+
tf_batch_free(ob);
|
|
238
|
+
return TF_ERROR;
|
|
239
|
+
}
|
|
240
|
+
st->ring[st->head] = val;
|
|
241
|
+
st->head = next_head;
|
|
242
|
+
st->count = next_count;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
*out = ob;
|
|
246
|
+
return TF_OK;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
static int window_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
|
|
250
|
+
(void)self; (void)side; *out = NULL; return TF_OK;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
static void window_destroy(tf_step *self) {
|
|
254
|
+
window_state *st = self->state;
|
|
255
|
+
if (st) { free(st->column); free(st->result); free(st->op_label); free(st->ring); free(st); }
|
|
256
|
+
free(self);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
static tf_step *tf_window_create_alias(const cJSON *args, const char *func, const char *label) {
|
|
261
|
+
if (!args || !func || !label) return NULL;
|
|
262
|
+
cJSON *copy = cJSON_Duplicate(args, 1);
|
|
263
|
+
if (!copy) return NULL;
|
|
264
|
+
cJSON_DeleteItemFromObjectCaseSensitive(copy, "func");
|
|
265
|
+
cJSON_DeleteItemFromObjectCaseSensitive(copy, "__op_label");
|
|
266
|
+
if (tf_json_add_string(copy, "__op_label", label) != TF_OK ||
|
|
267
|
+
tf_json_add_string(copy, "func", func) != TF_OK) {
|
|
268
|
+
cJSON_Delete(copy);
|
|
269
|
+
return NULL;
|
|
270
|
+
}
|
|
271
|
+
tf_step *step = tf_window_create(copy);
|
|
272
|
+
cJSON_Delete(copy);
|
|
273
|
+
return step;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
tf_step *tf_rolling_sum_create(const cJSON *args) {
|
|
277
|
+
return tf_window_create_alias(args, "sum", "rolling-sum");
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
tf_step *tf_rolling_mean_create(const cJSON *args) {
|
|
281
|
+
return tf_window_create_alias(args, "avg", "rolling-mean");
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
tf_step *tf_rolling_min_create(const cJSON *args) {
|
|
285
|
+
return tf_window_create_alias(args, "min", "rolling-min");
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
tf_step *tf_rolling_max_create(const cJSON *args) {
|
|
289
|
+
return tf_window_create_alias(args, "max", "rolling-max");
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
typedef enum {
|
|
293
|
+
BOOL_ROLL_ANY,
|
|
294
|
+
BOOL_ROLL_ALL,
|
|
295
|
+
} bool_roll_func;
|
|
296
|
+
|
|
297
|
+
typedef enum {
|
|
298
|
+
BOOL_NULLS_IGNORE,
|
|
299
|
+
BOOL_NULLS_FALSE,
|
|
300
|
+
BOOL_NULLS_TRUE,
|
|
301
|
+
BOOL_NULLS_PROPAGATE,
|
|
302
|
+
} bool_null_policy;
|
|
303
|
+
|
|
304
|
+
typedef struct {
|
|
305
|
+
char *column;
|
|
306
|
+
char *result;
|
|
307
|
+
bool_roll_func func;
|
|
308
|
+
bool_null_policy nulls;
|
|
309
|
+
size_t size;
|
|
310
|
+
uint8_t *values;
|
|
311
|
+
uint8_t *is_null;
|
|
312
|
+
size_t head;
|
|
313
|
+
size_t count;
|
|
314
|
+
} bool_window_state;
|
|
315
|
+
|
|
316
|
+
static int streq_ci(const char *a, const char *b) {
|
|
317
|
+
if (!a || !b) return 0;
|
|
318
|
+
while (*a && *b) {
|
|
319
|
+
unsigned char ca = (unsigned char)*a;
|
|
320
|
+
unsigned char cb = (unsigned char)*b;
|
|
321
|
+
if (tolower(ca) != tolower(cb)) return 0;
|
|
322
|
+
a++;
|
|
323
|
+
b++;
|
|
324
|
+
}
|
|
325
|
+
return *a == '\0' && *b == '\0';
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
static int parse_bool_null_policy(const cJSON *args, bool_null_policy *out) {
|
|
329
|
+
*out = BOOL_NULLS_IGNORE;
|
|
330
|
+
cJSON *nulls_j = cJSON_GetObjectItemCaseSensitive(args, "nulls");
|
|
331
|
+
if (!nulls_j) return TF_OK;
|
|
332
|
+
if (!cJSON_IsString(nulls_j)) {
|
|
333
|
+
tf_set_last_error("rolling-any/all: nulls must be ignore, false, true, or propagate");
|
|
334
|
+
return TF_ERROR;
|
|
335
|
+
}
|
|
336
|
+
const char *s = nulls_j->valuestring;
|
|
337
|
+
if (strcmp(s, "ignore") == 0) *out = BOOL_NULLS_IGNORE;
|
|
338
|
+
else if (strcmp(s, "false") == 0) *out = BOOL_NULLS_FALSE;
|
|
339
|
+
else if (strcmp(s, "true") == 0) *out = BOOL_NULLS_TRUE;
|
|
340
|
+
else if (strcmp(s, "propagate") == 0) *out = BOOL_NULLS_PROPAGATE;
|
|
341
|
+
else {
|
|
342
|
+
tf_set_last_error("rolling-any/all: nulls must be ignore, false, true, or propagate");
|
|
343
|
+
return TF_ERROR;
|
|
344
|
+
}
|
|
345
|
+
return TF_OK;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
static int cell_to_bool(const tf_batch *b, size_t row, int col, uint8_t *out) {
|
|
349
|
+
if (col < 0 || tf_batch_is_null(b, row, (size_t)col)) return 0;
|
|
350
|
+
switch (b->col_types[col]) {
|
|
351
|
+
case TF_TYPE_BOOL:
|
|
352
|
+
*out = tf_batch_get_bool(b, row, (size_t)col) ? 1 : 0;
|
|
353
|
+
return 1;
|
|
354
|
+
case TF_TYPE_INT64:
|
|
355
|
+
*out = tf_batch_get_int64(b, row, (size_t)col) != 0 ? 1 : 0;
|
|
356
|
+
return 1;
|
|
357
|
+
case TF_TYPE_FLOAT64:
|
|
358
|
+
*out = tf_batch_get_float64(b, row, (size_t)col) != 0.0 ? 1 : 0;
|
|
359
|
+
return 1;
|
|
360
|
+
case TF_TYPE_STRING: {
|
|
361
|
+
const char *s = tf_batch_get_string(b, row, (size_t)col);
|
|
362
|
+
if (!s) return 0;
|
|
363
|
+
if (streq_ci(s, "true") || streq_ci(s, "t") || streq_ci(s, "yes") ||
|
|
364
|
+
streq_ci(s, "y") || strcmp(s, "1") == 0) {
|
|
365
|
+
*out = 1;
|
|
366
|
+
return 1;
|
|
367
|
+
}
|
|
368
|
+
if (streq_ci(s, "false") || streq_ci(s, "f") || streq_ci(s, "no") ||
|
|
369
|
+
streq_ci(s, "n") || strcmp(s, "0") == 0 || strcmp(s, "") == 0) {
|
|
370
|
+
*out = 0;
|
|
371
|
+
return 1;
|
|
372
|
+
}
|
|
373
|
+
return 0;
|
|
374
|
+
}
|
|
375
|
+
default:
|
|
376
|
+
return 0;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
static int bool_window_process(tf_step *self, tf_batch *in, tf_batch **out,
|
|
381
|
+
tf_side_channels *side) {
|
|
382
|
+
(void)side;
|
|
383
|
+
bool_window_state *st = self->state;
|
|
384
|
+
*out = NULL;
|
|
385
|
+
|
|
386
|
+
const char *extra_names[1] = {st->result};
|
|
387
|
+
tf_type extra_types[1] = {TF_TYPE_BOOL};
|
|
388
|
+
tf_batch *ob = tf_batch_create(in->n_cols + 1, in->n_rows);
|
|
389
|
+
if (!ob) return TF_ERROR;
|
|
390
|
+
if (tf_batch_clone_with_extra_cols(ob, in, extra_names, extra_types, 1) != TF_OK) {
|
|
391
|
+
tf_batch_free(ob);
|
|
392
|
+
return TF_ERROR;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
int ci = tf_batch_col_index(in, st->column);
|
|
396
|
+
|
|
397
|
+
for (size_t r = 0; r < in->n_rows; r++) {
|
|
398
|
+
if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
|
|
399
|
+
tf_batch_free(ob);
|
|
400
|
+
return TF_ERROR;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
uint8_t value = 0;
|
|
404
|
+
uint8_t is_null = cell_to_bool(in, r, ci, &value) ? 0 : 1;
|
|
405
|
+
size_t next_head = (st->head + 1) % st->size;
|
|
406
|
+
size_t next_count = st->count < st->size ? st->count + 1 : st->count;
|
|
407
|
+
|
|
408
|
+
size_t non_null = 0;
|
|
409
|
+
int seen_null = 0;
|
|
410
|
+
int any_true = 0;
|
|
411
|
+
int any_false = 0;
|
|
412
|
+
for (size_t i = 0; i < next_count; i++) {
|
|
413
|
+
uint8_t item_null = i == st->head ? is_null : st->is_null[i];
|
|
414
|
+
uint8_t item_value = i == st->head ? value : st->values[i];
|
|
415
|
+
if (item_null) {
|
|
416
|
+
seen_null = 1;
|
|
417
|
+
continue;
|
|
418
|
+
}
|
|
419
|
+
non_null++;
|
|
420
|
+
if (item_value) any_true = 1;
|
|
421
|
+
else any_false = 1;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
int out_null = 0;
|
|
425
|
+
int out_value = 0;
|
|
426
|
+
switch (st->nulls) {
|
|
427
|
+
case BOOL_NULLS_IGNORE:
|
|
428
|
+
if (non_null == 0) out_null = 1;
|
|
429
|
+
else out_value = (st->func == BOOL_ROLL_ANY) ? any_true : !any_false;
|
|
430
|
+
break;
|
|
431
|
+
case BOOL_NULLS_FALSE:
|
|
432
|
+
out_value = (st->func == BOOL_ROLL_ANY) ? any_true : (!seen_null && !any_false);
|
|
433
|
+
break;
|
|
434
|
+
case BOOL_NULLS_TRUE:
|
|
435
|
+
out_value = (st->func == BOOL_ROLL_ANY) ? (seen_null || any_true) : !any_false;
|
|
436
|
+
break;
|
|
437
|
+
case BOOL_NULLS_PROPAGATE:
|
|
438
|
+
if (seen_null) out_null = 1;
|
|
439
|
+
else out_value = (st->func == BOOL_ROLL_ANY) ? any_true : !any_false;
|
|
440
|
+
break;
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
int write_rc = out_null ? tf_batch_set_null(ob, r, in->n_cols)
|
|
444
|
+
: tf_batch_set_bool(ob, r, in->n_cols, out_value != 0);
|
|
445
|
+
if (write_rc != TF_OK) {
|
|
446
|
+
tf_batch_free(ob);
|
|
447
|
+
return TF_ERROR;
|
|
448
|
+
}
|
|
449
|
+
if (tf_batch_expose_row(ob, r) != TF_OK) {
|
|
450
|
+
tf_batch_free(ob);
|
|
451
|
+
return TF_ERROR;
|
|
452
|
+
}
|
|
453
|
+
st->values[st->head] = value;
|
|
454
|
+
st->is_null[st->head] = is_null;
|
|
455
|
+
st->head = next_head;
|
|
456
|
+
st->count = next_count;
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
*out = ob;
|
|
460
|
+
return TF_OK;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
static int bool_window_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
|
|
464
|
+
(void)self; (void)side; *out = NULL; return TF_OK;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
static void bool_window_destroy(tf_step *self) {
|
|
468
|
+
bool_window_state *st = self->state;
|
|
469
|
+
if (st) {
|
|
470
|
+
free(st->column);
|
|
471
|
+
free(st->result);
|
|
472
|
+
free(st->values);
|
|
473
|
+
free(st->is_null);
|
|
474
|
+
free(st);
|
|
475
|
+
}
|
|
476
|
+
free(self);
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
static tf_step *tf_rolling_bool_create(const cJSON *args, bool_roll_func func) {
|
|
480
|
+
if (!args) return NULL;
|
|
481
|
+
cJSON *col_j = cJSON_GetObjectItemCaseSensitive(args, "column");
|
|
482
|
+
size_t win_size = 0;
|
|
483
|
+
int has_size = tf_json_get_size_arg(args, "size",
|
|
484
|
+
1, TF_MAX_WINDOW_SIZE,
|
|
485
|
+
&win_size, "rolling-any/all");
|
|
486
|
+
if (has_size < 0) return NULL;
|
|
487
|
+
if (!cJSON_IsString(col_j) || has_size == 0) {
|
|
488
|
+
tf_set_last_error("rolling-any/all: requires column and positive size");
|
|
489
|
+
return NULL;
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
bool_null_policy nulls;
|
|
493
|
+
if (parse_bool_null_policy(args, &nulls) != TF_OK) return NULL;
|
|
494
|
+
|
|
495
|
+
bool_window_state *st = tf_callocarray_checked(1, sizeof(bool_window_state));
|
|
496
|
+
if (!st) return NULL;
|
|
497
|
+
st->column = tf_strdup_checked(col_j->valuestring);
|
|
498
|
+
st->func = func;
|
|
499
|
+
st->nulls = nulls;
|
|
500
|
+
st->size = win_size;
|
|
501
|
+
st->values = tf_callocarray_checked(win_size, sizeof(uint8_t));
|
|
502
|
+
st->is_null = tf_callocarray_checked(win_size, sizeof(uint8_t));
|
|
503
|
+
|
|
504
|
+
cJSON *res_j = cJSON_GetObjectItemCaseSensitive(args, "result");
|
|
505
|
+
if (cJSON_IsString(res_j)) {
|
|
506
|
+
st->result = tf_strdup_checked(res_j->valuestring);
|
|
507
|
+
} else {
|
|
508
|
+
st->result = window_default_result_name(st->column,
|
|
509
|
+
func == BOOL_ROLL_ANY ? "any" : "all",
|
|
510
|
+
win_size);
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
if (!st->column || !st->result || !st->values || !st->is_null) {
|
|
514
|
+
free(st->column); free(st->result); free(st->values); free(st->is_null); free(st);
|
|
515
|
+
return NULL;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
tf_step *step = tf_callocarray_checked(1, sizeof(tf_step));
|
|
519
|
+
if (!step) { free(st->column); free(st->result); free(st->values); free(st->is_null); free(st); return NULL; }
|
|
520
|
+
step->process = bool_window_process;
|
|
521
|
+
step->flush = bool_window_flush;
|
|
522
|
+
step->destroy = bool_window_destroy;
|
|
523
|
+
step->state = st;
|
|
524
|
+
return step;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
tf_step *tf_rolling_any_create(const cJSON *args) {
|
|
528
|
+
return tf_rolling_bool_create(args, BOOL_ROLL_ANY);
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
tf_step *tf_rolling_all_create(const cJSON *args) {
|
|
532
|
+
return tf_rolling_bool_create(args, BOOL_ROLL_ALL);
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
tf_step *tf_window_create(const cJSON *args) {
|
|
536
|
+
if (!args) return NULL;
|
|
537
|
+
cJSON *col_j = cJSON_GetObjectItemCaseSensitive(args, "column");
|
|
538
|
+
cJSON *func_j = cJSON_GetObjectItemCaseSensitive(args, "func");
|
|
539
|
+
cJSON *label_j = cJSON_GetObjectItemCaseSensitive(args, "__op_label");
|
|
540
|
+
const char *label = cJSON_IsString(label_j) ? label_j->valuestring : "window";
|
|
541
|
+
size_t win_size = 0;
|
|
542
|
+
int has_size = tf_json_get_size_arg(args, "size",
|
|
543
|
+
1, TF_MAX_WINDOW_SIZE,
|
|
544
|
+
&win_size, label);
|
|
545
|
+
if (has_size < 0) return NULL;
|
|
546
|
+
if (!cJSON_IsString(col_j) || !col_j->valuestring[0] || has_size == 0 || !cJSON_IsString(func_j)) {
|
|
547
|
+
char msg[128];
|
|
548
|
+
snprintf(msg, sizeof(msg), "%s: column, size, and func are required", label);
|
|
549
|
+
tf_set_last_error(msg);
|
|
550
|
+
return NULL;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
win_func func;
|
|
554
|
+
if (parse_win_func(func_j->valuestring, &func) != TF_OK) {
|
|
555
|
+
char msg[128];
|
|
556
|
+
snprintf(msg, sizeof(msg), "%s: func must be avg, sum, min, max, or count", label);
|
|
557
|
+
tf_set_last_error(msg);
|
|
558
|
+
return NULL;
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
window_state *st = tf_callocarray_checked(1, sizeof(window_state));
|
|
562
|
+
if (!st) return NULL;
|
|
563
|
+
st->column = tf_strdup_checked(col_j->valuestring);
|
|
564
|
+
st->op_label = tf_strdup_checked(label);
|
|
565
|
+
st->func = func;
|
|
566
|
+
st->size = win_size;
|
|
567
|
+
st->ring = tf_callocarray_checked(win_size, sizeof(double));
|
|
568
|
+
if (!st->column || !st->op_label || !st->ring) {
|
|
569
|
+
free(st->column); free(st->op_label); free(st->ring); free(st); return NULL;
|
|
570
|
+
}
|
|
571
|
+
if (window_parse_missing_policy(args, label, &st->missing) != TF_OK ||
|
|
572
|
+
window_parse_type_policy(args, label, &st->on_type_error) != TF_OK) {
|
|
573
|
+
free(st->column); free(st->op_label); free(st->ring); free(st); return NULL;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
cJSON *res_j = cJSON_GetObjectItemCaseSensitive(args, "result");
|
|
577
|
+
if (cJSON_IsString(res_j)) {
|
|
578
|
+
st->result = tf_strdup_checked(res_j->valuestring);
|
|
579
|
+
} else {
|
|
580
|
+
st->result = window_default_result_name(st->column, func_j->valuestring, win_size);
|
|
581
|
+
}
|
|
582
|
+
if (!st->result) { free(st->column); free(st->op_label); free(st->ring); free(st); return NULL; }
|
|
583
|
+
|
|
584
|
+
tf_step *step = tf_callocarray_checked(1, sizeof(tf_step));
|
|
585
|
+
if (!step) { free(st->column); free(st->result); free(st->op_label); free(st->ring); free(st); return NULL; }
|
|
586
|
+
step->process = window_process;
|
|
587
|
+
step->flush = window_flush;
|
|
588
|
+
step->destroy = window_destroy;
|
|
589
|
+
step->state = st;
|
|
590
|
+
return step;
|
|
591
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* path_policy.c -- Internal helpers for host-policy-approved file paths.
|
|
3
|
+
*
|
|
4
|
+
* The public policy pass rewrites plan-owned file args to canonical paths.
|
|
5
|
+
* These helpers let operators reject later symlink/path swaps for those
|
|
6
|
+
* rewritten paths while preserving normal fopen() behavior for policy-neutral
|
|
7
|
+
* constructors and custom host resolvers.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
#include "internal.h"
|
|
11
|
+
#include "cJSON.h"
|
|
12
|
+
#include <stdio.h>
|
|
13
|
+
#include <stdlib.h>
|
|
14
|
+
#include <string.h>
|
|
15
|
+
|
|
16
|
+
#if !defined(_WIN32) && !defined(__EMSCRIPTEN__)
|
|
17
|
+
#include <fcntl.h>
|
|
18
|
+
#include <sys/stat.h>
|
|
19
|
+
#include <unistd.h>
|
|
20
|
+
#endif
|
|
21
|
+
|
|
22
|
+
const char *tf_policy_validated_path_arg(const cJSON *args, const char *path_arg) {
|
|
23
|
+
if (!args || !path_arg) return NULL;
|
|
24
|
+
const char *hidden = NULL;
|
|
25
|
+
if (strcmp(path_arg, "file") == 0) {
|
|
26
|
+
hidden = TF_POLICY_VALIDATED_FILE_PATH_ARG;
|
|
27
|
+
} else if (strcmp(path_arg, "rules_file") == 0 || strcmp(path_arg, "rulesFile") == 0) {
|
|
28
|
+
hidden = TF_POLICY_VALIDATED_RULES_FILE_PATH_ARG;
|
|
29
|
+
}
|
|
30
|
+
if (!hidden) return NULL;
|
|
31
|
+
const cJSON *item = cJSON_GetObjectItemCaseSensitive((cJSON *)args, hidden);
|
|
32
|
+
return cJSON_IsString(item) && item->valuestring && item->valuestring[0]
|
|
33
|
+
? item->valuestring
|
|
34
|
+
: NULL;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
#if !defined(_WIN32) && !defined(__EMSCRIPTEN__)
|
|
38
|
+
static int policy_fd_matches_validated_path(int fd, const char *path,
|
|
39
|
+
const char *validated_path) {
|
|
40
|
+
struct stat fd_stat;
|
|
41
|
+
struct stat path_stat;
|
|
42
|
+
if (fstat(fd, &fd_stat) != 0 || !S_ISREG(fd_stat.st_mode)) return TF_ERROR;
|
|
43
|
+
|
|
44
|
+
char *resolved = realpath(path, NULL);
|
|
45
|
+
if (!resolved) return TF_ERROR;
|
|
46
|
+
int same_path = strcmp(resolved, validated_path) == 0;
|
|
47
|
+
int stat_ok = stat(resolved, &path_stat) == 0;
|
|
48
|
+
free(resolved);
|
|
49
|
+
if (!same_path || !stat_ok) return TF_ERROR;
|
|
50
|
+
if (fd_stat.st_dev != path_stat.st_dev || fd_stat.st_ino != path_stat.st_ino) {
|
|
51
|
+
return TF_ERROR;
|
|
52
|
+
}
|
|
53
|
+
return TF_OK;
|
|
54
|
+
}
|
|
55
|
+
#endif
|
|
56
|
+
|
|
57
|
+
FILE *tf_policy_fopen_read(const char *path, const char *validated_path) {
|
|
58
|
+
if (!path || !path[0]) return NULL;
|
|
59
|
+
if (!validated_path || !validated_path[0]) return fopen(path, "rb");
|
|
60
|
+
|
|
61
|
+
#if defined(_WIN32) || defined(__EMSCRIPTEN__)
|
|
62
|
+
return fopen(path, "rb");
|
|
63
|
+
#else
|
|
64
|
+
int flags = O_RDONLY;
|
|
65
|
+
#ifdef O_CLOEXEC
|
|
66
|
+
flags |= O_CLOEXEC;
|
|
67
|
+
#endif
|
|
68
|
+
#ifdef O_NOFOLLOW
|
|
69
|
+
flags |= O_NOFOLLOW;
|
|
70
|
+
#endif
|
|
71
|
+
int fd = open(path, flags);
|
|
72
|
+
if (fd < 0) return NULL;
|
|
73
|
+
if (policy_fd_matches_validated_path(fd, path, validated_path) != TF_OK) {
|
|
74
|
+
close(fd);
|
|
75
|
+
tf_set_last_error("policy path changed before open");
|
|
76
|
+
return NULL;
|
|
77
|
+
}
|
|
78
|
+
FILE *f = fdopen(fd, "rb");
|
|
79
|
+
if (!f) {
|
|
80
|
+
close(fd);
|
|
81
|
+
return NULL;
|
|
82
|
+
}
|
|
83
|
+
return f;
|
|
84
|
+
#endif
|
|
85
|
+
}
|