tranfi 0.0.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -21
- package/NOTICE +8 -0
- package/README.md +627 -0
- package/app/assets/index-6quYZ5Ap.css +5 -0
- package/app/assets/index-BIAIKnrp.js +160 -0
- package/app/assets/materialdesignicons-webfont-B7mPwVP_.ttf +0 -0
- package/app/assets/materialdesignicons-webfont-CSr8KVlo.eot +0 -0
- package/app/assets/materialdesignicons-webfont-Dp5v-WZN.woff2 +0 -0
- package/app/assets/materialdesignicons-webfont-PXm3-2wK.woff +0 -0
- package/app/index.html +13 -0
- package/binding.gyp +121 -0
- package/csrc/arena.c +93 -0
- package/csrc/batch.c +976 -0
- package/csrc/buffer.c +154 -0
- package/csrc/cJSON.c +3386 -0
- package/csrc/cJSON.h +316 -0
- package/csrc/codec_csv.c +1951 -0
- package/csrc/codec_jsonl.c +1086 -0
- package/csrc/codec_table.c +248 -0
- package/csrc/codec_text.c +447 -0
- package/csrc/compiler.c +130 -0
- package/csrc/config.h +21 -0
- package/csrc/date_utils.h +94 -0
- package/csrc/dsl.c +5417 -0
- package/csrc/dsl.h +22 -0
- package/csrc/expr.c +1553 -0
- package/csrc/expr.h +58 -0
- package/csrc/internal.h +539 -0
- package/csrc/ir.c +166 -0
- package/csrc/ir.h +208 -0
- package/csrc/ir_schema.c +75 -0
- package/csrc/ir_serialize.c +166 -0
- package/csrc/ir_sql.c +1822 -0
- package/csrc/ir_validate.c +576 -0
- package/csrc/json_path.c +210 -0
- package/csrc/main.c +1241 -0
- package/csrc/memory_estimate.c +477 -0
- package/csrc/op_acf.c +283 -0
- package/csrc/op_across.c +477 -0
- package/csrc/op_anomaly.c +255 -0
- package/csrc/op_assert.c +761 -0
- package/csrc/op_bin.c +248 -0
- package/csrc/op_cast.c +523 -0
- package/csrc/op_clip.c +99 -0
- package/csrc/op_date_trunc.c +355 -0
- package/csrc/op_datetime.c +394 -0
- package/csrc/op_derive.c +216 -0
- package/csrc/op_diff.c +250 -0
- package/csrc/op_ewma.c +222 -0
- package/csrc/op_explode.c +206 -0
- package/csrc/op_fill_down.c +235 -0
- package/csrc/op_fill_null.c +268 -0
- package/csrc/op_filter.c +181 -0
- package/csrc/op_frequency.c +721 -0
- package/csrc/op_grep.c +181 -0
- package/csrc/op_group_agg.c +1956 -0
- package/csrc/op_hash.c +159 -0
- package/csrc/op_head.c +84 -0
- package/csrc/op_interpolate.c +445 -0
- package/csrc/op_join.c +2902 -0
- package/csrc/op_json_extract.c +227 -0
- package/csrc/op_json_filter.c +384 -0
- package/csrc/op_json_flatten.c +293 -0
- package/csrc/op_json_schema.c +503 -0
- package/csrc/op_label_encode.c +419 -0
- package/csrc/op_lag.c +181 -0
- package/csrc/op_lead.c +242 -0
- package/csrc/op_normalize.c +510 -0
- package/csrc/op_onehot.c +457 -0
- package/csrc/op_pivot.c +1754 -0
- package/csrc/op_quarantine.c +189 -0
- package/csrc/op_registry.c +3044 -0
- package/csrc/op_rename.c +129 -0
- package/csrc/op_replace.c +354 -0
- package/csrc/op_rleid.c +297 -0
- package/csrc/op_rowid.c +559 -0
- package/csrc/op_sample.c +158 -0
- package/csrc/op_schema.c +1341 -0
- package/csrc/op_schema_infer.c +252 -0
- package/csrc/op_select.c +340 -0
- package/csrc/op_set.c +3449 -0
- package/csrc/op_skip.c +95 -0
- package/csrc/op_sort.c +819 -0
- package/csrc/op_source_name.c +120 -0
- package/csrc/op_split.c +151 -0
- package/csrc/op_split_data.c +119 -0
- package/csrc/op_stack.c +271 -0
- package/csrc/op_stats.c +875 -0
- package/csrc/op_step.c +333 -0
- package/csrc/op_tail.c +105 -0
- package/csrc/op_tee.c +338 -0
- package/csrc/op_top.c +357 -0
- package/csrc/op_trim.c +138 -0
- package/csrc/op_unique.c +1343 -0
- package/csrc/op_unpivot.c +193 -0
- package/csrc/op_validate.c +648 -0
- package/csrc/op_window.c +591 -0
- package/csrc/path_policy.c +85 -0
- package/csrc/pipeline.c +1088 -0
- package/csrc/recipes.c +104 -0
- package/csrc/recipes.h +27 -0
- package/csrc/report.c +506 -0
- package/csrc/report.h +22 -0
- package/csrc/selector.c +1097 -0
- package/csrc/size_utils.c +348 -0
- package/csrc/spill.c +317 -0
- package/csrc/spill.h +21 -0
- package/csrc/tranfi.h +291 -0
- package/csrc/transform.h +209 -0
- package/csrc/transform_api.c +2237 -0
- package/csrc/transform_categorical.c +923 -0
- package/csrc/transform_internal.h +472 -0
- package/csrc/transform_json.c +3812 -0
- package/csrc/transform_numeric.c +1966 -0
- package/csrc/transform_sha256.c +154 -0
- package/csrc/transform_wasm.h +162 -0
- package/csrc/transform_wasm_api.c +1373 -0
- package/csrc/wasm_api.c +218 -0
- package/napi_api.c +534 -0
- package/napi_transform.c +1648 -0
- package/napi_transform.h +8 -0
- package/package.json +64 -59
- package/scripts/install-native.js +76 -0
- package/scripts/prepack.js +64 -0
- package/scripts/sync-csrc.js +23 -0
- package/src/cli.js +190 -0
- package/src/engines/duckdb.js +142 -0
- package/src/index.js +925 -0
- package/src/memory_policy.js +411 -0
- package/src/native.js +18 -0
- package/src/pipeline.js +709 -0
- package/src/recipe_json.js +80 -0
- package/src/server.js +279 -0
- package/src/transform.js +403 -0
- package/src/transform_error.js +10 -0
- package/src/wasm.js +21 -0
- package/wasm/index.js +732 -0
- package/wasm/package.json +1 -0
- package/wasm/tranfi_core.js +0 -0
- package/wasm/transform.js +1156 -0
- package/wasm/worker.js +786 -0
- package/dist/bundle.js +0 -1
- package/index.html +0 -18
- package/src/app.css +0 -169
- package/src/app.js +0 -203
- package/src/app.vue +0 -250
- package/src/bulma-input.vue +0 -110
- package/src/common-inputs.js +0 -28
- package/src/main.js +0 -20
- package/src/transforms.js +0 -166
- package/webpack.config.js +0 -108
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* size_utils.c -- checked size arithmetic and JSON size arguments.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
#if defined(__linux__) && !defined(_GNU_SOURCE)
|
|
6
|
+
#define _GNU_SOURCE
|
|
7
|
+
#endif
|
|
8
|
+
|
|
9
|
+
#include "internal.h"
|
|
10
|
+
#include "cJSON.h"
|
|
11
|
+
#include <stdio.h>
|
|
12
|
+
#include <stdlib.h>
|
|
13
|
+
#include <string.h>
|
|
14
|
+
|
|
15
|
+
int tf_format_float64(char *buf, size_t buf_size, double value) {
|
|
16
|
+
if (!buf || buf_size == 0) return TF_ERROR;
|
|
17
|
+
int n = snprintf(buf, buf_size, TF_FLOAT64_ROUNDTRIP_FORMAT, value);
|
|
18
|
+
if (n < 0 || (size_t)n >= buf_size) return TF_ERROR;
|
|
19
|
+
return TF_OK;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
int tf_size_add(size_t a, size_t b, size_t *out) {
|
|
23
|
+
if (!out) return TF_ERROR;
|
|
24
|
+
if (a > SIZE_MAX - b) return TF_ERROR;
|
|
25
|
+
*out = a + b;
|
|
26
|
+
return TF_OK;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
int tf_size_mul(size_t a, size_t b, size_t *out) {
|
|
30
|
+
if (!out) return TF_ERROR;
|
|
31
|
+
if (a != 0 && b > SIZE_MAX / a) return TF_ERROR;
|
|
32
|
+
*out = a * b;
|
|
33
|
+
return TF_OK;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
int tf_size_align(size_t x, size_t align, size_t *out) {
|
|
37
|
+
if (!out || align == 0) return TF_ERROR;
|
|
38
|
+
size_t rem = x % align;
|
|
39
|
+
if (rem == 0) {
|
|
40
|
+
*out = x;
|
|
41
|
+
return TF_OK;
|
|
42
|
+
}
|
|
43
|
+
return tf_size_add(x, align - rem, out);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
int tf_size_grow_pow2(size_t current, size_t min_value, size_t min_capacity, size_t *out) {
|
|
47
|
+
if (!out) return TF_ERROR;
|
|
48
|
+
size_t cap = current ? current : min_capacity;
|
|
49
|
+
if (cap == 0) cap = 1;
|
|
50
|
+
while (cap < min_value) {
|
|
51
|
+
if (cap > SIZE_MAX / 2) return TF_ERROR;
|
|
52
|
+
cap *= 2;
|
|
53
|
+
}
|
|
54
|
+
*out = cap;
|
|
55
|
+
return TF_OK;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
void *tf_mallocarray_checked(size_t count, size_t elem_size) {
|
|
59
|
+
size_t bytes = 0;
|
|
60
|
+
if (tf_size_mul(count, elem_size, &bytes) != TF_OK) return NULL;
|
|
61
|
+
if (bytes == 0) bytes = 1;
|
|
62
|
+
return malloc(bytes);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
void *tf_callocarray_checked(size_t count, size_t elem_size) {
|
|
66
|
+
if (count == 0 || elem_size == 0) return calloc(1, 1);
|
|
67
|
+
size_t bytes = 0;
|
|
68
|
+
if (tf_size_mul(count, elem_size, &bytes) != TF_OK) return NULL;
|
|
69
|
+
return calloc(1, bytes);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
void *tf_reallocarray_checked(void *ptr, size_t count, size_t elem_size) {
|
|
73
|
+
size_t bytes = 0;
|
|
74
|
+
if (tf_size_mul(count, elem_size, &bytes) != TF_OK) return NULL;
|
|
75
|
+
if (bytes == 0) bytes = 1;
|
|
76
|
+
return realloc(ptr, bytes);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
char *tf_strdup_checked(const char *s) {
|
|
80
|
+
const char *src = s ? s : "";
|
|
81
|
+
size_t len = strlen(src);
|
|
82
|
+
size_t bytes = 0;
|
|
83
|
+
if (tf_size_add(len, 1, &bytes) != TF_OK) return NULL;
|
|
84
|
+
char *copy = tf_mallocarray_checked(bytes, sizeof(char));
|
|
85
|
+
if (!copy) return NULL;
|
|
86
|
+
memcpy(copy, src, bytes);
|
|
87
|
+
return copy;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
char *tf_string_append_suffix_checked(const char *prefix, const char *suffix) {
|
|
91
|
+
const char *a = prefix ? prefix : "";
|
|
92
|
+
const char *b = suffix ? suffix : "";
|
|
93
|
+
size_t a_len = strlen(a);
|
|
94
|
+
size_t b_len = strlen(b);
|
|
95
|
+
size_t text_len = 0;
|
|
96
|
+
size_t bytes = 0;
|
|
97
|
+
if (tf_size_add(a_len, b_len, &text_len) != TF_OK ||
|
|
98
|
+
tf_size_add(text_len, 1, &bytes) != TF_OK) return NULL;
|
|
99
|
+
char *copy = tf_mallocarray_checked(bytes, sizeof(char));
|
|
100
|
+
if (!copy) return NULL;
|
|
101
|
+
memcpy(copy, a, a_len);
|
|
102
|
+
memcpy(copy + a_len, b, b_len);
|
|
103
|
+
copy[text_len] = '\0';
|
|
104
|
+
return copy;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
static void tf_index_swap(size_t *a, size_t *b) {
|
|
108
|
+
size_t tmp = *a;
|
|
109
|
+
*a = *b;
|
|
110
|
+
*b = tmp;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
static void tf_sort_indices_sift_down(size_t *indices, size_t start, size_t end,
|
|
114
|
+
tf_index_compare_fn compare, const void *ctx) {
|
|
115
|
+
size_t root = start;
|
|
116
|
+
while (root <= (end - 1) / 2) {
|
|
117
|
+
size_t child = root * 2 + 1;
|
|
118
|
+
size_t swap_idx = root;
|
|
119
|
+
if (compare(ctx, indices[swap_idx], indices[child]) < 0) {
|
|
120
|
+
swap_idx = child;
|
|
121
|
+
}
|
|
122
|
+
if (child + 1 <= end &&
|
|
123
|
+
compare(ctx, indices[swap_idx], indices[child + 1]) < 0) {
|
|
124
|
+
swap_idx = child + 1;
|
|
125
|
+
}
|
|
126
|
+
if (swap_idx == root) return;
|
|
127
|
+
tf_index_swap(&indices[root], &indices[swap_idx]);
|
|
128
|
+
root = swap_idx;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
static void tf_sort_indices_heap_range(size_t *indices, size_t n,
|
|
133
|
+
tf_index_compare_fn compare, const void *ctx) {
|
|
134
|
+
size_t start = (n - 2) / 2 + 1;
|
|
135
|
+
while (start > 0) {
|
|
136
|
+
start--;
|
|
137
|
+
tf_sort_indices_sift_down(indices, start, n - 1, compare, ctx);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
size_t end = n - 1;
|
|
141
|
+
while (end > 0) {
|
|
142
|
+
tf_index_swap(&indices[0], &indices[end]);
|
|
143
|
+
end--;
|
|
144
|
+
if (end > 0) tf_sort_indices_sift_down(indices, 0, end, compare, ctx);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
static void tf_sort_indices_insertion_range(size_t *indices, size_t lo, size_t hi,
|
|
149
|
+
tf_index_compare_fn compare, const void *ctx) {
|
|
150
|
+
for (size_t i = lo + 1; i < hi; i++) {
|
|
151
|
+
size_t v = indices[i];
|
|
152
|
+
size_t j = i;
|
|
153
|
+
while (j > lo && compare(ctx, v, indices[j - 1]) < 0) {
|
|
154
|
+
indices[j] = indices[j - 1];
|
|
155
|
+
j--;
|
|
156
|
+
}
|
|
157
|
+
indices[j] = v;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
static size_t tf_sort_indices_median3(size_t *indices, size_t a, size_t b, size_t c,
|
|
162
|
+
tf_index_compare_fn compare, const void *ctx) {
|
|
163
|
+
size_t va = indices[a];
|
|
164
|
+
size_t vb = indices[b];
|
|
165
|
+
size_t vc = indices[c];
|
|
166
|
+
|
|
167
|
+
if (compare(ctx, va, vb) < 0) {
|
|
168
|
+
if (compare(ctx, vb, vc) < 0) return vb;
|
|
169
|
+
return compare(ctx, va, vc) < 0 ? vc : va;
|
|
170
|
+
}
|
|
171
|
+
if (compare(ctx, va, vc) < 0) return va;
|
|
172
|
+
return compare(ctx, vb, vc) < 0 ? vc : vb;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
static size_t tf_sort_indices_partition(size_t *indices, size_t lo, size_t hi, size_t pivot,
|
|
176
|
+
tf_index_compare_fn compare, const void *ctx) {
|
|
177
|
+
size_t i = lo;
|
|
178
|
+
size_t j = hi;
|
|
179
|
+
for (;;) {
|
|
180
|
+
while (compare(ctx, indices[i], pivot) < 0) i++;
|
|
181
|
+
do {
|
|
182
|
+
j--;
|
|
183
|
+
} while (compare(ctx, pivot, indices[j]) < 0);
|
|
184
|
+
if (i >= j) return i;
|
|
185
|
+
tf_index_swap(&indices[i], &indices[j]);
|
|
186
|
+
i++;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
static size_t tf_sort_indices_depth_limit(size_t n) {
|
|
191
|
+
size_t depth = 0;
|
|
192
|
+
while (n > 1) {
|
|
193
|
+
depth++;
|
|
194
|
+
n >>= 1;
|
|
195
|
+
}
|
|
196
|
+
return depth * 2;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
static void tf_sort_indices_intro_loop(size_t *indices, size_t lo, size_t hi, size_t depth_limit,
|
|
200
|
+
tf_index_compare_fn compare, const void *ctx) {
|
|
201
|
+
enum { INSERTION_THRESHOLD = 16 };
|
|
202
|
+
|
|
203
|
+
while (hi - lo > INSERTION_THRESHOLD) {
|
|
204
|
+
if (depth_limit == 0) {
|
|
205
|
+
tf_sort_indices_heap_range(indices + lo, hi - lo, compare, ctx);
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
depth_limit--;
|
|
209
|
+
|
|
210
|
+
size_t mid = lo + (hi - lo) / 2;
|
|
211
|
+
size_t pivot = tf_sort_indices_median3(indices, lo, mid, hi - 1, compare, ctx);
|
|
212
|
+
size_t cut = tf_sort_indices_partition(indices, lo, hi, pivot, compare, ctx);
|
|
213
|
+
if (cut <= lo || cut >= hi) {
|
|
214
|
+
tf_sort_indices_heap_range(indices + lo, hi - lo, compare, ctx);
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
if (cut - lo < hi - cut) {
|
|
219
|
+
tf_sort_indices_intro_loop(indices, lo, cut, depth_limit, compare, ctx);
|
|
220
|
+
lo = cut;
|
|
221
|
+
} else {
|
|
222
|
+
tf_sort_indices_intro_loop(indices, cut, hi, depth_limit, compare, ctx);
|
|
223
|
+
hi = cut;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
tf_sort_indices_insertion_range(indices, lo, hi, compare, ctx);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
typedef struct {
|
|
230
|
+
tf_index_compare_fn compare;
|
|
231
|
+
const void *ctx;
|
|
232
|
+
} tf_sort_indices_qsort_ctx;
|
|
233
|
+
|
|
234
|
+
static int tf_sort_indices_total_compare(const void *ctx, size_t ia, size_t ib) {
|
|
235
|
+
const tf_sort_indices_qsort_ctx *qctx = (const tf_sort_indices_qsort_ctx *)ctx;
|
|
236
|
+
int result = qctx->compare(qctx->ctx, ia, ib);
|
|
237
|
+
if (result != 0) return result;
|
|
238
|
+
return ia < ib ? -1 : ia > ib ? 1 : 0;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
#if defined(__GLIBC__)
|
|
242
|
+
static int tf_sort_indices_qsort_compare_glibc(const void *a, const void *b, void *thunk) {
|
|
243
|
+
const tf_sort_indices_qsort_ctx *qctx = (const tf_sort_indices_qsort_ctx *)thunk;
|
|
244
|
+
size_t ia = *(const size_t *)a;
|
|
245
|
+
size_t ib = *(const size_t *)b;
|
|
246
|
+
return tf_sort_indices_total_compare(qctx, ia, ib);
|
|
247
|
+
}
|
|
248
|
+
#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__)
|
|
249
|
+
static int tf_sort_indices_qsort_compare_bsd(void *thunk, const void *a, const void *b) {
|
|
250
|
+
const tf_sort_indices_qsort_ctx *qctx = (const tf_sort_indices_qsort_ctx *)thunk;
|
|
251
|
+
size_t ia = *(const size_t *)a;
|
|
252
|
+
size_t ib = *(const size_t *)b;
|
|
253
|
+
return tf_sort_indices_total_compare(qctx, ia, ib);
|
|
254
|
+
}
|
|
255
|
+
#endif
|
|
256
|
+
|
|
257
|
+
void tf_sort_indices(size_t *indices, size_t n, tf_index_compare_fn compare, const void *ctx) {
|
|
258
|
+
if (!indices || !compare || n < 2) return;
|
|
259
|
+
|
|
260
|
+
#if defined(__GLIBC__)
|
|
261
|
+
tf_sort_indices_qsort_ctx qctx = { compare, ctx };
|
|
262
|
+
qsort_r(indices, n, sizeof(size_t), tf_sort_indices_qsort_compare_glibc, &qctx);
|
|
263
|
+
#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__)
|
|
264
|
+
tf_sort_indices_qsort_ctx qctx = { compare, ctx };
|
|
265
|
+
qsort_r(indices, n, sizeof(size_t), &qctx, tf_sort_indices_qsort_compare_bsd);
|
|
266
|
+
#else
|
|
267
|
+
tf_sort_indices_qsort_ctx qctx = { compare, ctx };
|
|
268
|
+
tf_sort_indices_intro_loop(indices, 0, n, tf_sort_indices_depth_limit(n),
|
|
269
|
+
tf_sort_indices_total_compare, &qctx);
|
|
270
|
+
#endif
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
int tf_check_byte_limit(size_t n, size_t max_value,
|
|
274
|
+
const char *context, const char *name) {
|
|
275
|
+
if (n <= max_value) return TF_OK;
|
|
276
|
+
char msg[256];
|
|
277
|
+
snprintf(msg, sizeof(msg), "%s: %s exceeds maximum %zu bytes",
|
|
278
|
+
context ? context : "size", name ? name : "value", max_value);
|
|
279
|
+
tf_set_last_error(msg);
|
|
280
|
+
return TF_ERROR;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
int tf_string_length_bounded(const char *s, size_t max_value,
|
|
284
|
+
size_t *out, const char *context,
|
|
285
|
+
const char *name) {
|
|
286
|
+
if (!s || !out) return TF_ERROR;
|
|
287
|
+
for (size_t i = 0; ; i++) {
|
|
288
|
+
if (s[i] == '\0') {
|
|
289
|
+
*out = i;
|
|
290
|
+
return TF_OK;
|
|
291
|
+
}
|
|
292
|
+
if (i == max_value) break;
|
|
293
|
+
}
|
|
294
|
+
return tf_check_byte_limit(max_value == SIZE_MAX ? SIZE_MAX : max_value + 1,
|
|
295
|
+
max_value, context, name);
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
int tf_json_size_value(const cJSON *item, const char *name,
|
|
299
|
+
size_t min_value, size_t max_value,
|
|
300
|
+
size_t *out, const char *context) {
|
|
301
|
+
if (!out) return -1;
|
|
302
|
+
if (!item) return 0;
|
|
303
|
+
double d = item->valuedouble;
|
|
304
|
+
if (!cJSON_IsNumber(item) || !(d >= 0.0) ||
|
|
305
|
+
d < (double)min_value || d > (double)max_value) {
|
|
306
|
+
char msg[256];
|
|
307
|
+
snprintf(msg, sizeof(msg), "%s: %s must be an integer between %zu and %zu",
|
|
308
|
+
context ? context : "argument", name, min_value, max_value);
|
|
309
|
+
tf_set_last_error(msg);
|
|
310
|
+
return -1;
|
|
311
|
+
}
|
|
312
|
+
size_t parsed = (size_t)d;
|
|
313
|
+
if ((double)parsed != d) {
|
|
314
|
+
char msg[256];
|
|
315
|
+
snprintf(msg, sizeof(msg), "%s: %s must be an integer between %zu and %zu",
|
|
316
|
+
context ? context : "argument", name, min_value, max_value);
|
|
317
|
+
tf_set_last_error(msg);
|
|
318
|
+
return -1;
|
|
319
|
+
}
|
|
320
|
+
*out = parsed;
|
|
321
|
+
return 1;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
int tf_json_get_size_arg(const cJSON *args, const char *name,
|
|
325
|
+
size_t min_value, size_t max_value,
|
|
326
|
+
size_t *out, const char *context) {
|
|
327
|
+
if (!out) return -1;
|
|
328
|
+
if (!args || !name) return 0;
|
|
329
|
+
const cJSON *item = cJSON_GetObjectItemCaseSensitive(args, name);
|
|
330
|
+
if (!item) return 0;
|
|
331
|
+
return tf_json_size_value(item, name, min_value, max_value, out, context);
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
int tf_json_get_size_arg_any(const cJSON *args, const char *name,
|
|
335
|
+
const char *alt_name,
|
|
336
|
+
size_t min_value, size_t max_value,
|
|
337
|
+
size_t *out, const char *context) {
|
|
338
|
+
if (!out) return -1;
|
|
339
|
+
if (!args || !name) return 0;
|
|
340
|
+
const cJSON *item = cJSON_GetObjectItemCaseSensitive(args, name);
|
|
341
|
+
const char *used_name = name;
|
|
342
|
+
if (!item && alt_name) {
|
|
343
|
+
item = cJSON_GetObjectItemCaseSensitive(args, alt_name);
|
|
344
|
+
used_name = alt_name;
|
|
345
|
+
}
|
|
346
|
+
if (!item) return 0;
|
|
347
|
+
return tf_json_size_value(item, used_name, min_value, max_value, out, context);
|
|
348
|
+
}
|
package/csrc/spill.c
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* spill.c -- secure spill session implementation.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
#include "spill.h"
|
|
6
|
+
#include "internal.h"
|
|
7
|
+
|
|
8
|
+
#ifdef __EMSCRIPTEN__
|
|
9
|
+
|
|
10
|
+
int tf_spill_session_create(const char *root, tf_spill_session **out) {
|
|
11
|
+
(void)root;
|
|
12
|
+
if (out) *out = NULL;
|
|
13
|
+
tf_set_last_error("spill is not supported in WASM");
|
|
14
|
+
return TF_ERROR;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
int tf_spill_open_run(tf_spill_session *s, const char *label, int *fd, char **path_out) {
|
|
18
|
+
(void)s; (void)label;
|
|
19
|
+
if (fd) *fd = -1;
|
|
20
|
+
if (path_out) *path_out = NULL;
|
|
21
|
+
tf_set_last_error("spill is not supported in WASM");
|
|
22
|
+
return TF_ERROR;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
FILE *tf_spill_open_run_file(tf_spill_session *s, const char *label, char **path_out) {
|
|
26
|
+
(void)s; (void)label;
|
|
27
|
+
if (path_out) *path_out = NULL;
|
|
28
|
+
tf_set_last_error("spill is not supported in WASM");
|
|
29
|
+
return NULL;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
int tf_spill_cleanup(tf_spill_session *s) {
|
|
33
|
+
(void)s;
|
|
34
|
+
return TF_OK;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
#else
|
|
38
|
+
|
|
39
|
+
#include <errno.h>
|
|
40
|
+
#include <fcntl.h>
|
|
41
|
+
#include <stdint.h>
|
|
42
|
+
#include <stdatomic.h>
|
|
43
|
+
#include <stdio.h>
|
|
44
|
+
#include <stdlib.h>
|
|
45
|
+
#include <string.h>
|
|
46
|
+
#include <sys/stat.h>
|
|
47
|
+
#include <sys/types.h>
|
|
48
|
+
#include <time.h>
|
|
49
|
+
#include <unistd.h>
|
|
50
|
+
|
|
51
|
+
#ifndef O_CLOEXEC
|
|
52
|
+
#define O_CLOEXEC 0
|
|
53
|
+
#endif
|
|
54
|
+
#ifndef O_DIRECTORY
|
|
55
|
+
#define O_DIRECTORY 0
|
|
56
|
+
#endif
|
|
57
|
+
#ifndef O_NOFOLLOW
|
|
58
|
+
#define O_NOFOLLOW 0
|
|
59
|
+
#endif
|
|
60
|
+
|
|
61
|
+
struct tf_spill_session {
|
|
62
|
+
int dirfd;
|
|
63
|
+
char *path;
|
|
64
|
+
char **names;
|
|
65
|
+
size_t n_names;
|
|
66
|
+
size_t cap_names;
|
|
67
|
+
size_t seq;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
static void set_spill_errno(const char *prefix, const char *path) {
|
|
71
|
+
char msg[512];
|
|
72
|
+
snprintf(msg, sizeof(msg), "%s '%s': %s", prefix, path ? path : "", strerror(errno));
|
|
73
|
+
tf_set_last_error(msg);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
static uint64_t random_u64(void) {
|
|
77
|
+
uint64_t v = 0;
|
|
78
|
+
int fd = open("/dev/urandom", O_RDONLY | O_CLOEXEC);
|
|
79
|
+
if (fd >= 0) {
|
|
80
|
+
ssize_t n = read(fd, &v, sizeof(v));
|
|
81
|
+
close(fd);
|
|
82
|
+
if (n == (ssize_t)sizeof(v)) return v;
|
|
83
|
+
}
|
|
84
|
+
static atomic_uint_fast64_t counter = 0;
|
|
85
|
+
uint64_t c = (uint64_t)atomic_fetch_add_explicit(&counter, 1, memory_order_relaxed) + 1u;
|
|
86
|
+
v = (uint64_t)time(NULL);
|
|
87
|
+
v ^= ((uint64_t)getpid()) << 32;
|
|
88
|
+
v ^= (uint64_t)(uintptr_t)&v;
|
|
89
|
+
v ^= c * UINT64_C(0x9e3779b97f4a7c15);
|
|
90
|
+
return v;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
static int append_name(tf_spill_session *s, char *name) {
|
|
94
|
+
if (s->n_names == s->cap_names) {
|
|
95
|
+
size_t need = 0;
|
|
96
|
+
size_t new_cap = 0;
|
|
97
|
+
if (tf_size_add(s->n_names, 1, &need) != TF_OK ||
|
|
98
|
+
tf_size_grow_pow2(s->cap_names, need, 16, &new_cap) != TF_OK) {
|
|
99
|
+
return TF_ERROR;
|
|
100
|
+
}
|
|
101
|
+
char **tmp = tf_reallocarray_checked(s->names, new_cap, sizeof(char *));
|
|
102
|
+
if (!tmp) return TF_ERROR;
|
|
103
|
+
s->names = tmp;
|
|
104
|
+
s->cap_names = new_cap;
|
|
105
|
+
}
|
|
106
|
+
s->names[s->n_names++] = name;
|
|
107
|
+
return TF_OK;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
static char *join_path(const char *dir, const char *name) {
|
|
111
|
+
size_t dlen = strlen(dir);
|
|
112
|
+
size_t nlen = strlen(name);
|
|
113
|
+
int need_sep = dlen > 0 && dir[dlen - 1] != '/';
|
|
114
|
+
char *path = malloc(dlen + (need_sep ? 1u : 0u) + nlen + 1u);
|
|
115
|
+
if (!path) return NULL;
|
|
116
|
+
memcpy(path, dir, dlen);
|
|
117
|
+
size_t off = dlen;
|
|
118
|
+
if (need_sep) path[off++] = '/';
|
|
119
|
+
memcpy(path + off, name, nlen + 1u);
|
|
120
|
+
return path;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
static void sanitize_label(const char *label, char out[32]) {
|
|
124
|
+
size_t j = 0;
|
|
125
|
+
if (!label || !label[0]) label = "run";
|
|
126
|
+
for (size_t i = 0; label[i] && j < 31; i++) {
|
|
127
|
+
unsigned char ch = (unsigned char)label[i];
|
|
128
|
+
if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
|
|
129
|
+
(ch >= '0' && ch <= '9') || ch == '-' || ch == '_') {
|
|
130
|
+
out[j++] = (char)ch;
|
|
131
|
+
} else {
|
|
132
|
+
out[j++] = '-';
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
if (j == 0) out[j++] = 'r';
|
|
136
|
+
out[j] = '\0';
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
static int validate_root_fd(int fd, const char *root) {
|
|
140
|
+
struct stat st;
|
|
141
|
+
if (fstat(fd, &st) != 0) {
|
|
142
|
+
set_spill_errno("spill: cannot stat root", root);
|
|
143
|
+
return TF_ERROR;
|
|
144
|
+
}
|
|
145
|
+
if (!S_ISDIR(st.st_mode)) {
|
|
146
|
+
tf_set_last_error("spill: root is not a directory");
|
|
147
|
+
return TF_ERROR;
|
|
148
|
+
}
|
|
149
|
+
#ifdef S_ISVTX
|
|
150
|
+
if ((st.st_mode & S_IWOTH) && !(st.st_mode & S_ISVTX)) {
|
|
151
|
+
#else
|
|
152
|
+
if (st.st_mode & S_IWOTH) {
|
|
153
|
+
#endif
|
|
154
|
+
tf_set_last_error("spill: root is world-writable without sticky bit");
|
|
155
|
+
return TF_ERROR;
|
|
156
|
+
}
|
|
157
|
+
if (st.st_uid != geteuid()) {
|
|
158
|
+
tf_set_last_error("spill: root is not owned by the current user");
|
|
159
|
+
return TF_ERROR;
|
|
160
|
+
}
|
|
161
|
+
return TF_OK;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
int tf_spill_session_create(const char *root, tf_spill_session **out) {
|
|
165
|
+
if (!out) {
|
|
166
|
+
tf_set_last_error("spill: output session pointer is required");
|
|
167
|
+
return TF_ERROR;
|
|
168
|
+
}
|
|
169
|
+
*out = NULL;
|
|
170
|
+
if (!root || !root[0]) {
|
|
171
|
+
tf_set_last_error("spill: root directory is required");
|
|
172
|
+
return TF_ERROR;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
int rootfd = open(root, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW);
|
|
176
|
+
if (rootfd < 0) {
|
|
177
|
+
set_spill_errno("spill: cannot open root", root);
|
|
178
|
+
return TF_ERROR;
|
|
179
|
+
}
|
|
180
|
+
if (validate_root_fd(rootfd, root) != TF_OK) {
|
|
181
|
+
close(rootfd);
|
|
182
|
+
return TF_ERROR;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
tf_spill_session *s = calloc(1, sizeof(*s));
|
|
186
|
+
if (!s) {
|
|
187
|
+
close(rootfd);
|
|
188
|
+
tf_set_last_error("spill: out of memory");
|
|
189
|
+
return TF_ERROR;
|
|
190
|
+
}
|
|
191
|
+
s->dirfd = -1;
|
|
192
|
+
|
|
193
|
+
char name[96];
|
|
194
|
+
int created = 0;
|
|
195
|
+
for (int attempt = 0; attempt < 128; attempt++) {
|
|
196
|
+
snprintf(name, sizeof(name), "tranfi-spill-%016llx-%016llx",
|
|
197
|
+
(unsigned long long)random_u64(), (unsigned long long)random_u64());
|
|
198
|
+
if (mkdirat(rootfd, name, 0700) == 0) {
|
|
199
|
+
created = 1;
|
|
200
|
+
break;
|
|
201
|
+
}
|
|
202
|
+
if (errno != EEXIST) {
|
|
203
|
+
set_spill_errno("spill: cannot create private directory under", root);
|
|
204
|
+
close(rootfd);
|
|
205
|
+
free(s);
|
|
206
|
+
return TF_ERROR;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
if (!created) {
|
|
210
|
+
tf_set_last_error("spill: could not allocate a unique private directory");
|
|
211
|
+
close(rootfd);
|
|
212
|
+
free(s);
|
|
213
|
+
return TF_ERROR;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
s->path = join_path(root, name);
|
|
217
|
+
if (!s->path) {
|
|
218
|
+
unlinkat(rootfd, name, AT_REMOVEDIR);
|
|
219
|
+
close(rootfd);
|
|
220
|
+
free(s);
|
|
221
|
+
tf_set_last_error("spill: out of memory");
|
|
222
|
+
return TF_ERROR;
|
|
223
|
+
}
|
|
224
|
+
s->dirfd = openat(rootfd, name, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW);
|
|
225
|
+
close(rootfd);
|
|
226
|
+
if (s->dirfd < 0) {
|
|
227
|
+
set_spill_errno("spill: cannot open private directory", s->path);
|
|
228
|
+
rmdir(s->path);
|
|
229
|
+
free(s->path);
|
|
230
|
+
free(s);
|
|
231
|
+
return TF_ERROR;
|
|
232
|
+
}
|
|
233
|
+
(void)fchmod(s->dirfd, 0700);
|
|
234
|
+
if (out) *out = s;
|
|
235
|
+
return TF_OK;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
int tf_spill_open_run(tf_spill_session *s, const char *label, int *fd, char **path_out) {
|
|
239
|
+
if (fd) *fd = -1;
|
|
240
|
+
if (path_out) *path_out = NULL;
|
|
241
|
+
if (!s || s->dirfd < 0) {
|
|
242
|
+
tf_set_last_error("spill: invalid spill session");
|
|
243
|
+
return TF_ERROR;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
char clean[32];
|
|
247
|
+
sanitize_label(label, clean);
|
|
248
|
+
for (int attempt = 0; attempt < 128; attempt++) {
|
|
249
|
+
char name_buf[128];
|
|
250
|
+
snprintf(name_buf, sizeof(name_buf), "%s-%zu-%016llx.bin",
|
|
251
|
+
clean, s->seq++, (unsigned long long)random_u64());
|
|
252
|
+
int runfd = openat(s->dirfd, name_buf,
|
|
253
|
+
O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC,
|
|
254
|
+
0600);
|
|
255
|
+
if (runfd < 0) {
|
|
256
|
+
if (errno == EEXIST) continue;
|
|
257
|
+
set_spill_errno("spill: cannot create run", name_buf);
|
|
258
|
+
return TF_ERROR;
|
|
259
|
+
}
|
|
260
|
+
char *tracked = strdup(name_buf);
|
|
261
|
+
char *path = join_path(s->path, name_buf);
|
|
262
|
+
if (!tracked || !path || append_name(s, tracked) != TF_OK) {
|
|
263
|
+
close(runfd);
|
|
264
|
+
unlinkat(s->dirfd, name_buf, 0);
|
|
265
|
+
free(tracked);
|
|
266
|
+
free(path);
|
|
267
|
+
tf_set_last_error("spill: out of memory");
|
|
268
|
+
return TF_ERROR;
|
|
269
|
+
}
|
|
270
|
+
if (fd) *fd = runfd;
|
|
271
|
+
else close(runfd);
|
|
272
|
+
if (path_out) *path_out = path;
|
|
273
|
+
else free(path);
|
|
274
|
+
return TF_OK;
|
|
275
|
+
}
|
|
276
|
+
tf_set_last_error("spill: could not allocate a unique run file");
|
|
277
|
+
return TF_ERROR;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
FILE *tf_spill_open_run_file(tf_spill_session *s, const char *label, char **path_out) {
|
|
281
|
+
int fd = -1;
|
|
282
|
+
char *path = NULL;
|
|
283
|
+
if (tf_spill_open_run(s, label, &fd, &path) != TF_OK) return NULL;
|
|
284
|
+
FILE *f = fdopen(fd, "wb");
|
|
285
|
+
if (!f) {
|
|
286
|
+
char msg[512];
|
|
287
|
+
snprintf(msg, sizeof(msg), "spill: cannot fdopen run '%s': %s", path ? path : "", strerror(errno));
|
|
288
|
+
tf_set_last_error(msg);
|
|
289
|
+
close(fd);
|
|
290
|
+
if (path) remove(path);
|
|
291
|
+
free(path);
|
|
292
|
+
if (path_out) *path_out = NULL;
|
|
293
|
+
return NULL;
|
|
294
|
+
}
|
|
295
|
+
if (path_out) *path_out = path;
|
|
296
|
+
else free(path);
|
|
297
|
+
return f;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
int tf_spill_cleanup(tf_spill_session *s) {
|
|
301
|
+
if (!s) return TF_OK;
|
|
302
|
+
int rc = TF_OK;
|
|
303
|
+
if (s->dirfd >= 0) {
|
|
304
|
+
for (size_t i = 0; i < s->n_names; i++) {
|
|
305
|
+
if (s->names[i] && unlinkat(s->dirfd, s->names[i], 0) != 0 && errno != ENOENT) rc = TF_ERROR;
|
|
306
|
+
}
|
|
307
|
+
close(s->dirfd);
|
|
308
|
+
}
|
|
309
|
+
for (size_t i = 0; i < s->n_names; i++) free(s->names[i]);
|
|
310
|
+
free(s->names);
|
|
311
|
+
if (s->path && rmdir(s->path) != 0 && errno != ENOENT) rc = TF_ERROR;
|
|
312
|
+
free(s->path);
|
|
313
|
+
free(s);
|
|
314
|
+
return rc;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
#endif
|
package/csrc/spill.h
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* spill.h -- internal secure spill-file helper.
|
|
3
|
+
*
|
|
4
|
+
* Spill users create one private session directory under a host-approved root,
|
|
5
|
+
* then create individual run files inside it with exclusive openat().
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
#ifndef TF_SPILL_H
|
|
9
|
+
#define TF_SPILL_H
|
|
10
|
+
|
|
11
|
+
#include "tranfi.h"
|
|
12
|
+
#include <stdio.h>
|
|
13
|
+
|
|
14
|
+
typedef struct tf_spill_session tf_spill_session;
|
|
15
|
+
|
|
16
|
+
int tf_spill_session_create(const char *root, tf_spill_session **out);
|
|
17
|
+
int tf_spill_open_run(tf_spill_session *s, const char *label, int *fd, char **path_out);
|
|
18
|
+
FILE *tf_spill_open_run_file(tf_spill_session *s, const char *label, char **path_out);
|
|
19
|
+
int tf_spill_cleanup(tf_spill_session *s);
|
|
20
|
+
|
|
21
|
+
#endif
|