tranfi 0.0.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -21
- package/NOTICE +8 -0
- package/README.md +627 -0
- package/app/assets/index-6quYZ5Ap.css +5 -0
- package/app/assets/index-BIAIKnrp.js +160 -0
- package/app/assets/materialdesignicons-webfont-B7mPwVP_.ttf +0 -0
- package/app/assets/materialdesignicons-webfont-CSr8KVlo.eot +0 -0
- package/app/assets/materialdesignicons-webfont-Dp5v-WZN.woff2 +0 -0
- package/app/assets/materialdesignicons-webfont-PXm3-2wK.woff +0 -0
- package/app/index.html +13 -0
- package/binding.gyp +121 -0
- package/csrc/arena.c +93 -0
- package/csrc/batch.c +976 -0
- package/csrc/buffer.c +154 -0
- package/csrc/cJSON.c +3386 -0
- package/csrc/cJSON.h +316 -0
- package/csrc/codec_csv.c +1951 -0
- package/csrc/codec_jsonl.c +1086 -0
- package/csrc/codec_table.c +248 -0
- package/csrc/codec_text.c +447 -0
- package/csrc/compiler.c +130 -0
- package/csrc/config.h +21 -0
- package/csrc/date_utils.h +94 -0
- package/csrc/dsl.c +5417 -0
- package/csrc/dsl.h +22 -0
- package/csrc/expr.c +1553 -0
- package/csrc/expr.h +58 -0
- package/csrc/internal.h +539 -0
- package/csrc/ir.c +166 -0
- package/csrc/ir.h +208 -0
- package/csrc/ir_schema.c +75 -0
- package/csrc/ir_serialize.c +166 -0
- package/csrc/ir_sql.c +1822 -0
- package/csrc/ir_validate.c +576 -0
- package/csrc/json_path.c +210 -0
- package/csrc/main.c +1241 -0
- package/csrc/memory_estimate.c +477 -0
- package/csrc/op_acf.c +283 -0
- package/csrc/op_across.c +477 -0
- package/csrc/op_anomaly.c +255 -0
- package/csrc/op_assert.c +761 -0
- package/csrc/op_bin.c +248 -0
- package/csrc/op_cast.c +523 -0
- package/csrc/op_clip.c +99 -0
- package/csrc/op_date_trunc.c +355 -0
- package/csrc/op_datetime.c +394 -0
- package/csrc/op_derive.c +216 -0
- package/csrc/op_diff.c +250 -0
- package/csrc/op_ewma.c +222 -0
- package/csrc/op_explode.c +206 -0
- package/csrc/op_fill_down.c +235 -0
- package/csrc/op_fill_null.c +268 -0
- package/csrc/op_filter.c +181 -0
- package/csrc/op_frequency.c +721 -0
- package/csrc/op_grep.c +181 -0
- package/csrc/op_group_agg.c +1956 -0
- package/csrc/op_hash.c +159 -0
- package/csrc/op_head.c +84 -0
- package/csrc/op_interpolate.c +445 -0
- package/csrc/op_join.c +2902 -0
- package/csrc/op_json_extract.c +227 -0
- package/csrc/op_json_filter.c +384 -0
- package/csrc/op_json_flatten.c +293 -0
- package/csrc/op_json_schema.c +503 -0
- package/csrc/op_label_encode.c +419 -0
- package/csrc/op_lag.c +181 -0
- package/csrc/op_lead.c +242 -0
- package/csrc/op_normalize.c +510 -0
- package/csrc/op_onehot.c +457 -0
- package/csrc/op_pivot.c +1754 -0
- package/csrc/op_quarantine.c +189 -0
- package/csrc/op_registry.c +3044 -0
- package/csrc/op_rename.c +129 -0
- package/csrc/op_replace.c +354 -0
- package/csrc/op_rleid.c +297 -0
- package/csrc/op_rowid.c +559 -0
- package/csrc/op_sample.c +158 -0
- package/csrc/op_schema.c +1341 -0
- package/csrc/op_schema_infer.c +252 -0
- package/csrc/op_select.c +340 -0
- package/csrc/op_set.c +3449 -0
- package/csrc/op_skip.c +95 -0
- package/csrc/op_sort.c +819 -0
- package/csrc/op_source_name.c +120 -0
- package/csrc/op_split.c +151 -0
- package/csrc/op_split_data.c +119 -0
- package/csrc/op_stack.c +271 -0
- package/csrc/op_stats.c +875 -0
- package/csrc/op_step.c +333 -0
- package/csrc/op_tail.c +105 -0
- package/csrc/op_tee.c +338 -0
- package/csrc/op_top.c +357 -0
- package/csrc/op_trim.c +138 -0
- package/csrc/op_unique.c +1343 -0
- package/csrc/op_unpivot.c +193 -0
- package/csrc/op_validate.c +648 -0
- package/csrc/op_window.c +591 -0
- package/csrc/path_policy.c +85 -0
- package/csrc/pipeline.c +1088 -0
- package/csrc/recipes.c +104 -0
- package/csrc/recipes.h +27 -0
- package/csrc/report.c +506 -0
- package/csrc/report.h +22 -0
- package/csrc/selector.c +1097 -0
- package/csrc/size_utils.c +348 -0
- package/csrc/spill.c +317 -0
- package/csrc/spill.h +21 -0
- package/csrc/tranfi.h +291 -0
- package/csrc/transform.h +209 -0
- package/csrc/transform_api.c +2237 -0
- package/csrc/transform_categorical.c +923 -0
- package/csrc/transform_internal.h +472 -0
- package/csrc/transform_json.c +3812 -0
- package/csrc/transform_numeric.c +1966 -0
- package/csrc/transform_sha256.c +154 -0
- package/csrc/transform_wasm.h +162 -0
- package/csrc/transform_wasm_api.c +1373 -0
- package/csrc/wasm_api.c +218 -0
- package/napi_api.c +534 -0
- package/napi_transform.c +1648 -0
- package/napi_transform.h +8 -0
- package/package.json +64 -59
- package/scripts/install-native.js +76 -0
- package/scripts/prepack.js +64 -0
- package/scripts/sync-csrc.js +23 -0
- package/src/cli.js +190 -0
- package/src/engines/duckdb.js +142 -0
- package/src/index.js +925 -0
- package/src/memory_policy.js +411 -0
- package/src/native.js +18 -0
- package/src/pipeline.js +709 -0
- package/src/recipe_json.js +80 -0
- package/src/server.js +279 -0
- package/src/transform.js +403 -0
- package/src/transform_error.js +10 -0
- package/src/wasm.js +21 -0
- package/wasm/index.js +732 -0
- package/wasm/package.json +1 -0
- package/wasm/tranfi_core.js +0 -0
- package/wasm/transform.js +1156 -0
- package/wasm/worker.js +786 -0
- package/dist/bundle.js +0 -1
- package/index.html +0 -18
- package/src/app.css +0 -169
- package/src/app.js +0 -203
- package/src/app.vue +0 -250
- package/src/bulma-input.vue +0 -110
- package/src/common-inputs.js +0 -28
- package/src/main.js +0 -20
- package/src/transforms.js +0 -166
- package/webpack.config.js +0 -108
package/csrc/compiler.c
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* compiler.c — L2 IR → L1 native target compilation.
|
|
3
|
+
*
|
|
4
|
+
* Iterates IR nodes, looks up each op in the registry,
|
|
5
|
+
* and calls create_native() to build live decoder/steps/encoder structs.
|
|
6
|
+
* This is the single native construction path for JSON IR plans.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
#include "ir.h"
|
|
10
|
+
#include "internal.h"
|
|
11
|
+
#include <stdlib.h>
|
|
12
|
+
#include <string.h>
|
|
13
|
+
#include <stdio.h>
|
|
14
|
+
|
|
15
|
+
static void set_error(char **error, const char *msg) {
|
|
16
|
+
if (error) {
|
|
17
|
+
free(*error);
|
|
18
|
+
size_t len = strlen(msg) + 1;
|
|
19
|
+
*error = malloc(len);
|
|
20
|
+
if (*error) memcpy(*error, msg, len);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
int tf_compile_native(const tf_ir_plan *plan,
|
|
25
|
+
tf_decoder **out_decoder, tf_step ***out_steps,
|
|
26
|
+
size_t *out_n_steps, tf_encoder **out_encoder,
|
|
27
|
+
char **error) {
|
|
28
|
+
*out_decoder = NULL;
|
|
29
|
+
*out_steps = NULL;
|
|
30
|
+
*out_n_steps = 0;
|
|
31
|
+
*out_encoder = NULL;
|
|
32
|
+
if (error) *error = NULL;
|
|
33
|
+
|
|
34
|
+
if (!plan || plan->n_nodes == 0) {
|
|
35
|
+
set_error(error, "empty plan");
|
|
36
|
+
return TF_ERROR;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/* Allocate steps array (max = n_nodes, since some are decoder/encoder) */
|
|
40
|
+
tf_step **steps = calloc(plan->n_nodes, sizeof(tf_step *));
|
|
41
|
+
if (!steps) {
|
|
42
|
+
set_error(error, "out of memory");
|
|
43
|
+
return TF_ERROR;
|
|
44
|
+
}
|
|
45
|
+
size_t n_steps = 0;
|
|
46
|
+
tf_decoder *decoder = NULL;
|
|
47
|
+
tf_encoder *encoder = NULL;
|
|
48
|
+
|
|
49
|
+
for (size_t i = 0; i < plan->n_nodes; i++) {
|
|
50
|
+
const tf_ir_node *node = &plan->nodes[i];
|
|
51
|
+
const tf_op_entry *entry = tf_op_registry_find(node->op);
|
|
52
|
+
|
|
53
|
+
if (!entry) {
|
|
54
|
+
char buf[256];
|
|
55
|
+
snprintf(buf, sizeof(buf), "unknown op: '%s'", node->op);
|
|
56
|
+
set_error(error, buf);
|
|
57
|
+
goto fail;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
if (!entry->create_native) {
|
|
61
|
+
if (entry->tier == TF_TIER_ECOSYSTEM) {
|
|
62
|
+
/* Ecosystem stubs are skipped — host must handle */
|
|
63
|
+
char buf[256];
|
|
64
|
+
snprintf(buf, sizeof(buf), "op '%s' has no native target", node->op);
|
|
65
|
+
set_error(error, buf);
|
|
66
|
+
goto fail;
|
|
67
|
+
}
|
|
68
|
+
/* Core ops with NULL constructor are no-ops (e.g. flatten) */
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
void *obj = entry->create_native(node->args);
|
|
73
|
+
if (!obj) {
|
|
74
|
+
const char *detail = tf_last_error();
|
|
75
|
+
char buf[512];
|
|
76
|
+
if (detail && detail[0])
|
|
77
|
+
snprintf(buf, sizeof(buf), "failed to create '%s': %s", node->op, detail);
|
|
78
|
+
else
|
|
79
|
+
snprintf(buf, sizeof(buf), "failed to create '%s'", node->op);
|
|
80
|
+
set_error(error, buf);
|
|
81
|
+
goto fail;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
switch (entry->kind) {
|
|
85
|
+
case TF_OP_DECODER:
|
|
86
|
+
if (decoder) {
|
|
87
|
+
((tf_decoder *)obj)->destroy((tf_decoder *)obj);
|
|
88
|
+
set_error(error, "multiple decoders");
|
|
89
|
+
goto fail;
|
|
90
|
+
}
|
|
91
|
+
decoder = (tf_decoder *)obj;
|
|
92
|
+
break;
|
|
93
|
+
case TF_OP_ENCODER:
|
|
94
|
+
if (encoder) {
|
|
95
|
+
((tf_encoder *)obj)->destroy((tf_encoder *)obj);
|
|
96
|
+
set_error(error, "multiple encoders");
|
|
97
|
+
goto fail;
|
|
98
|
+
}
|
|
99
|
+
encoder = (tf_encoder *)obj;
|
|
100
|
+
break;
|
|
101
|
+
case TF_OP_TRANSFORM:
|
|
102
|
+
steps[n_steps++] = (tf_step *)obj;
|
|
103
|
+
break;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if (!decoder) {
|
|
108
|
+
set_error(error, "plan missing decoder");
|
|
109
|
+
goto fail;
|
|
110
|
+
}
|
|
111
|
+
if (!encoder) {
|
|
112
|
+
set_error(error, "plan missing encoder");
|
|
113
|
+
goto fail;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
*out_decoder = decoder;
|
|
117
|
+
*out_steps = steps;
|
|
118
|
+
*out_n_steps = n_steps;
|
|
119
|
+
*out_encoder = encoder;
|
|
120
|
+
return TF_OK;
|
|
121
|
+
|
|
122
|
+
fail:
|
|
123
|
+
if (decoder) decoder->destroy(decoder);
|
|
124
|
+
if (encoder) encoder->destroy(encoder);
|
|
125
|
+
for (size_t i = 0; i < n_steps; i++) {
|
|
126
|
+
if (steps[i]) steps[i]->destroy(steps[i]);
|
|
127
|
+
}
|
|
128
|
+
free(steps);
|
|
129
|
+
return TF_ERROR;
|
|
130
|
+
}
|
package/csrc/config.h
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* config.h -- Tranfi C feature-test configuration.
|
|
3
|
+
*
|
|
4
|
+
* Build files also define these macros at compile-command level so they are
|
|
5
|
+
* visible before any system header in every translation unit. This header
|
|
6
|
+
* documents the source contract and protects internal/public headers included
|
|
7
|
+
* first.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
#ifndef TF_CONFIG_H
|
|
11
|
+
#define TF_CONFIG_H
|
|
12
|
+
|
|
13
|
+
#ifndef _POSIX_C_SOURCE
|
|
14
|
+
#define _POSIX_C_SOURCE 200809L
|
|
15
|
+
#endif
|
|
16
|
+
|
|
17
|
+
#ifndef _XOPEN_SOURCE
|
|
18
|
+
#define _XOPEN_SOURCE 700
|
|
19
|
+
#endif
|
|
20
|
+
|
|
21
|
+
#endif /* TF_CONFIG_H */
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* date_utils.h — Date/timestamp conversion utilities (header-only).
|
|
3
|
+
*
|
|
4
|
+
* Uses Howard Hinnant's civil date algorithms (public domain).
|
|
5
|
+
* Epoch: 1970-01-01. Date = int32_t days. Timestamp = int64_t microseconds.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
#ifndef TF_DATE_UTILS_H
|
|
9
|
+
#define TF_DATE_UTILS_H
|
|
10
|
+
|
|
11
|
+
#include <stdint.h>
|
|
12
|
+
#include <stdio.h>
|
|
13
|
+
|
|
14
|
+
/* Days from civil date to epoch (1970-01-01 = 0). */
|
|
15
|
+
static inline int32_t tf_date_from_ymd(int y, int m, int d) {
|
|
16
|
+
y -= (m <= 2);
|
|
17
|
+
int era = (y >= 0 ? y : y - 399) / 400;
|
|
18
|
+
unsigned yoe = (unsigned)(y - era * 400);
|
|
19
|
+
unsigned doy = (153 * (m > 2 ? m - 3 : m + 9) + 2) / 5 + (unsigned)d - 1;
|
|
20
|
+
unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
|
|
21
|
+
return era * 146097 + (int)doe - 719468;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/* Epoch days back to civil date. */
|
|
25
|
+
static inline void tf_date_to_ymd(int32_t days, int *y, int *m, int *d) {
|
|
26
|
+
days += 719468;
|
|
27
|
+
int era = (days >= 0 ? days : days - 146096) / 146097;
|
|
28
|
+
unsigned doe = (unsigned)(days - era * 146097);
|
|
29
|
+
unsigned yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365;
|
|
30
|
+
*y = (int)yoe + era * 400;
|
|
31
|
+
unsigned doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
|
32
|
+
unsigned mp = (5 * doy + 2) / 153;
|
|
33
|
+
*d = (int)(doy - (153 * mp + 2) / 5 + 1);
|
|
34
|
+
*m = (int)(mp < 10 ? mp + 3 : mp - 9);
|
|
35
|
+
*y += (*m <= 2);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/* Microseconds since epoch to date/time components. */
|
|
39
|
+
static inline void tf_timestamp_to_parts(int64_t us, int *y, int *mo, int *d,
|
|
40
|
+
int *h, int *mi, int *s, int *frac_us) {
|
|
41
|
+
int64_t day_us = 86400LL * 1000000LL;
|
|
42
|
+
int32_t days = (int32_t)(us / day_us);
|
|
43
|
+
int64_t rem = us % day_us;
|
|
44
|
+
if (rem < 0) { days--; rem += day_us; }
|
|
45
|
+
tf_date_to_ymd(days, y, mo, d);
|
|
46
|
+
*frac_us = (int)(rem % 1000000);
|
|
47
|
+
rem /= 1000000;
|
|
48
|
+
*s = (int)(rem % 60); rem /= 60;
|
|
49
|
+
*mi = (int)(rem % 60); rem /= 60;
|
|
50
|
+
*h = (int)rem;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/* Date/time components to microseconds since epoch. */
|
|
54
|
+
static inline int64_t tf_timestamp_from_parts(int y, int mo, int d,
|
|
55
|
+
int h, int mi, int s, int frac_us) {
|
|
56
|
+
int32_t days = tf_date_from_ymd(y, mo, d);
|
|
57
|
+
return (int64_t)days * 86400LL * 1000000LL
|
|
58
|
+
+ (int64_t)h * 3600000000LL
|
|
59
|
+
+ (int64_t)mi * 60000000LL
|
|
60
|
+
+ (int64_t)s * 1000000LL
|
|
61
|
+
+ frac_us;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/* Weekday: 0=Sunday..6=Saturday. */
|
|
65
|
+
static inline int tf_date_weekday(int32_t days) {
|
|
66
|
+
return days >= -4 ? (days + 4) % 7 : (days + 5) % 7 + 6;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/* Format date as YYYY-MM-DD into buf (must be >= 11 bytes). Returns buf. */
|
|
70
|
+
static inline char *tf_date_format(int32_t days, char *buf, size_t buflen) {
|
|
71
|
+
int y, m, d;
|
|
72
|
+
tf_date_to_ymd(days, &y, &m, &d);
|
|
73
|
+
snprintf(buf, buflen, "%04d-%02d-%02d", y, m, d);
|
|
74
|
+
return buf;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/* Format timestamp as ISO 8601 into buf (must be >= 32 bytes). Returns buf. */
|
|
78
|
+
static inline char *tf_timestamp_format(int64_t us, char *buf, size_t buflen) {
|
|
79
|
+
int y, mo, d, h, mi, s, frac;
|
|
80
|
+
tf_timestamp_to_parts(us, &y, &mo, &d, &h, &mi, &s, &frac);
|
|
81
|
+
if (frac > 0) {
|
|
82
|
+
int len = snprintf(buf, buflen, "%04d-%02d-%02dT%02d:%02d:%02d.%06d",
|
|
83
|
+
y, mo, d, h, mi, s, frac);
|
|
84
|
+
/* Trim trailing zeros */
|
|
85
|
+
while (len > 0 && buf[len - 1] == '0') len--;
|
|
86
|
+
buf[len++] = 'Z';
|
|
87
|
+
buf[len] = '\0';
|
|
88
|
+
} else {
|
|
89
|
+
snprintf(buf, buflen, "%04d-%02d-%02dT%02d:%02d:%02dZ", y, mo, d, h, mi, s);
|
|
90
|
+
}
|
|
91
|
+
return buf;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
#endif /* TF_DATE_UTILS_H */
|