tranfi 0.1.2 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/LICENSE +177 -0
  2. package/NOTICE +8 -0
  3. package/README.md +443 -51
  4. package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
  5. package/app/index.html +1 -1
  6. package/binding.gyp +55 -3
  7. package/csrc/arena.c +7 -5
  8. package/csrc/batch.c +818 -71
  9. package/csrc/buffer.c +84 -8
  10. package/csrc/cJSON.c +262 -19
  11. package/csrc/cJSON.h +17 -1
  12. package/csrc/codec_csv.c +1074 -181
  13. package/csrc/codec_jsonl.c +830 -118
  14. package/csrc/codec_table.c +108 -78
  15. package/csrc/codec_text.c +286 -68
  16. package/csrc/compiler.c +31 -3
  17. package/csrc/config.h +21 -0
  18. package/csrc/dsl.c +4722 -485
  19. package/csrc/expr.c +363 -55
  20. package/csrc/expr.h +2 -0
  21. package/csrc/internal.h +316 -27
  22. package/csrc/ir.c +65 -18
  23. package/csrc/ir.h +41 -0
  24. package/csrc/ir_schema.c +20 -5
  25. package/csrc/ir_serialize.c +68 -6
  26. package/csrc/ir_sql.c +796 -185
  27. package/csrc/ir_validate.c +462 -6
  28. package/csrc/json_path.c +210 -0
  29. package/csrc/main.c +879 -30
  30. package/csrc/memory_estimate.c +477 -0
  31. package/csrc/op_acf.c +171 -21
  32. package/csrc/op_across.c +477 -0
  33. package/csrc/op_anomaly.c +167 -32
  34. package/csrc/op_assert.c +761 -0
  35. package/csrc/op_bin.c +168 -29
  36. package/csrc/op_cast.c +383 -55
  37. package/csrc/op_clip.c +30 -19
  38. package/csrc/op_date_trunc.c +208 -34
  39. package/csrc/op_datetime.c +259 -77
  40. package/csrc/op_derive.c +65 -97
  41. package/csrc/op_diff.c +146 -30
  42. package/csrc/op_ewma.c +149 -30
  43. package/csrc/op_explode.c +124 -26
  44. package/csrc/op_fill_down.c +125 -53
  45. package/csrc/op_fill_null.c +176 -31
  46. package/csrc/op_filter.c +89 -40
  47. package/csrc/op_frequency.c +571 -43
  48. package/csrc/op_grep.c +36 -18
  49. package/csrc/op_group_agg.c +1790 -119
  50. package/csrc/op_hash.c +48 -15
  51. package/csrc/op_head.c +21 -86
  52. package/csrc/op_interpolate.c +268 -62
  53. package/csrc/op_join.c +2700 -182
  54. package/csrc/op_json_extract.c +227 -0
  55. package/csrc/op_json_filter.c +384 -0
  56. package/csrc/op_json_flatten.c +293 -0
  57. package/csrc/op_json_schema.c +503 -0
  58. package/csrc/op_label_encode.c +328 -53
  59. package/csrc/op_lag.c +181 -0
  60. package/csrc/op_lead.c +141 -89
  61. package/csrc/op_normalize.c +363 -79
  62. package/csrc/op_onehot.c +345 -73
  63. package/csrc/op_pivot.c +1546 -162
  64. package/csrc/op_quarantine.c +189 -0
  65. package/csrc/op_registry.c +2062 -166
  66. package/csrc/op_rename.c +41 -50
  67. package/csrc/op_replace.c +270 -118
  68. package/csrc/op_rleid.c +297 -0
  69. package/csrc/op_rowid.c +559 -0
  70. package/csrc/op_sample.c +80 -23
  71. package/csrc/op_schema.c +1341 -0
  72. package/csrc/op_schema_infer.c +252 -0
  73. package/csrc/op_select.c +265 -65
  74. package/csrc/op_set.c +3449 -0
  75. package/csrc/op_skip.c +30 -87
  76. package/csrc/op_sort.c +670 -124
  77. package/csrc/op_source_name.c +120 -0
  78. package/csrc/op_split.c +65 -28
  79. package/csrc/op_split_data.c +41 -9
  80. package/csrc/op_stack.c +178 -222
  81. package/csrc/op_stats.c +206 -110
  82. package/csrc/op_step.c +217 -55
  83. package/csrc/op_tail.c +21 -12
  84. package/csrc/op_tee.c +338 -0
  85. package/csrc/op_top.c +260 -53
  86. package/csrc/op_trim.c +48 -19
  87. package/csrc/op_unique.c +1193 -150
  88. package/csrc/op_unpivot.c +100 -66
  89. package/csrc/op_validate.c +601 -24
  90. package/csrc/op_window.c +492 -51
  91. package/csrc/path_policy.c +85 -0
  92. package/csrc/pipeline.c +872 -99
  93. package/csrc/recipes.c +3 -1
  94. package/csrc/report.c +73 -30
  95. package/csrc/selector.c +1097 -0
  96. package/csrc/size_utils.c +352 -0
  97. package/csrc/spill.c +317 -0
  98. package/csrc/spill.h +21 -0
  99. package/csrc/tranfi.h +169 -1
  100. package/csrc/transform.h +209 -0
  101. package/csrc/transform_api.c +2237 -0
  102. package/csrc/transform_categorical.c +923 -0
  103. package/csrc/transform_internal.h +472 -0
  104. package/csrc/transform_json.c +3812 -0
  105. package/csrc/transform_numeric.c +1966 -0
  106. package/csrc/transform_sha256.c +154 -0
  107. package/csrc/transform_wasm.h +162 -0
  108. package/csrc/transform_wasm_api.c +1373 -0
  109. package/csrc/wasm_api.c +70 -9
  110. package/napi_api.c +219 -11
  111. package/napi_transform.c +1648 -0
  112. package/napi_transform.h +8 -0
  113. package/package.json +27 -11
  114. package/scripts/install-native.js +76 -0
  115. package/scripts/prepack.js +64 -0
  116. package/scripts/sync-csrc.js +23 -0
  117. package/src/cli.js +81 -41
  118. package/src/engines/duckdb.js +45 -12
  119. package/src/index.js +661 -42
  120. package/src/memory_policy.js +411 -0
  121. package/src/native.js +1 -5
  122. package/src/pipeline.js +454 -31
  123. package/src/recipe_json.js +80 -0
  124. package/src/server.js +10 -8
  125. package/src/transform.js +403 -0
  126. package/src/transform_error.js +10 -0
  127. package/src/wasm.js +6 -4
  128. package/wasm/index.js +498 -10
  129. package/wasm/tranfi_core.js +0 -0
  130. package/wasm/transform.js +1156 -0
  131. package/wasm/worker.js +786 -0
  132. package/csrc/plan.c +0 -206
package/csrc/plan.c DELETED
@@ -1,206 +0,0 @@
1
- /*
2
- * plan.c — Parse a JSON pipeline plan and instantiate decoder, steps, encoder.
3
- *
4
- * Plan format:
5
- * {
6
- * "steps": [
7
- * {"op": "codec.csv.decode", "args": {"delimiter": ","}},
8
- * {"op": "filter", "args": {"expr": "col('age') > 25"}},
9
- * {"op": "codec.csv.encode", "args": {}}
10
- * ]
11
- * }
12
- */
13
-
14
- #include "internal.h"
15
- #include "cJSON.h"
16
- #include <stdlib.h>
17
- #include <string.h>
18
- #include <stdio.h>
19
-
20
- static void set_error(char **error, const char *msg) {
21
- if (error) {
22
- free(*error);
23
- size_t len = strlen(msg) + 1;
24
- *error = malloc(len);
25
- if (*error) memcpy(*error, msg, len);
26
- }
27
- }
28
-
29
- static void set_errorf(char **error, const char *fmt, const char *detail) {
30
- if (error) {
31
- char buf[256];
32
- snprintf(buf, sizeof(buf), fmt, detail);
33
- set_error(error, buf);
34
- }
35
- }
36
-
37
- int tf_plan_parse(const char *json, size_t len,
38
- tf_decoder **out_decoder, tf_step ***out_steps, size_t *out_n_steps,
39
- tf_encoder **out_encoder, char **error) {
40
- *out_decoder = NULL;
41
- *out_steps = NULL;
42
- *out_n_steps = 0;
43
- *out_encoder = NULL;
44
-
45
- /* Parse JSON */
46
- cJSON *root = cJSON_ParseWithLength(json, len);
47
- if (!root) {
48
- set_error(error, "invalid JSON in plan");
49
- return TF_ERROR;
50
- }
51
-
52
- cJSON *steps_arr = cJSON_GetObjectItemCaseSensitive(root, "steps");
53
- if (!cJSON_IsArray(steps_arr)) {
54
- set_error(error, "plan must have a 'steps' array");
55
- cJSON_Delete(root);
56
- return TF_ERROR;
57
- }
58
-
59
- int n_json_steps = cJSON_GetArraySize(steps_arr);
60
- if (n_json_steps == 0) {
61
- set_error(error, "plan has no steps");
62
- cJSON_Delete(root);
63
- return TF_ERROR;
64
- }
65
-
66
- /* Temporary arrays for transforms (max = n_json_steps) */
67
- tf_step **steps = calloc(n_json_steps, sizeof(tf_step *));
68
- size_t n_steps = 0;
69
- tf_decoder *decoder = NULL;
70
- tf_encoder *encoder = NULL;
71
- int err = TF_OK;
72
-
73
- for (int i = 0; i < n_json_steps; i++) {
74
- cJSON *step_json = cJSON_GetArrayItem(steps_arr, i);
75
- cJSON *op_json = cJSON_GetObjectItemCaseSensitive(step_json, "op");
76
- cJSON *args_json = cJSON_GetObjectItemCaseSensitive(step_json, "args");
77
-
78
- if (!cJSON_IsString(op_json)) {
79
- set_errorf(error, "step %d missing 'op' string", "");
80
- err = TF_ERROR;
81
- break;
82
- }
83
- const char *op = op_json->valuestring;
84
-
85
- /* Match op to constructor */
86
- if (strcmp(op, "codec.csv.decode") == 0) {
87
- if (decoder) {
88
- set_error(error, "multiple decoders not supported");
89
- err = TF_ERROR;
90
- break;
91
- }
92
- decoder = tf_csv_decoder_create(args_json);
93
- if (!decoder) {
94
- set_error(error, "failed to create CSV decoder");
95
- err = TF_ERROR;
96
- break;
97
- }
98
- } else if (strcmp(op, "codec.csv.encode") == 0) {
99
- if (encoder) {
100
- set_error(error, "multiple encoders not supported");
101
- err = TF_ERROR;
102
- break;
103
- }
104
- encoder = tf_csv_encoder_create(args_json);
105
- if (!encoder) {
106
- set_error(error, "failed to create CSV encoder");
107
- err = TF_ERROR;
108
- break;
109
- }
110
- } else if (strcmp(op, "codec.jsonl.decode") == 0) {
111
- if (decoder) {
112
- set_error(error, "multiple decoders not supported");
113
- err = TF_ERROR;
114
- break;
115
- }
116
- decoder = tf_jsonl_decoder_create(args_json);
117
- if (!decoder) {
118
- set_error(error, "failed to create JSONL decoder");
119
- err = TF_ERROR;
120
- break;
121
- }
122
- } else if (strcmp(op, "codec.jsonl.encode") == 0) {
123
- if (encoder) {
124
- set_error(error, "multiple encoders not supported");
125
- err = TF_ERROR;
126
- break;
127
- }
128
- encoder = tf_jsonl_encoder_create(args_json);
129
- if (!encoder) {
130
- set_error(error, "failed to create JSONL encoder");
131
- err = TF_ERROR;
132
- break;
133
- }
134
- } else if (strcmp(op, "filter") == 0) {
135
- tf_step *s = tf_filter_create(args_json);
136
- if (!s) {
137
- set_error(error, "failed to create filter step");
138
- err = TF_ERROR;
139
- break;
140
- }
141
- steps[n_steps++] = s;
142
- } else if (strcmp(op, "select") == 0) {
143
- tf_step *s = tf_select_create(args_json);
144
- if (!s) {
145
- set_error(error, "failed to create select step");
146
- err = TF_ERROR;
147
- break;
148
- }
149
- steps[n_steps++] = s;
150
- } else if (strcmp(op, "rename") == 0) {
151
- tf_step *s = tf_rename_create(args_json);
152
- if (!s) {
153
- set_error(error, "failed to create rename step");
154
- err = TF_ERROR;
155
- break;
156
- }
157
- steps[n_steps++] = s;
158
- } else if (strcmp(op, "head") == 0) {
159
- tf_step *s = tf_head_create(args_json);
160
- if (!s) {
161
- set_error(error, "failed to create head step");
162
- err = TF_ERROR;
163
- break;
164
- }
165
- steps[n_steps++] = s;
166
- } else {
167
- set_errorf(error, "unknown op: '%s'", op);
168
- err = TF_ERROR;
169
- break;
170
- }
171
- }
172
-
173
- cJSON_Delete(root);
174
-
175
- if (err != TF_OK) {
176
- /* Cleanup on error */
177
- if (decoder) decoder->destroy(decoder);
178
- if (encoder) encoder->destroy(encoder);
179
- for (size_t i = 0; i < n_steps; i++) {
180
- if (steps[i]) steps[i]->destroy(steps[i]);
181
- }
182
- free(steps);
183
- return TF_ERROR;
184
- }
185
-
186
- if (!decoder) {
187
- set_error(error, "plan has no decoder (need a codec.*.decode step)");
188
- if (encoder) encoder->destroy(encoder);
189
- for (size_t i = 0; i < n_steps; i++) steps[i]->destroy(steps[i]);
190
- free(steps);
191
- return TF_ERROR;
192
- }
193
- if (!encoder) {
194
- set_error(error, "plan has no encoder (need a codec.*.encode step)");
195
- decoder->destroy(decoder);
196
- for (size_t i = 0; i < n_steps; i++) steps[i]->destroy(steps[i]);
197
- free(steps);
198
- return TF_ERROR;
199
- }
200
-
201
- *out_decoder = decoder;
202
- *out_steps = steps;
203
- *out_n_steps = n_steps;
204
- *out_encoder = encoder;
205
- return TF_OK;
206
- }