tranfi 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/LICENSE +177 -0
  2. package/NOTICE +8 -0
  3. package/README.md +272 -40
  4. package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
  5. package/app/index.html +1 -1
  6. package/binding.gyp +55 -3
  7. package/csrc/arena.c +7 -5
  8. package/csrc/batch.c +818 -71
  9. package/csrc/buffer.c +84 -8
  10. package/csrc/cJSON.c +262 -19
  11. package/csrc/cJSON.h +17 -1
  12. package/csrc/codec_csv.c +1074 -181
  13. package/csrc/codec_jsonl.c +830 -118
  14. package/csrc/codec_table.c +108 -78
  15. package/csrc/codec_text.c +286 -68
  16. package/csrc/compiler.c +31 -3
  17. package/csrc/config.h +21 -0
  18. package/csrc/dsl.c +4722 -485
  19. package/csrc/expr.c +363 -55
  20. package/csrc/expr.h +2 -0
  21. package/csrc/internal.h +316 -27
  22. package/csrc/ir.c +65 -18
  23. package/csrc/ir.h +41 -0
  24. package/csrc/ir_schema.c +20 -5
  25. package/csrc/ir_serialize.c +68 -6
  26. package/csrc/ir_sql.c +796 -185
  27. package/csrc/ir_validate.c +462 -6
  28. package/csrc/json_path.c +210 -0
  29. package/csrc/main.c +879 -30
  30. package/csrc/memory_estimate.c +477 -0
  31. package/csrc/op_acf.c +171 -21
  32. package/csrc/op_across.c +477 -0
  33. package/csrc/op_anomaly.c +167 -32
  34. package/csrc/op_assert.c +761 -0
  35. package/csrc/op_bin.c +168 -29
  36. package/csrc/op_cast.c +383 -55
  37. package/csrc/op_clip.c +30 -19
  38. package/csrc/op_date_trunc.c +208 -34
  39. package/csrc/op_datetime.c +259 -77
  40. package/csrc/op_derive.c +65 -97
  41. package/csrc/op_diff.c +146 -30
  42. package/csrc/op_ewma.c +149 -30
  43. package/csrc/op_explode.c +124 -26
  44. package/csrc/op_fill_down.c +125 -53
  45. package/csrc/op_fill_null.c +176 -31
  46. package/csrc/op_filter.c +89 -40
  47. package/csrc/op_frequency.c +571 -43
  48. package/csrc/op_grep.c +36 -18
  49. package/csrc/op_group_agg.c +1790 -119
  50. package/csrc/op_hash.c +48 -15
  51. package/csrc/op_head.c +21 -86
  52. package/csrc/op_interpolate.c +268 -62
  53. package/csrc/op_join.c +2700 -182
  54. package/csrc/op_json_extract.c +227 -0
  55. package/csrc/op_json_filter.c +384 -0
  56. package/csrc/op_json_flatten.c +293 -0
  57. package/csrc/op_json_schema.c +503 -0
  58. package/csrc/op_label_encode.c +328 -53
  59. package/csrc/op_lag.c +181 -0
  60. package/csrc/op_lead.c +141 -89
  61. package/csrc/op_normalize.c +363 -79
  62. package/csrc/op_onehot.c +345 -73
  63. package/csrc/op_pivot.c +1546 -162
  64. package/csrc/op_quarantine.c +189 -0
  65. package/csrc/op_registry.c +2062 -166
  66. package/csrc/op_rename.c +41 -50
  67. package/csrc/op_replace.c +270 -118
  68. package/csrc/op_rleid.c +297 -0
  69. package/csrc/op_rowid.c +559 -0
  70. package/csrc/op_sample.c +80 -23
  71. package/csrc/op_schema.c +1341 -0
  72. package/csrc/op_schema_infer.c +252 -0
  73. package/csrc/op_select.c +265 -65
  74. package/csrc/op_set.c +3449 -0
  75. package/csrc/op_skip.c +30 -87
  76. package/csrc/op_sort.c +670 -124
  77. package/csrc/op_source_name.c +120 -0
  78. package/csrc/op_split.c +65 -28
  79. package/csrc/op_split_data.c +41 -9
  80. package/csrc/op_stack.c +178 -222
  81. package/csrc/op_stats.c +206 -110
  82. package/csrc/op_step.c +217 -55
  83. package/csrc/op_tail.c +21 -12
  84. package/csrc/op_tee.c +338 -0
  85. package/csrc/op_top.c +260 -53
  86. package/csrc/op_trim.c +48 -19
  87. package/csrc/op_unique.c +1193 -150
  88. package/csrc/op_unpivot.c +100 -66
  89. package/csrc/op_validate.c +601 -24
  90. package/csrc/op_window.c +492 -51
  91. package/csrc/path_policy.c +85 -0
  92. package/csrc/pipeline.c +872 -99
  93. package/csrc/recipes.c +3 -1
  94. package/csrc/report.c +73 -30
  95. package/csrc/selector.c +1097 -0
  96. package/csrc/size_utils.c +348 -0
  97. package/csrc/spill.c +317 -0
  98. package/csrc/spill.h +21 -0
  99. package/csrc/tranfi.h +169 -1
  100. package/csrc/transform.h +209 -0
  101. package/csrc/transform_api.c +2237 -0
  102. package/csrc/transform_categorical.c +923 -0
  103. package/csrc/transform_internal.h +472 -0
  104. package/csrc/transform_json.c +3812 -0
  105. package/csrc/transform_numeric.c +1966 -0
  106. package/csrc/transform_sha256.c +154 -0
  107. package/csrc/transform_wasm.h +162 -0
  108. package/csrc/transform_wasm_api.c +1373 -0
  109. package/csrc/wasm_api.c +70 -9
  110. package/napi_api.c +219 -11
  111. package/napi_transform.c +1648 -0
  112. package/napi_transform.h +8 -0
  113. package/package.json +27 -11
  114. package/scripts/install-native.js +76 -0
  115. package/scripts/prepack.js +64 -0
  116. package/scripts/sync-csrc.js +23 -0
  117. package/src/cli.js +8 -11
  118. package/src/engines/duckdb.js +45 -12
  119. package/src/index.js +661 -42
  120. package/src/memory_policy.js +411 -0
  121. package/src/native.js +1 -5
  122. package/src/pipeline.js +454 -31
  123. package/src/recipe_json.js +80 -0
  124. package/src/server.js +10 -8
  125. package/src/transform.js +403 -0
  126. package/src/transform_error.js +10 -0
  127. package/src/wasm.js +6 -4
  128. package/wasm/index.js +498 -10
  129. package/wasm/tranfi_core.js +0 -0
  130. package/wasm/transform.js +1156 -0
  131. package/wasm/worker.js +786 -0
  132. package/csrc/plan.c +0 -206
@@ -16,10 +16,8 @@
16
16
 
17
17
  #include "internal.h"
18
18
  #include "cJSON.h"
19
- #include "date_utils.h"
20
19
  #include <stdlib.h>
21
20
  #include <string.h>
22
- #include <stdio.h>
23
21
 
24
22
  #define TABLE_MAX_COLS 256
25
23
  #define TABLE_DEFAULT_WIDTH 40
@@ -36,33 +34,57 @@ typedef struct {
36
34
 
37
35
  /* Format a cell value as a string. Caller must free. */
38
36
  static char *cell_to_string(tf_batch *b, size_t row, size_t col) {
39
- if (tf_batch_is_null(b, row, col)) return strdup("");
40
-
41
- char buf[128];
42
- switch (b->col_types[col]) {
43
- case TF_TYPE_BOOL:
44
- return strdup(tf_batch_get_bool(b, row, col) ? "true" : "false");
45
- case TF_TYPE_INT64:
46
- snprintf(buf, sizeof(buf), "%lld", (long long)tf_batch_get_int64(b, row, col));
47
- return strdup(buf);
48
- case TF_TYPE_FLOAT64:
49
- snprintf(buf, sizeof(buf), "%g", tf_batch_get_float64(b, row, col));
50
- return strdup(buf);
51
- case TF_TYPE_STRING:
52
- return strdup(tf_batch_get_string(b, row, col));
53
- case TF_TYPE_DATE: {
54
- char dbuf[16];
55
- tf_date_format(tf_batch_get_date(b, row, col), dbuf, sizeof(dbuf));
56
- return strdup(dbuf);
57
- }
58
- case TF_TYPE_TIMESTAMP: {
59
- char tsbuf[40];
60
- tf_timestamp_format(tf_batch_get_timestamp(b, row, col), tsbuf, sizeof(tsbuf));
61
- return strdup(tsbuf);
37
+ char buf[64];
38
+ const char *text = NULL;
39
+ if (tf_batch_format_cell_as_string(b, row, col, TF_CELL_STRING_HUMAN,
40
+ buf, sizeof(buf), &text) != TF_OK) {
41
+ return NULL;
42
+ }
43
+ return strdup(text ? text : "");
44
+ }
45
+
46
+ static int table_capture_schema(table_encoder_state *st, const tf_batch *in) {
47
+ size_t n_cols = in->n_cols < TABLE_MAX_COLS ? in->n_cols : TABLE_MAX_COLS;
48
+ if (n_cols == 0) return TF_OK;
49
+ char **names = tf_callocarray_checked(n_cols, sizeof(char *));
50
+ if (!names) return TF_ERROR;
51
+ for (size_t i = 0; i < n_cols; i++) {
52
+ names[i] = strdup(in->col_names[i] ? in->col_names[i] : "");
53
+ if (!names[i]) {
54
+ for (size_t j = 0; j < i; j++) free(names[j]);
55
+ free(names);
56
+ return TF_ERROR;
62
57
  }
63
- default:
64
- return strdup("");
65
58
  }
59
+ st->col_names = names;
60
+ st->n_cols = n_cols;
61
+ return TF_OK;
62
+ }
63
+
64
+ static int table_ensure_value_capacity(table_encoder_state *st, size_t min_rows) {
65
+ if (st->n_cols == 0 || min_rows <= st->capacity) return TF_OK;
66
+ size_t new_cap = 0;
67
+ size_t cells = 0;
68
+ if (tf_size_grow_pow2(st->capacity, min_rows, 64, &new_cap) != TF_OK ||
69
+ tf_size_mul(new_cap, st->n_cols, &cells) != TF_OK) {
70
+ return TF_ERROR;
71
+ }
72
+ char **new_values = tf_reallocarray_checked(st->values, cells, sizeof(char *));
73
+ if (!new_values) return TF_ERROR;
74
+ st->values = new_values;
75
+ st->capacity = new_cap;
76
+ return TF_OK;
77
+ }
78
+
79
+ static int table_write_repeat(tf_buffer *out, char ch, size_t count) {
80
+ char pad[256];
81
+ memset(pad, ch, sizeof(pad));
82
+ while (count > 0) {
83
+ size_t chunk = count < sizeof(pad) ? count : sizeof(pad);
84
+ if (tf_buffer_write(out, (const uint8_t *)pad, chunk) != TF_OK) return TF_ERROR;
85
+ count -= chunk;
86
+ }
87
+ return TF_OK;
66
88
  }
67
89
 
68
90
  static int table_encode(tf_encoder *self, tf_batch *in, tf_buffer *out) {
@@ -71,29 +93,32 @@ static int table_encode(tf_encoder *self, tf_batch *in, tf_buffer *out) {
71
93
 
72
94
  /* Capture column names on first batch */
73
95
  if (!st->col_names && in->n_cols > 0) {
74
- st->n_cols = in->n_cols < TABLE_MAX_COLS ? in->n_cols : TABLE_MAX_COLS;
75
- st->col_names = malloc(st->n_cols * sizeof(char *));
76
- for (size_t i = 0; i < st->n_cols; i++) {
77
- st->col_names[i] = strdup(in->col_names[i]);
78
- }
96
+ if (table_capture_schema(st, in) != TF_OK) return TF_ERROR;
79
97
  }
80
98
 
81
99
  /* Buffer all cell values as strings */
82
100
  for (size_t r = 0; r < in->n_rows; r++) {
83
101
  if (st->max_rows > 0 && st->n_rows >= st->max_rows) break;
84
-
85
- if (st->n_rows >= st->capacity) {
86
- size_t new_cap = st->capacity == 0 ? 64 : st->capacity * 2;
87
- st->values = realloc(st->values, new_cap * st->n_cols * sizeof(char *));
88
- if (!st->values) return TF_ERROR;
89
- st->capacity = new_cap;
102
+ size_t next_rows = 0;
103
+ if (tf_size_add(st->n_rows, 1, &next_rows) != TF_OK ||
104
+ table_ensure_value_capacity(st, next_rows) != TF_OK) {
105
+ return TF_ERROR;
90
106
  }
91
107
 
92
- size_t base = st->n_rows * st->n_cols;
93
- for (size_t c = 0; c < st->n_cols; c++) {
108
+ size_t base = 0;
109
+ if (tf_size_mul(st->n_rows, st->n_cols, &base) != TF_OK) return TF_ERROR;
110
+ size_t c = 0;
111
+ for (; c < st->n_cols; c++) {
94
112
  st->values[base + c] = cell_to_string(in, r, c);
113
+ if (!st->values[base + c]) {
114
+ for (size_t j = 0; j < c; j++) {
115
+ free(st->values[base + j]);
116
+ st->values[base + j] = NULL;
117
+ }
118
+ return TF_ERROR;
119
+ }
95
120
  }
96
- st->n_rows++;
121
+ st->n_rows = next_rows;
97
122
  }
98
123
 
99
124
  return TF_OK;
@@ -104,7 +129,8 @@ static int table_flush(tf_encoder *self, tf_buffer *out) {
104
129
  if (!st->col_names || st->n_cols == 0) return TF_OK;
105
130
 
106
131
  /* Compute column widths */
107
- size_t *widths = calloc(st->n_cols, sizeof(size_t));
132
+ size_t *widths = tf_callocarray_checked(st->n_cols, sizeof(size_t));
133
+ if (!widths) return TF_ERROR;
108
134
  for (size_t c = 0; c < st->n_cols; c++) {
109
135
  widths[c] = strlen(st->col_names[c]);
110
136
  }
@@ -120,59 +146,55 @@ static int table_flush(tf_encoder *self, tf_buffer *out) {
120
146
  widths[c] = st->max_width;
121
147
  }
122
148
 
123
- char pad[256];
149
+ #define TABLE_WRITE(data, len) \
150
+ do { \
151
+ if (tf_buffer_write(out, (const uint8_t *)(data), (len)) != TF_OK) goto fail; \
152
+ } while (0)
124
153
 
125
154
  /* Header row */
126
- tf_buffer_write(out, (const uint8_t *)"| ", 2);
155
+ TABLE_WRITE("| ", 2);
127
156
  for (size_t c = 0; c < st->n_cols; c++) {
128
- if (c > 0) tf_buffer_write(out, (const uint8_t *)" | ", 3);
157
+ if (c > 0) TABLE_WRITE(" | ", 3);
129
158
  const char *name = st->col_names[c];
130
159
  size_t nlen = strlen(name);
131
160
  size_t w = widths[c];
132
161
  if (nlen > w) nlen = w;
133
- tf_buffer_write(out, (const uint8_t *)name, nlen);
134
- if (nlen < w) {
135
- size_t plen = w - nlen;
136
- if (plen > sizeof(pad)) plen = sizeof(pad);
137
- memset(pad, ' ', plen);
138
- tf_buffer_write(out, (const uint8_t *)pad, plen);
139
- }
162
+ TABLE_WRITE(name, nlen);
163
+ if (nlen < w && table_write_repeat(out, ' ', w - nlen) != TF_OK) goto fail;
140
164
  }
141
- tf_buffer_write(out, (const uint8_t *)" |\n", 3);
165
+ TABLE_WRITE(" |\n", 3);
142
166
 
143
167
  /* Separator row */
144
- tf_buffer_write(out, (const uint8_t *)"| ", 2);
168
+ TABLE_WRITE("| ", 2);
145
169
  for (size_t c = 0; c < st->n_cols; c++) {
146
- if (c > 0) tf_buffer_write(out, (const uint8_t *)" | ", 3);
147
- size_t w = widths[c];
148
- if (w > sizeof(pad)) w = sizeof(pad);
149
- memset(pad, '-', w);
150
- tf_buffer_write(out, (const uint8_t *)pad, w);
170
+ if (c > 0) TABLE_WRITE(" | ", 3);
171
+ if (table_write_repeat(out, '-', widths[c]) != TF_OK) goto fail;
151
172
  }
152
- tf_buffer_write(out, (const uint8_t *)" |\n", 3);
173
+ TABLE_WRITE(" |\n", 3);
153
174
 
154
175
  /* Data rows */
155
176
  for (size_t r = 0; r < st->n_rows; r++) {
156
- tf_buffer_write(out, (const uint8_t *)"| ", 2);
177
+ TABLE_WRITE("| ", 2);
157
178
  for (size_t c = 0; c < st->n_cols; c++) {
158
- if (c > 0) tf_buffer_write(out, (const uint8_t *)" | ", 3);
179
+ if (c > 0) TABLE_WRITE(" | ", 3);
159
180
  const char *val = st->values[r * st->n_cols + c];
160
181
  size_t vlen = strlen(val);
161
182
  size_t w = widths[c];
162
183
  if (vlen > w) vlen = w;
163
- tf_buffer_write(out, (const uint8_t *)val, vlen);
164
- if (vlen < w) {
165
- size_t plen = w - vlen;
166
- if (plen > sizeof(pad)) plen = sizeof(pad);
167
- memset(pad, ' ', plen);
168
- tf_buffer_write(out, (const uint8_t *)pad, plen);
169
- }
184
+ TABLE_WRITE(val, vlen);
185
+ if (vlen < w && table_write_repeat(out, ' ', w - vlen) != TF_OK) goto fail;
170
186
  }
171
- tf_buffer_write(out, (const uint8_t *)" |\n", 3);
187
+ TABLE_WRITE(" |\n", 3);
172
188
  }
173
189
 
190
+ #undef TABLE_WRITE
174
191
  free(widths);
175
192
  return TF_OK;
193
+
194
+ fail:
195
+ #undef TABLE_WRITE
196
+ free(widths);
197
+ return TF_ERROR;
176
198
  }
177
199
 
178
200
  static void table_encoder_destroy(tf_encoder *self) {
@@ -183,7 +205,10 @@ static void table_encoder_destroy(tf_encoder *self) {
183
205
  free(st->col_names);
184
206
  }
185
207
  if (st->values) {
186
- for (size_t i = 0; i < st->n_rows * st->n_cols; i++) free(st->values[i]);
208
+ size_t n_cells = 0;
209
+ if (tf_size_mul(st->n_rows, st->n_cols, &n_cells) == TF_OK) {
210
+ for (size_t i = 0; i < n_cells; i++) free(st->values[i]);
211
+ }
187
212
  free(st->values);
188
213
  }
189
214
  free(st);
@@ -199,13 +224,18 @@ tf_encoder *tf_table_encoder_create(const cJSON *args) {
199
224
  st->max_rows = 0;
200
225
 
201
226
  if (args) {
202
- cJSON *mw = cJSON_GetObjectItemCaseSensitive(args, "max_width");
203
- if (cJSON_IsNumber(mw) && mw->valueint > 0)
204
- st->max_width = (size_t)mw->valueint;
205
-
206
- cJSON *mr = cJSON_GetObjectItemCaseSensitive(args, "max_rows");
207
- if (cJSON_IsNumber(mr) && mr->valueint > 0)
208
- st->max_rows = (size_t)mr->valueint;
227
+ size_t parsed_size = 0;
228
+ int has_max_width = tf_json_get_size_arg(args, "max_width",
229
+ 1, TF_MAX_TABLE_WIDTH,
230
+ &parsed_size, "table");
231
+ if (has_max_width < 0) { free(st); return NULL; }
232
+ if (has_max_width > 0) st->max_width = parsed_size;
233
+
234
+ int has_max_rows = tf_json_get_size_arg(args, "max_rows",
235
+ 0, TF_MAX_TABLE_ROWS,
236
+ &parsed_size, "table");
237
+ if (has_max_rows < 0) { free(st); return NULL; }
238
+ if (has_max_rows > 0) st->max_rows = parsed_size;
209
239
  }
210
240
 
211
241
  tf_encoder *enc = malloc(sizeof(tf_encoder));