tranfi 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/LICENSE +177 -0
  2. package/NOTICE +8 -0
  3. package/README.md +272 -40
  4. package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
  5. package/app/index.html +1 -1
  6. package/binding.gyp +55 -3
  7. package/csrc/arena.c +7 -5
  8. package/csrc/batch.c +818 -71
  9. package/csrc/buffer.c +84 -8
  10. package/csrc/cJSON.c +262 -19
  11. package/csrc/cJSON.h +17 -1
  12. package/csrc/codec_csv.c +1074 -181
  13. package/csrc/codec_jsonl.c +830 -118
  14. package/csrc/codec_table.c +108 -78
  15. package/csrc/codec_text.c +286 -68
  16. package/csrc/compiler.c +31 -3
  17. package/csrc/config.h +21 -0
  18. package/csrc/dsl.c +4722 -485
  19. package/csrc/expr.c +363 -55
  20. package/csrc/expr.h +2 -0
  21. package/csrc/internal.h +316 -27
  22. package/csrc/ir.c +65 -18
  23. package/csrc/ir.h +41 -0
  24. package/csrc/ir_schema.c +20 -5
  25. package/csrc/ir_serialize.c +68 -6
  26. package/csrc/ir_sql.c +796 -185
  27. package/csrc/ir_validate.c +462 -6
  28. package/csrc/json_path.c +210 -0
  29. package/csrc/main.c +879 -30
  30. package/csrc/memory_estimate.c +477 -0
  31. package/csrc/op_acf.c +171 -21
  32. package/csrc/op_across.c +477 -0
  33. package/csrc/op_anomaly.c +167 -32
  34. package/csrc/op_assert.c +761 -0
  35. package/csrc/op_bin.c +168 -29
  36. package/csrc/op_cast.c +383 -55
  37. package/csrc/op_clip.c +30 -19
  38. package/csrc/op_date_trunc.c +208 -34
  39. package/csrc/op_datetime.c +259 -77
  40. package/csrc/op_derive.c +65 -97
  41. package/csrc/op_diff.c +146 -30
  42. package/csrc/op_ewma.c +149 -30
  43. package/csrc/op_explode.c +124 -26
  44. package/csrc/op_fill_down.c +125 -53
  45. package/csrc/op_fill_null.c +176 -31
  46. package/csrc/op_filter.c +89 -40
  47. package/csrc/op_frequency.c +571 -43
  48. package/csrc/op_grep.c +36 -18
  49. package/csrc/op_group_agg.c +1790 -119
  50. package/csrc/op_hash.c +48 -15
  51. package/csrc/op_head.c +21 -86
  52. package/csrc/op_interpolate.c +268 -62
  53. package/csrc/op_join.c +2700 -182
  54. package/csrc/op_json_extract.c +227 -0
  55. package/csrc/op_json_filter.c +384 -0
  56. package/csrc/op_json_flatten.c +293 -0
  57. package/csrc/op_json_schema.c +503 -0
  58. package/csrc/op_label_encode.c +328 -53
  59. package/csrc/op_lag.c +181 -0
  60. package/csrc/op_lead.c +141 -89
  61. package/csrc/op_normalize.c +363 -79
  62. package/csrc/op_onehot.c +345 -73
  63. package/csrc/op_pivot.c +1546 -162
  64. package/csrc/op_quarantine.c +189 -0
  65. package/csrc/op_registry.c +2062 -166
  66. package/csrc/op_rename.c +41 -50
  67. package/csrc/op_replace.c +270 -118
  68. package/csrc/op_rleid.c +297 -0
  69. package/csrc/op_rowid.c +559 -0
  70. package/csrc/op_sample.c +80 -23
  71. package/csrc/op_schema.c +1341 -0
  72. package/csrc/op_schema_infer.c +252 -0
  73. package/csrc/op_select.c +265 -65
  74. package/csrc/op_set.c +3449 -0
  75. package/csrc/op_skip.c +30 -87
  76. package/csrc/op_sort.c +670 -124
  77. package/csrc/op_source_name.c +120 -0
  78. package/csrc/op_split.c +65 -28
  79. package/csrc/op_split_data.c +41 -9
  80. package/csrc/op_stack.c +178 -222
  81. package/csrc/op_stats.c +206 -110
  82. package/csrc/op_step.c +217 -55
  83. package/csrc/op_tail.c +21 -12
  84. package/csrc/op_tee.c +338 -0
  85. package/csrc/op_top.c +260 -53
  86. package/csrc/op_trim.c +48 -19
  87. package/csrc/op_unique.c +1193 -150
  88. package/csrc/op_unpivot.c +100 -66
  89. package/csrc/op_validate.c +601 -24
  90. package/csrc/op_window.c +492 -51
  91. package/csrc/path_policy.c +85 -0
  92. package/csrc/pipeline.c +872 -99
  93. package/csrc/recipes.c +3 -1
  94. package/csrc/report.c +73 -30
  95. package/csrc/selector.c +1097 -0
  96. package/csrc/size_utils.c +348 -0
  97. package/csrc/spill.c +317 -0
  98. package/csrc/spill.h +21 -0
  99. package/csrc/tranfi.h +169 -1
  100. package/csrc/transform.h +209 -0
  101. package/csrc/transform_api.c +2237 -0
  102. package/csrc/transform_categorical.c +923 -0
  103. package/csrc/transform_internal.h +472 -0
  104. package/csrc/transform_json.c +3812 -0
  105. package/csrc/transform_numeric.c +1966 -0
  106. package/csrc/transform_sha256.c +154 -0
  107. package/csrc/transform_wasm.h +162 -0
  108. package/csrc/transform_wasm_api.c +1373 -0
  109. package/csrc/wasm_api.c +70 -9
  110. package/napi_api.c +219 -11
  111. package/napi_transform.c +1648 -0
  112. package/napi_transform.h +8 -0
  113. package/package.json +27 -11
  114. package/scripts/install-native.js +76 -0
  115. package/scripts/prepack.js +64 -0
  116. package/scripts/sync-csrc.js +23 -0
  117. package/src/cli.js +8 -11
  118. package/src/engines/duckdb.js +45 -12
  119. package/src/index.js +661 -42
  120. package/src/memory_policy.js +411 -0
  121. package/src/native.js +1 -5
  122. package/src/pipeline.js +454 -31
  123. package/src/recipe_json.js +80 -0
  124. package/src/server.js +10 -8
  125. package/src/transform.js +403 -0
  126. package/src/transform_error.js +10 -0
  127. package/src/wasm.js +6 -4
  128. package/wasm/index.js +498 -10
  129. package/wasm/tranfi_core.js +0 -0
  130. package/wasm/transform.js +1156 -0
  131. package/wasm/worker.js +786 -0
  132. package/csrc/plan.c +0 -206
package/csrc/op_grep.c CHANGED
@@ -34,15 +34,22 @@ static int grep_process(tf_step *self, tf_batch *in, tf_batch **out,
34
34
  if (col_idx < 0) {
35
35
  /* Column not found — pass through or drop all depending on invert */
36
36
  if (st->invert) {
37
- *out = in;
38
- /* Don't free — caller owns it, but we need to copy */
39
37
  tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
40
38
  if (!ob) return TF_ERROR;
41
- for (size_t c = 0; c < in->n_cols; c++)
42
- tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
43
- for (size_t r = 0; r < in->n_rows; r++)
44
- tf_batch_copy_row(ob, r, in, r);
45
- ob->n_rows = in->n_rows;
39
+ if (tf_batch_clone_schema(ob, in) != TF_OK) {
40
+ tf_batch_free(ob);
41
+ return TF_ERROR;
42
+ }
43
+ for (size_t r = 0; r < in->n_rows; r++) {
44
+ if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
45
+ tf_batch_free(ob);
46
+ return TF_ERROR;
47
+ }
48
+ if (tf_batch_expose_row(ob, r) != TF_OK) {
49
+ tf_batch_free(ob);
50
+ return TF_ERROR;
51
+ }
52
+ }
46
53
  *out = ob;
47
54
  }
48
55
  return TF_OK;
@@ -51,8 +58,10 @@ static int grep_process(tf_step *self, tf_batch *in, tf_batch **out,
51
58
  /* Create output batch with same schema */
52
59
  tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
53
60
  if (!ob) return TF_ERROR;
54
- for (size_t c = 0; c < in->n_cols; c++)
55
- tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
61
+ if (tf_batch_clone_schema(ob, in) != TF_OK) {
62
+ tf_batch_free(ob);
63
+ return TF_ERROR;
64
+ }
56
65
 
57
66
  size_t out_row = 0;
58
67
  for (size_t r = 0; r < in->n_rows; r++) {
@@ -77,10 +86,16 @@ static int grep_process(tf_step *self, tf_batch *in, tf_batch **out,
77
86
  return TF_ERROR;
78
87
  }
79
88
 
80
- tf_batch_copy_row(ob, out_row, in, r);
89
+ if (tf_batch_copy_row(ob, out_row, in, r) != TF_OK) {
90
+ tf_batch_free(ob);
91
+ return TF_ERROR;
92
+ }
93
+ if (tf_batch_expose_row(ob, out_row) != TF_OK) {
94
+ tf_batch_free(ob);
95
+ return TF_ERROR;
96
+ }
81
97
  out_row++;
82
98
  }
83
- ob->n_rows = out_row;
84
99
 
85
100
  st->rows_in += in->n_rows;
86
101
  st->rows_out += out_row;
@@ -88,9 +103,12 @@ static int grep_process(tf_step *self, tf_batch *in, tf_batch **out,
88
103
  if (side && side->stats) {
89
104
  char stats_buf[128];
90
105
  snprintf(stats_buf, sizeof(stats_buf),
91
- "{\"op\":\"grep\",\"rows_in\":%zu,\"rows_out\":%zu}\n",
106
+ "{\"op\":\"grep\",\"rows_in\":%zu,\"rows_out\":%zu}",
92
107
  in->n_rows, out_row);
93
- tf_buffer_write_str(side->stats, stats_buf);
108
+ if (tf_buffer_write_line(side->stats, stats_buf) != TF_OK) {
109
+ tf_batch_free(ob);
110
+ return TF_ERROR;
111
+ }
94
112
  }
95
113
 
96
114
  if (out_row > 0) {
@@ -124,17 +142,17 @@ tf_step *tf_grep_create(const cJSON *args) {
124
142
  cJSON *pattern_json = cJSON_GetObjectItemCaseSensitive(args, "pattern");
125
143
  if (!cJSON_IsString(pattern_json)) return NULL;
126
144
 
127
- grep_state *st = calloc(1, sizeof(grep_state));
145
+ grep_state *st = tf_callocarray_checked(1, sizeof(grep_state));
128
146
  if (!st) return NULL;
129
147
 
130
- st->pattern = strdup(pattern_json->valuestring);
148
+ st->pattern = tf_strdup_checked(pattern_json->valuestring);
131
149
  if (!st->pattern) { free(st); return NULL; }
132
150
 
133
151
  cJSON *column_json = cJSON_GetObjectItemCaseSensitive(args, "column");
134
152
  if (cJSON_IsString(column_json))
135
- st->column = strdup(column_json->valuestring);
153
+ st->column = tf_strdup_checked(column_json->valuestring);
136
154
  else
137
- st->column = strdup("_line");
155
+ st->column = tf_strdup_checked("_line");
138
156
  if (!st->column) { free(st->pattern); free(st); return NULL; }
139
157
 
140
158
  cJSON *invert_json = cJSON_GetObjectItemCaseSensitive(args, "invert");
@@ -153,7 +171,7 @@ tf_step *tf_grep_create(const cJSON *args) {
153
171
  }
154
172
  }
155
173
 
156
- tf_step *step = malloc(sizeof(tf_step));
174
+ tf_step *step = tf_callocarray_checked(1, sizeof(tf_step));
157
175
  if (!step) { free(st->pattern); free(st->column); free(st); return NULL; }
158
176
  step->process = grep_process;
159
177
  step->flush = grep_flush;