tranfi 0.1.2 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/LICENSE +177 -0
  2. package/NOTICE +8 -0
  3. package/README.md +443 -51
  4. package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
  5. package/app/index.html +1 -1
  6. package/binding.gyp +55 -3
  7. package/csrc/arena.c +7 -5
  8. package/csrc/batch.c +818 -71
  9. package/csrc/buffer.c +84 -8
  10. package/csrc/cJSON.c +262 -19
  11. package/csrc/cJSON.h +17 -1
  12. package/csrc/codec_csv.c +1074 -181
  13. package/csrc/codec_jsonl.c +830 -118
  14. package/csrc/codec_table.c +108 -78
  15. package/csrc/codec_text.c +286 -68
  16. package/csrc/compiler.c +31 -3
  17. package/csrc/config.h +21 -0
  18. package/csrc/dsl.c +4722 -485
  19. package/csrc/expr.c +363 -55
  20. package/csrc/expr.h +2 -0
  21. package/csrc/internal.h +316 -27
  22. package/csrc/ir.c +65 -18
  23. package/csrc/ir.h +41 -0
  24. package/csrc/ir_schema.c +20 -5
  25. package/csrc/ir_serialize.c +68 -6
  26. package/csrc/ir_sql.c +796 -185
  27. package/csrc/ir_validate.c +462 -6
  28. package/csrc/json_path.c +210 -0
  29. package/csrc/main.c +879 -30
  30. package/csrc/memory_estimate.c +477 -0
  31. package/csrc/op_acf.c +171 -21
  32. package/csrc/op_across.c +477 -0
  33. package/csrc/op_anomaly.c +167 -32
  34. package/csrc/op_assert.c +761 -0
  35. package/csrc/op_bin.c +168 -29
  36. package/csrc/op_cast.c +383 -55
  37. package/csrc/op_clip.c +30 -19
  38. package/csrc/op_date_trunc.c +208 -34
  39. package/csrc/op_datetime.c +259 -77
  40. package/csrc/op_derive.c +65 -97
  41. package/csrc/op_diff.c +146 -30
  42. package/csrc/op_ewma.c +149 -30
  43. package/csrc/op_explode.c +124 -26
  44. package/csrc/op_fill_down.c +125 -53
  45. package/csrc/op_fill_null.c +176 -31
  46. package/csrc/op_filter.c +89 -40
  47. package/csrc/op_frequency.c +571 -43
  48. package/csrc/op_grep.c +36 -18
  49. package/csrc/op_group_agg.c +1790 -119
  50. package/csrc/op_hash.c +48 -15
  51. package/csrc/op_head.c +21 -86
  52. package/csrc/op_interpolate.c +268 -62
  53. package/csrc/op_join.c +2700 -182
  54. package/csrc/op_json_extract.c +227 -0
  55. package/csrc/op_json_filter.c +384 -0
  56. package/csrc/op_json_flatten.c +293 -0
  57. package/csrc/op_json_schema.c +503 -0
  58. package/csrc/op_label_encode.c +328 -53
  59. package/csrc/op_lag.c +181 -0
  60. package/csrc/op_lead.c +141 -89
  61. package/csrc/op_normalize.c +363 -79
  62. package/csrc/op_onehot.c +345 -73
  63. package/csrc/op_pivot.c +1546 -162
  64. package/csrc/op_quarantine.c +189 -0
  65. package/csrc/op_registry.c +2062 -166
  66. package/csrc/op_rename.c +41 -50
  67. package/csrc/op_replace.c +270 -118
  68. package/csrc/op_rleid.c +297 -0
  69. package/csrc/op_rowid.c +559 -0
  70. package/csrc/op_sample.c +80 -23
  71. package/csrc/op_schema.c +1341 -0
  72. package/csrc/op_schema_infer.c +252 -0
  73. package/csrc/op_select.c +265 -65
  74. package/csrc/op_set.c +3449 -0
  75. package/csrc/op_skip.c +30 -87
  76. package/csrc/op_sort.c +670 -124
  77. package/csrc/op_source_name.c +120 -0
  78. package/csrc/op_split.c +65 -28
  79. package/csrc/op_split_data.c +41 -9
  80. package/csrc/op_stack.c +178 -222
  81. package/csrc/op_stats.c +206 -110
  82. package/csrc/op_step.c +217 -55
  83. package/csrc/op_tail.c +21 -12
  84. package/csrc/op_tee.c +338 -0
  85. package/csrc/op_top.c +260 -53
  86. package/csrc/op_trim.c +48 -19
  87. package/csrc/op_unique.c +1193 -150
  88. package/csrc/op_unpivot.c +100 -66
  89. package/csrc/op_validate.c +601 -24
  90. package/csrc/op_window.c +492 -51
  91. package/csrc/path_policy.c +85 -0
  92. package/csrc/pipeline.c +872 -99
  93. package/csrc/recipes.c +3 -1
  94. package/csrc/report.c +73 -30
  95. package/csrc/selector.c +1097 -0
  96. package/csrc/size_utils.c +352 -0
  97. package/csrc/spill.c +317 -0
  98. package/csrc/spill.h +21 -0
  99. package/csrc/tranfi.h +169 -1
  100. package/csrc/transform.h +209 -0
  101. package/csrc/transform_api.c +2237 -0
  102. package/csrc/transform_categorical.c +923 -0
  103. package/csrc/transform_internal.h +472 -0
  104. package/csrc/transform_json.c +3812 -0
  105. package/csrc/transform_numeric.c +1966 -0
  106. package/csrc/transform_sha256.c +154 -0
  107. package/csrc/transform_wasm.h +162 -0
  108. package/csrc/transform_wasm_api.c +1373 -0
  109. package/csrc/wasm_api.c +70 -9
  110. package/napi_api.c +219 -11
  111. package/napi_transform.c +1648 -0
  112. package/napi_transform.h +8 -0
  113. package/package.json +27 -11
  114. package/scripts/install-native.js +76 -0
  115. package/scripts/prepack.js +64 -0
  116. package/scripts/sync-csrc.js +23 -0
  117. package/src/cli.js +81 -41
  118. package/src/engines/duckdb.js +45 -12
  119. package/src/index.js +661 -42
  120. package/src/memory_policy.js +411 -0
  121. package/src/native.js +1 -5
  122. package/src/pipeline.js +454 -31
  123. package/src/recipe_json.js +80 -0
  124. package/src/server.js +10 -8
  125. package/src/transform.js +403 -0
  126. package/src/transform_error.js +10 -0
  127. package/src/wasm.js +6 -4
  128. package/wasm/index.js +498 -10
  129. package/wasm/tranfi_core.js +0 -0
  130. package/wasm/transform.js +1156 -0
  131. package/wasm/worker.js +786 -0
  132. package/csrc/plan.c +0 -206
package/app/index.html CHANGED
@@ -4,7 +4,7 @@
4
4
  <meta charset="UTF-8">
5
5
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
6
  <title>Tranfi</title>
7
- <script type="module" crossorigin src="/assets/index-pDFMluyz.js"></script>
7
+ <script type="module" crossorigin src="/assets/index-BIAIKnrp.js"></script>
8
8
  <link rel="stylesheet" crossorigin href="/assets/index-6quYZ5Ap.css">
9
9
  </head>
10
10
  <body>
package/binding.gyp CHANGED
@@ -4,13 +4,24 @@
4
4
  "target_name": "tranfi_napi",
5
5
  "sources": [
6
6
  "napi_api.c",
7
+ "napi_transform.c",
7
8
  "csrc/cJSON.c",
9
+ "csrc/transform_api.c",
10
+ "csrc/transform_categorical.c",
11
+ "csrc/transform_json.c",
12
+ "csrc/transform_numeric.c",
13
+ "csrc/transform_sha256.c",
8
14
  "csrc/arena.c",
9
15
  "csrc/buffer.c",
16
+ "csrc/size_utils.c",
17
+ "csrc/path_policy.c",
18
+ "csrc/spill.c",
10
19
  "csrc/batch.c",
11
20
  "csrc/pipeline.c",
12
- "csrc/plan.c",
13
21
  "csrc/expr.c",
22
+ "csrc/selector.c",
23
+ "csrc/memory_estimate.c",
24
+ "csrc/json_path.c",
14
25
  "csrc/codec_csv.c",
15
26
  "csrc/codec_jsonl.c",
16
27
  "csrc/codec_text.c",
@@ -21,10 +32,17 @@
21
32
  "csrc/op_head.c",
22
33
  "csrc/op_skip.c",
23
34
  "csrc/op_derive.c",
35
+ "csrc/op_source_name.c",
36
+ "csrc/op_across.c",
24
37
  "csrc/op_stats.c",
25
38
  "csrc/op_unique.c",
26
39
  "csrc/op_sort.c",
27
40
  "csrc/op_validate.c",
41
+ "csrc/op_quarantine.c",
42
+ "csrc/op_assert.c",
43
+ "csrc/op_schema.c",
44
+ "csrc/op_schema_infer.c",
45
+ "csrc/op_tee.c",
28
46
  "csrc/op_trim.c",
29
47
  "csrc/op_fill_null.c",
30
48
  "csrc/op_cast.c",
@@ -45,11 +63,28 @@
45
63
  "csrc/op_frequency.c",
46
64
  "csrc/op_datetime.c",
47
65
  "csrc/op_grep.c",
66
+ "csrc/op_json_extract.c",
67
+ "csrc/op_json_filter.c",
68
+ "csrc/op_json_schema.c",
69
+ "csrc/op_json_flatten.c",
48
70
  "csrc/op_pivot.c",
49
71
  "csrc/op_join.c",
72
+ "csrc/op_set.c",
50
73
  "csrc/op_stack.c",
51
74
  "csrc/op_lead.c",
75
+ "csrc/op_lag.c",
76
+ "csrc/op_rowid.c",
77
+ "csrc/op_rleid.c",
52
78
  "csrc/op_date_trunc.c",
79
+ "csrc/op_onehot.c",
80
+ "csrc/op_label_encode.c",
81
+ "csrc/op_ewma.c",
82
+ "csrc/op_diff.c",
83
+ "csrc/op_anomaly.c",
84
+ "csrc/op_split_data.c",
85
+ "csrc/op_interpolate.c",
86
+ "csrc/op_normalize.c",
87
+ "csrc/op_acf.c",
53
88
  "csrc/ir.c",
54
89
  "csrc/ir_serialize.c",
55
90
  "csrc/ir_sql.c",
@@ -58,12 +93,29 @@
58
93
  "csrc/op_registry.c",
59
94
  "csrc/compiler.c",
60
95
  "csrc/dsl.c",
61
- "csrc/recipes.c"
96
+ "csrc/recipes.c",
97
+ "csrc/report.c"
62
98
  ],
63
99
  "include_dirs": [
64
100
  "csrc"
65
101
  ],
66
- "cflags": ["-std=c11", "-Wall", "-D_POSIX_C_SOURCE=200809L"]
102
+ "cflags": [
103
+ "-std=c11",
104
+ "-Wall",
105
+ "-DNAPI_VERSION=8",
106
+ "-D_POSIX_C_SOURCE=200809L",
107
+ "-D_XOPEN_SOURCE=700",
108
+ "-D_FORTIFY_SOURCE=3",
109
+ "-Werror=implicit-function-declaration",
110
+ "-Werror=incompatible-pointer-types",
111
+ "-Wformat",
112
+ "-Werror=format-security",
113
+ "-Werror=unused-result",
114
+ "-fno-common",
115
+ "-fstack-protector-strong",
116
+ "-fno-fast-math",
117
+ "-ffp-contract=off"
118
+ ]
67
119
  }
68
120
  ]
69
121
  }
package/csrc/arena.c CHANGED
@@ -8,7 +8,6 @@
8
8
  #include <string.h>
9
9
 
10
10
  #define DEFAULT_BLOCK_SIZE (64 * 1024) /* 64 KB */
11
- #define ALIGN_UP(x, a) (((x) + (a) - 1) & ~((a) - 1))
12
11
 
13
12
  static tf_arena_block *block_create(size_t cap) {
14
13
  tf_arena_block *blk = malloc(sizeof(tf_arena_block));
@@ -32,6 +31,7 @@ static void block_free(tf_arena_block *blk) {
32
31
 
33
32
  tf_arena *tf_arena_create(size_t block_size) {
34
33
  if (block_size == 0) block_size = DEFAULT_BLOCK_SIZE;
34
+ if (tf_size_align(block_size, 8, &block_size) != TF_OK) return NULL;
35
35
  tf_arena *a = malloc(sizeof(tf_arena));
36
36
  if (!a) return NULL;
37
37
  a->block_size = block_size;
@@ -43,12 +43,13 @@ tf_arena *tf_arena_create(size_t block_size) {
43
43
 
44
44
  void *tf_arena_alloc(tf_arena *a, size_t size) {
45
45
  if (!a || size == 0) return NULL;
46
- size = ALIGN_UP(size, 8);
46
+ if (tf_size_align(size, 8, &size) != TF_OK) return NULL;
47
47
 
48
48
  /* Try current block */
49
- if (a->current->used + size <= a->current->cap) {
49
+ size_t used_after = 0;
50
+ if (tf_size_add(a->current->used, size, &used_after) == TF_OK && used_after <= a->current->cap) {
50
51
  void *ptr = a->current->data + a->current->used;
51
- a->current->used += size;
52
+ a->current->used = used_after;
52
53
  return ptr;
53
54
  }
54
55
 
@@ -67,7 +68,8 @@ void *tf_arena_alloc(tf_arena *a, size_t size) {
67
68
 
68
69
  char *tf_arena_strdup(tf_arena *a, const char *s) {
69
70
  if (!s) return NULL;
70
- size_t len = strlen(s) + 1;
71
+ size_t len = 0;
72
+ if (tf_size_add(strlen(s), 1, &len) != TF_OK) return NULL;
71
73
  char *copy = tf_arena_alloc(a, len);
72
74
  if (copy) memcpy(copy, s, len);
73
75
  return copy;