tranfi 0.0.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/LICENSE +177 -21
  2. package/NOTICE +8 -0
  3. package/README.md +627 -0
  4. package/app/assets/index-6quYZ5Ap.css +5 -0
  5. package/app/assets/index-BIAIKnrp.js +160 -0
  6. package/app/assets/materialdesignicons-webfont-B7mPwVP_.ttf +0 -0
  7. package/app/assets/materialdesignicons-webfont-CSr8KVlo.eot +0 -0
  8. package/app/assets/materialdesignicons-webfont-Dp5v-WZN.woff2 +0 -0
  9. package/app/assets/materialdesignicons-webfont-PXm3-2wK.woff +0 -0
  10. package/app/index.html +13 -0
  11. package/binding.gyp +121 -0
  12. package/csrc/arena.c +93 -0
  13. package/csrc/batch.c +976 -0
  14. package/csrc/buffer.c +154 -0
  15. package/csrc/cJSON.c +3386 -0
  16. package/csrc/cJSON.h +316 -0
  17. package/csrc/codec_csv.c +1951 -0
  18. package/csrc/codec_jsonl.c +1086 -0
  19. package/csrc/codec_table.c +248 -0
  20. package/csrc/codec_text.c +447 -0
  21. package/csrc/compiler.c +130 -0
  22. package/csrc/config.h +21 -0
  23. package/csrc/date_utils.h +94 -0
  24. package/csrc/dsl.c +5417 -0
  25. package/csrc/dsl.h +22 -0
  26. package/csrc/expr.c +1553 -0
  27. package/csrc/expr.h +58 -0
  28. package/csrc/internal.h +539 -0
  29. package/csrc/ir.c +166 -0
  30. package/csrc/ir.h +208 -0
  31. package/csrc/ir_schema.c +75 -0
  32. package/csrc/ir_serialize.c +166 -0
  33. package/csrc/ir_sql.c +1822 -0
  34. package/csrc/ir_validate.c +576 -0
  35. package/csrc/json_path.c +210 -0
  36. package/csrc/main.c +1241 -0
  37. package/csrc/memory_estimate.c +477 -0
  38. package/csrc/op_acf.c +283 -0
  39. package/csrc/op_across.c +477 -0
  40. package/csrc/op_anomaly.c +255 -0
  41. package/csrc/op_assert.c +761 -0
  42. package/csrc/op_bin.c +248 -0
  43. package/csrc/op_cast.c +523 -0
  44. package/csrc/op_clip.c +99 -0
  45. package/csrc/op_date_trunc.c +355 -0
  46. package/csrc/op_datetime.c +394 -0
  47. package/csrc/op_derive.c +216 -0
  48. package/csrc/op_diff.c +250 -0
  49. package/csrc/op_ewma.c +222 -0
  50. package/csrc/op_explode.c +206 -0
  51. package/csrc/op_fill_down.c +235 -0
  52. package/csrc/op_fill_null.c +268 -0
  53. package/csrc/op_filter.c +181 -0
  54. package/csrc/op_frequency.c +721 -0
  55. package/csrc/op_grep.c +181 -0
  56. package/csrc/op_group_agg.c +1956 -0
  57. package/csrc/op_hash.c +159 -0
  58. package/csrc/op_head.c +84 -0
  59. package/csrc/op_interpolate.c +445 -0
  60. package/csrc/op_join.c +2902 -0
  61. package/csrc/op_json_extract.c +227 -0
  62. package/csrc/op_json_filter.c +384 -0
  63. package/csrc/op_json_flatten.c +293 -0
  64. package/csrc/op_json_schema.c +503 -0
  65. package/csrc/op_label_encode.c +419 -0
  66. package/csrc/op_lag.c +181 -0
  67. package/csrc/op_lead.c +242 -0
  68. package/csrc/op_normalize.c +510 -0
  69. package/csrc/op_onehot.c +457 -0
  70. package/csrc/op_pivot.c +1754 -0
  71. package/csrc/op_quarantine.c +189 -0
  72. package/csrc/op_registry.c +3044 -0
  73. package/csrc/op_rename.c +129 -0
  74. package/csrc/op_replace.c +354 -0
  75. package/csrc/op_rleid.c +297 -0
  76. package/csrc/op_rowid.c +559 -0
  77. package/csrc/op_sample.c +158 -0
  78. package/csrc/op_schema.c +1341 -0
  79. package/csrc/op_schema_infer.c +252 -0
  80. package/csrc/op_select.c +340 -0
  81. package/csrc/op_set.c +3449 -0
  82. package/csrc/op_skip.c +95 -0
  83. package/csrc/op_sort.c +819 -0
  84. package/csrc/op_source_name.c +120 -0
  85. package/csrc/op_split.c +151 -0
  86. package/csrc/op_split_data.c +119 -0
  87. package/csrc/op_stack.c +271 -0
  88. package/csrc/op_stats.c +875 -0
  89. package/csrc/op_step.c +333 -0
  90. package/csrc/op_tail.c +105 -0
  91. package/csrc/op_tee.c +338 -0
  92. package/csrc/op_top.c +357 -0
  93. package/csrc/op_trim.c +138 -0
  94. package/csrc/op_unique.c +1343 -0
  95. package/csrc/op_unpivot.c +193 -0
  96. package/csrc/op_validate.c +648 -0
  97. package/csrc/op_window.c +591 -0
  98. package/csrc/path_policy.c +85 -0
  99. package/csrc/pipeline.c +1088 -0
  100. package/csrc/recipes.c +104 -0
  101. package/csrc/recipes.h +27 -0
  102. package/csrc/report.c +506 -0
  103. package/csrc/report.h +22 -0
  104. package/csrc/selector.c +1097 -0
  105. package/csrc/size_utils.c +348 -0
  106. package/csrc/spill.c +317 -0
  107. package/csrc/spill.h +21 -0
  108. package/csrc/tranfi.h +291 -0
  109. package/csrc/transform.h +209 -0
  110. package/csrc/transform_api.c +2237 -0
  111. package/csrc/transform_categorical.c +923 -0
  112. package/csrc/transform_internal.h +472 -0
  113. package/csrc/transform_json.c +3812 -0
  114. package/csrc/transform_numeric.c +1966 -0
  115. package/csrc/transform_sha256.c +154 -0
  116. package/csrc/transform_wasm.h +162 -0
  117. package/csrc/transform_wasm_api.c +1373 -0
  118. package/csrc/wasm_api.c +218 -0
  119. package/napi_api.c +534 -0
  120. package/napi_transform.c +1648 -0
  121. package/napi_transform.h +8 -0
  122. package/package.json +64 -59
  123. package/scripts/install-native.js +76 -0
  124. package/scripts/prepack.js +64 -0
  125. package/scripts/sync-csrc.js +23 -0
  126. package/src/cli.js +190 -0
  127. package/src/engines/duckdb.js +142 -0
  128. package/src/index.js +925 -0
  129. package/src/memory_policy.js +411 -0
  130. package/src/native.js +18 -0
  131. package/src/pipeline.js +709 -0
  132. package/src/recipe_json.js +80 -0
  133. package/src/server.js +279 -0
  134. package/src/transform.js +403 -0
  135. package/src/transform_error.js +10 -0
  136. package/src/wasm.js +21 -0
  137. package/wasm/index.js +732 -0
  138. package/wasm/package.json +1 -0
  139. package/wasm/tranfi_core.js +0 -0
  140. package/wasm/transform.js +1156 -0
  141. package/wasm/worker.js +786 -0
  142. package/dist/bundle.js +0 -1
  143. package/index.html +0 -18
  144. package/src/app.css +0 -169
  145. package/src/app.js +0 -203
  146. package/src/app.vue +0 -250
  147. package/src/bulma-input.vue +0 -110
  148. package/src/common-inputs.js +0 -28
  149. package/src/main.js +0 -20
  150. package/src/transforms.js +0 -166
  151. package/webpack.config.js +0 -108
@@ -1,110 +0,0 @@
1
- <template>
2
- <div class="field" v-if="input.type == 'int' || input.type == 'float' || input.type == 'number'">
3
- <label v-bind:for="input.name" class="is-size-7">{{ input.name }}</label>
4
- <div class="control">
5
- <input
6
- v-model="input.value"
7
- v-bind:id="input.name"
8
- v-bind:step="input.type == 'int' ? 1 : 0.001"
9
- v-bind:placeholder="input.placeholder ? input.placeholder : input.name"
10
- v-bind:min="input.min"
11
- v-bind:max="input.max"
12
- v-on:change="changeHandler"
13
- class="input"
14
- type="number"
15
- >
16
- </div>
17
- </div>
18
-
19
- <div class="field" v-if="input.type == 'string' || input.type == 'color'">
20
- <label v-bind:for="input.name" class="is-size-7">{{ input.name }}</label>
21
- <div class="control">
22
- <input
23
- v-model="input.value"
24
- v-bind:id="input.name"
25
- v-bind:placeholder="input.placeholder ? input.placeholder : input.name"
26
- v-on:change="changeHandler"
27
- class="input"
28
- >
29
- </div>
30
- </div>
31
-
32
- <div class="field" v-if="input.type == 'text'">
33
- <label v-bind:for="input.name" class="is-size-7">{{ input.name }}</label>
34
- <div class="control">
35
- <textarea
36
- v-model="input.value"
37
- v-bind:id="input.name"
38
- v-bind:placeholder="input.placeholder ? input.placeholder : input.name"
39
- v-on:change="changeHandler"
40
- class="textarea"
41
- ></textarea>
42
- </div>
43
- </div>
44
-
45
- <div class="field" v-if="input.type == 'checkbox' || input.type == 'bool'">
46
- <div class="control">
47
- <label class="checkbox is-size-7">
48
- <input
49
- v-model="input.value"
50
- v-bind:id="input.name"
51
- v-on:change="changeHandler"
52
- type="checkbox"
53
- >
54
- {{ input.name }}
55
- </label>
56
- </div>
57
- </div>
58
-
59
- <div class="field" v-if="input.type == 'categorical' || input.type == 'select'">
60
- <label v-bind:for="input.name" class="is-size-7">{{ input.name }}</label>
61
- <div class="control">
62
- <div class="select is-fullwidth">
63
- <select
64
- v-model="input.value"
65
- v-bind:id="input.name"
66
- v-on:change="changeHandler"
67
- >
68
- <option
69
- v-for="(option, oi) in input.options"
70
- >{{ option }}</option>
71
- </select>
72
- </div>
73
- </div>
74
- </div>
75
-
76
- <div class="field" v-if="input.type == 'file'">
77
- <label v-bind:for="input.name" class="is-size-7">{{ input.name }}</label>
78
- <div class="control">
79
- <div class="file has-name is-fullwidth" v-bind:class="{ 'is-primary': !input.file }">
80
- <label class="file-label">
81
- <input
82
- v-bind:id="input.name"
83
- v-on:change="loadFile"
84
- class="file-input"
85
- type="file"
86
- >
87
- <span class="file-cta">
88
- <span class="file-label">
89
- Open
90
- </span>
91
- </span>
92
- <span class="file-name">
93
- <span v-if="input.file && input.file.name">{{ input.file.name }}</span>
94
- <span v-else>No file selected</span>
95
- </span>
96
- </label>
97
- </div>
98
- </div>
99
- </div>
100
-
101
- <div class="field is-horizontal" v-if="input.type == 'group'">
102
- <div class="field-body">
103
- <vue-input v-for="(el, index) in input.elements" v-bind:input="el" ></vue-input>
104
- </div>
105
- </div>
106
- </template>
107
-
108
- <script>
109
- export { component as default } from "./common-inputs.js"
110
- </script>
@@ -1,28 +0,0 @@
1
- const FileReader = window['FileReader']
2
-
3
- const component = {
4
- props: ['input'],
5
- emits: ['inchange'],
6
- methods: {
7
- changeHandler () {
8
- if (this.input.reactive) {
9
- this.$emit('inchange')
10
- }
11
- },
12
- loadFile (e) {
13
- const reader = new FileReader()
14
- this.input.file = e.target.files[0]
15
- // reader.readAsText(this.input.file)
16
- // reader.onload = () => {
17
- // this.input.value = reader.result
18
- // if (typeof this.input.cb !== 'undefined') {
19
- // this.input.cb.run()
20
- // } else if (this.input.reactive) {
21
- // this.$emit('inchange')
22
- // }
23
- // }
24
- }
25
- }
26
- }
27
-
28
- export { component }
package/src/main.js DELETED
@@ -1,20 +0,0 @@
1
- import { createApp } from 'vue'
2
-
3
- import App from './app.vue'
4
-
5
- import Equal from 'equal-vue'
6
- import 'equal-vue/dist/style.css'
7
-
8
- import bulmaInput from './bulma-input.vue'
9
- import draggable from 'vuedraggable'
10
-
11
- const app = createApp(App)
12
- app.use(Equal)
13
-
14
- app.component('vue-input', bulmaInput)
15
- app.component('draggable', draggable)
16
-
17
- const tranfi = app.mount('#tranfi-container')
18
-
19
- window['tranfi'] = vm
20
-
package/src/transforms.js DELETED
@@ -1,166 +0,0 @@
1
- const ReadStream = require('filestream').read
2
- const ParseStream = require('csv-parse')
3
- const JSONStream = require('JSONStream')
4
- const HTTPStream = require('stream-http')
5
- const through2 = require('through2')
6
-
7
- let FileLoader = class FileLoader {
8
- static inputType = 'text'
9
-
10
- static outputType = 'text'
11
-
12
- static type = 'loader'
13
-
14
- static initStream (params) {
15
- const stream = new ReadStream(params['File'], {chunkSize: params['Chunk size']})
16
- stream.setEncoding('utf8')
17
- // if (params['Encoding'] === 'UTF-8') {}
18
- return stream
19
- }
20
-
21
- constructor (id) {
22
- this.name = this.constructor.name
23
- this.inputs = [
24
- { name: 'File', type: 'file' },
25
- // { name: 'Encoding', type: 'categorical', options: ['UTF-8', 'Buffer'], default: 'UTF-8' },
26
- { name: 'Chunk size', type: 'int', default: 10000 },
27
- ]
28
- this.inputType = this.constructor.inputType
29
- this.outputType = this.constructor.outputType
30
- this.id = id
31
- this.expanded = false
32
- this.displayInputs = [0]
33
- }
34
- }
35
-
36
- let HTTPLoader = class HTTPLoader {
37
- static inputType = 'text'
38
-
39
- static outputType = 'text'
40
-
41
- static type = 'loader'
42
-
43
- static initStream (params) {
44
- return new Promise((resolve, reject) => {
45
- HTTPStream.get(params['URL'], function (stream) {
46
- stream.setEncoding('utf8')
47
- // if (params['Encoding'] === 'UTF-8') {}
48
- // stream.setEncoding(params['Encoding'] === 'UTF-8' ? 'utf8' : 'buffer')
49
- resolve(stream)
50
- })
51
- })
52
- }
53
-
54
- constructor (id) {
55
- this.name = this.constructor.name
56
- this.inputs = [
57
- { name: 'URL', type: 'string' },
58
- // { name: 'Encoding', type: 'categorical', options: ['UTF-8', 'Buffer'], default: 'UTF-8' },
59
- ]
60
- this.inputType = this.constructor.inputType
61
- this.outputType = this.constructor.outputType
62
- this.id = id
63
- this.expanded = false
64
- this.displayInputs = [0]
65
- }
66
- }
67
-
68
- let CSVParser = class CSVParser {
69
- static inputType = 'text'
70
-
71
- static outputType = 'object'
72
-
73
- static type = 'parser'
74
-
75
- static initStream (params) {
76
- const stream = ParseStream({
77
- 'delimiter': params['Delimiter'],
78
- 'cast': true,
79
- 'columns': true,
80
- 'comment': '#',
81
- 'trim': params['Trim'],
82
- 'relax_column_count': true,
83
- 'skip_empty_lines': params['Skip empty lines'],
84
- 'skip_lines_with_error': true
85
- })
86
- return stream
87
- }
88
-
89
- constructor (id) {
90
- this.name = this.constructor.name
91
- this.inputs = [
92
- { name: 'Delimiter', type: 'string', default: ',' },
93
- { name: 'Trim', type: 'bool', default: true },
94
- { name: 'Skip empty lines', type: 'bool', default: true }
95
- ]
96
- this.inputType = this.constructor.inputType
97
- this.outputType = this.constructor.outputType
98
- this.id = id
99
- this.expanded = false
100
- this.displayInputs = []
101
- }
102
- }
103
-
104
- let JSONParser = class JSONParser {
105
- static inputType = 'text'
106
-
107
- static outputType = 'object'
108
-
109
- static type = 'parser'
110
-
111
- static initStream (params) {
112
- const stream = JSONStream.parse(params['Path'])
113
- return stream
114
- }
115
-
116
- constructor (id) {
117
- this.name = this.constructor.name
118
- this.inputs = [
119
- { name: 'Path', type: 'string', default: '*' },
120
- ]
121
- this.inputType = this.constructor.inputType
122
- this.outputType = this.constructor.outputType
123
- this.id = id
124
- this.expanded = false
125
- this.displayInputs = []
126
- }
127
- }
128
-
129
- let TextOutput = class TextOutput {
130
- static inputType = 'text'
131
-
132
- static outputType = 'html'
133
-
134
- static type = 'output'
135
-
136
- static initStream (params) {
137
- const textOutput = document.getElementById('text-output')
138
- textOutput.style.display = 'block'
139
- textOutput.innerText = ''
140
- const stream = through2(function (chunk, enc, callback) {
141
- textOutput.innerText += chunk
142
- this.push(chunk)
143
- callback()
144
- })
145
- return stream
146
- }
147
-
148
- constructor (id) {
149
- this.name = this.constructor.name
150
- this.inputs = []
151
- this.inputType = this.constructor.inputType
152
- this.outputType = this.constructor.outputType
153
- this.id = id
154
- this.expanded = false
155
- this.displayInputs = []
156
- }
157
- }
158
- module.exports = {
159
- FileLoader,
160
- HTTPLoader,
161
- CSVParser,
162
- JSONParser,
163
- TextOutput
164
- }
165
-
166
-
package/webpack.config.js DELETED
@@ -1,108 +0,0 @@
1
- const path = require('path')
2
- const { VueLoaderPlugin } = require('vue-loader') // Support Vue's SFCs
3
- const TerserPlugin = require('terser-webpack-plugin') // Minimize code
4
- const webpack = require('webpack')
5
- const package = require('./package.json')
6
-
7
- module.exports = (env) => {
8
- const config = {
9
- entry: './src/main.js',
10
- output: {
11
- filename: 'bundle.js',
12
- path: path.resolve(__dirname, 'dist'),
13
- },
14
- module: {
15
- rules: [
16
- {
17
- test: /\.vue$/,
18
- loader: 'vue-loader'
19
- },
20
- {
21
- test: /\.js$/,
22
- loader: 'babel-loader'
23
- },
24
- {
25
- test: /\.mjs$/i, // lodash error (element)
26
- resolve: { byDependency: { esm: { fullySpecified: false } } }
27
- },
28
- {
29
- test: /\.html/,
30
- type: 'asset/source'
31
- },
32
- {
33
- test: /\.css$/,
34
- use: [
35
- 'vue-style-loader',
36
- 'css-loader',
37
- ]
38
- },
39
- {
40
- test: /worker\.js$/,
41
- loader: "worker-loader",
42
- options: {
43
- inline: 'no-fallback'
44
- },
45
- },
46
- {
47
- test: /\.scss$/,
48
- use: [
49
- 'vue-style-loader',
50
- 'css-loader',
51
- 'sass-loader'
52
- ]
53
- }
54
- ]
55
- },
56
- plugins: [
57
- new VueLoaderPlugin(),
58
- // Replace those values in the code
59
- new webpack.DefinePlugin({
60
- 'VERSION': JSON.stringify(package.version),
61
- }),
62
- // https://stackoverflow.com/a/68723223/2998960
63
- // Work around for Buffer is undefined:
64
- // https://github.com/webpack/changelog-v5/issues/10
65
- new webpack.ProvidePlugin({
66
- Buffer: ['buffer', 'Buffer'],
67
- }),
68
- new webpack.ProvidePlugin({
69
- process: 'process/browser',
70
- }),
71
- ],
72
- // Load different versions of vue based on RUNTIME value
73
- resolve: {
74
- fallback: {
75
- 'buffer': require.resolve('buffer'),
76
- 'stream': require.resolve('stream-browserify'),
77
- 'url': require.resolve('url')
78
- },
79
- alias: {
80
- vue$: 'vue/dist/vue.runtime.esm-bundler.js'
81
- },
82
- },
83
- // Update recommended size limit
84
- performance: {
85
- hints: false,
86
- maxEntrypointSize: 512000,
87
- maxAssetSize: 512000
88
- },
89
- // Remove comments
90
- optimization: {
91
- minimize: true,
92
- minimizer: [new TerserPlugin({
93
- extractComments: false,
94
- terserOptions: {
95
- format: {
96
- comments: false,
97
- },
98
- }
99
- })],
100
- },
101
- // Source map
102
- devtool: env.DEVELOPMENT
103
- ? 'eval-source-map'
104
- : false
105
- }
106
-
107
- return config
108
- }