tranfi 0.0.2 → 0.2.0

package/napi_transform.h

@@ -0,0 +1,8 @@
+#ifndef TRANFI_NAPI_TRANSFORM_H
+#define TRANFI_NAPI_TRANSFORM_H
+
+#include <node_api.h>
+
+napi_status tranfi_napi_define_transform(napi_env env, napi_value exports);
+
+#endif

package/package.json

@@ -1,68 +1,73 @@
 {
   "name": "tranfi",
-  "version": "0.0.2",
-  "description": "Stream-based tool for file processing",
-  "main": "src/main.js",
+  "version": "0.2.0",
+  "description": "Streaming ETL language + runtime",
+  "type": "commonjs",
+  "main": "src/index.js",
+  "bin": {
+    "tranfi": "src/cli.js"
+  },
+  "exports": {
+    ".": {
+      "require": "./src/index.js",
+      "default": "./src/index.js"
+    },
+    "./wasm": {
+      "require": "./wasm/index.js",
+      "default": "./wasm/index.js"
+    },
+    "./wasm/worker": {
+      "require": "./wasm/worker.js",
+      "default": "./wasm/worker.js"
+    }
+  },
+  "files": [
+    "src/",
+    "wasm/",
+    "csrc/",
+    "app/",
+    "napi_api.c",
+    "napi_transform.c",
+    "napi_transform.h",
+    "binding.gyp",
+    "scripts/",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
+  ],
   "scripts": {
-    "build-dev": "webpack --mode=development --progress --stats-children --env DEVELOPMENT",
-    "build": "webpack --mode=production --progress && webpack --mode=production --progress --env RUNTIME",
-    "watch": "nodemon --watch . --ignore dist --ext vue,js,css,html --exec 'npm run build-dev'",
-    "prepublishOnly": "npm run build",
-    "test": "jest test.js --detectOpenHandles",
-    "test-head": "HEADLESS=false npm test"
+    "prepack": "node scripts/prepack.js",
+    "install": "node scripts/install-native.js",
+    "build:native": "node scripts/sync-csrc.js && node-gyp rebuild",
+    "test": "node --test ../test/test_install_native.js && node ../test/test_node.js && node ../test/test_transform_node.js && node ../test/test_transform_wasm_node.js && node ../test/test_transform_worker_node.js"
+  },
+  "peerDependencies": {
+    "duckdb": ">=1.0.0",
+    "@duckdb/duckdb-wasm": ">=1.0.0"
   },
-  "author": "Anton Zemlyansky",
-  "license": "MIT",
+  "peerDependenciesMeta": {
+    "duckdb": {
+      "optional": true
+    },
+    "@duckdb/duckdb-wasm": {
+      "optional": true
+    }
+  },
+  "keywords": [
+    "etl",
+    "csv",
+    "streaming",
+    "data",
+    "pipeline",
+    "transform"
+  ],
+  "license": "Apache-2.0",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/tranfi/tranfi"
-  },
-  "dependencies": {
-    "@jseeio/jsee": "^0.2.8",
-    "buffer": "^6.0.3",
-    "bulma": "^0.9.3",
-    "csv-parse": "^4.16.3",
-    "element-plus": "^1.2.0-beta.6",
-    "equal-vue": "^0.79.3",
-    "file-saver": "^2.0.2",
-    "filestream": "^5.0.0",
-    "filtrex": "^2.2.3",
-    "JSONStream": "^1.3.5",
-    "notyf": "^3.10.0",
-    "process": "^0.11.10",
-    "stream-browserify": "^3.0.0",
-    "stream-http": "^3.2.0",
-    "through2": "^4.0.2",
-    "unzip-stream": "^0.3.1",
-    "url": "^0.11.0",
-    "vue": "^3.2.23",
-    "vue-style-loader": "^4.1.3",
-    "vue3-json-viewer": "^1.0.4",
-    "vuedraggable": "^4.1.0",
-    "vuex": "^4.0.2"
+    "url": "https://github.com/tranfi/tranfi"
   },
   "devDependencies": {
-    "@babel/core": "^7.16.5",
-    "@babel/plugin-transform-runtime": "^7.4.4",
-    "@babel/preset-env": "^7.16.5",
-    "babel-loader": "^8.2.3",
-    "css-loader": "^6.5.1",
-    "http-server": "^14.0.0",
-    "jest": "^27.4.5",
-    "jest-puppeteer": "^6.0.3",
-    "node-sass": "^7.0.0",
-    "nodemon": "^2.0.15",
-    "puppeteer": "^1.20.0",
-    "sass-loader": "^12.4.0",
-    "source-map-loader": "^3.0.0",
-    "style-loader": "^3.3.1",
-    "tape": "^4.9.1",
-    "terser-webpack-plugin": "^5.3.0",
-    "uglify-es": "^3.3.9",
-    "vue-loader": "^17.0.0",
-    "vue-template-compiler": "^2.6.14",
-    "webpack": "^5.65.0",
-    "webpack-cli": "^4.9.1",
-    "worker-loader": "^3.0.8"
-  }
+    "duckdb": "^1.4.4"
+  },
+  "author": "Anton Zemlyansky"
 }

package/scripts/install-native.js

@@ -0,0 +1,76 @@
+'use strict'
+
+const { spawnSync } = require('child_process')
+const path = require('path')
+
+function describeFailure(label, result, logger) {
+  if (result && result.error) {
+    logger.error(`tranfi: ${label} failed: ${result.error.message}`)
+  } else if (result && result.signal) {
+    logger.error(`tranfi: ${label} terminated by signal ${result.signal}`)
+  } else {
+    logger.error(`tranfi: ${label} exited with status ${result ? result.status : 'unknown'}`)
+  }
+}
+
+function successful(result) {
+  return Boolean(result) && result.error === undefined && result.signal === null
+    && result.status === 0
+}
+
+function installNative({
+  env = process.env,
+  execPath = process.execPath,
+  platform = process.platform,
+  spawn = spawnSync,
+  logger = console,
+  packageRoot = path.resolve(__dirname, '..')
+} = {}) {
+  if (env.TRANFI_SKIP_NATIVE_BUILD === '1') {
+    logger.warn(
+      'tranfi: skipping the native addon because TRANFI_SKIP_NATIVE_BUILD=1; ' +
+      "use require('tranfi/wasm') explicitly because prepared transforms on " +
+      "require('tranfi') need the native addon"
+    )
+    return 0
+  }
+
+  const syncResult = spawn(execPath, [path.join(__dirname, 'sync-csrc.js')], {
+    cwd: packageRoot,
+    stdio: 'inherit'
+  })
+  if (!successful(syncResult)) {
+    describeFailure('C-source synchronization', syncResult, logger)
+    return Number.isInteger(syncResult && syncResult.status) && syncResult.status !== 0
+      ? syncResult.status
+      : 1
+  }
+
+  const nodeGyp = env.npm_config_node_gyp || 'node-gyp'
+  const nodeGypIsScript = /\.[cm]?js$/i.test(nodeGyp)
+  const buildCommand = nodeGypIsScript ? execPath : nodeGyp
+  const buildArgs = nodeGypIsScript ? [nodeGyp, 'rebuild'] : ['rebuild']
+  const buildResult = spawn(buildCommand, buildArgs, {
+    cwd: packageRoot,
+    stdio: 'inherit',
+    shell: platform === 'win32' && !nodeGypIsScript
+  })
+  if (!successful(buildResult)) {
+    describeFailure('native addon build', buildResult, logger)
+    logger.error(
+      'tranfi: installation requires a working native toolchain for the default ' +
+      "Node entry; set TRANFI_SKIP_NATIVE_BUILD=1 only for explicit 'tranfi/wasm' use"
+    )
+    return Number.isInteger(buildResult && buildResult.status) && buildResult.status !== 0
+      ? buildResult.status
+      : 1
+  }
+
+  return 0
+}
+
+if (require.main === module) {
+  process.exitCode = installNative()
+}
+
+module.exports = { installNative }

package/scripts/prepack.js

@@ -0,0 +1,64 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+const { spawnSync } = require('child_process')
+
+const scriptsDir = __dirname
+const pkgDir = path.resolve(scriptsDir, '..')
+const repoDir = path.resolve(pkgDir, '..')
+
+function run(cmd, args, cwd, extraEnv = {}) {
+  const result = spawnSync(cmd, args, {
+    cwd,
+    stdio: 'inherit',
+    env: { ...process.env, ...extraEnv }
+  })
+  if (result.status !== 0) {
+    process.exit(result.status || 1)
+  }
+}
+
+run(process.execPath, [path.join(scriptsDir, 'sync-csrc.js')], pkgDir)
+
+const wasmFile = path.join(pkgDir, 'wasm', 'tranfi_core.js')
+if (!fs.existsSync(wasmFile)) {
+  const env = { EM_CACHE: process.env.EM_CACHE || path.join(repoDir, 'build-wasm', '.emcache') }
+  if (!process.env.EMSDK_PYTHON && fs.existsSync('/usr/bin/python3')) {
+    env.EMSDK_PYTHON = '/usr/bin/python3'
+  }
+  run('bash', [path.join(repoDir, 'scripts', 'build-wasm.sh')], repoDir, env)
+}
+
+const appDir = path.join(repoDir, 'app')
+const appPublicWasmDir = path.join(appDir, 'public', 'wasm')
+const appSource = path.join(appDir, 'dist')
+const appDest = path.join(pkgDir, 'app')
+
+fs.mkdirSync(appPublicWasmDir, { recursive: true })
+fs.copyFileSync(wasmFile, path.join(appPublicWasmDir, 'tranfi_core.js'))
+
+if (fs.existsSync(path.join(appDir, 'package.json'))) {
+  run('npm', ['run', 'build'], appDir)
+}
+
+if (fs.existsSync(appSource)) {
+  fs.rmSync(appDest, { recursive: true, force: true })
+  fs.cpSync(appSource, appDest, { recursive: true })
+  for (const extra of ['wasm', 'lib']) {
+    fs.rmSync(path.join(appDest, extra), { recursive: true, force: true })
+  }
+}
+
+const required = [
+  path.join(pkgDir, 'csrc'),
+  path.join(pkgDir, 'wasm', 'tranfi_core.js'),
+  path.join(pkgDir, 'app', 'index.html')
+]
+
+for (const target of required) {
+  if (!fs.existsSync(target)) {
+    console.error('prepack: missing required publish artifact: ' + target)
+    process.exit(1)
+  }
+}

package/scripts/sync-csrc.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+
+const pkgDir = path.resolve(__dirname, '..')
+const repoDir = path.resolve(pkgDir, '..')
+const srcDir = path.join(repoDir, 'src')
+const outDir = path.join(pkgDir, 'csrc')
+
+if (!fs.existsSync(srcDir)) {
+  console.log('sync-csrc: src/ not found (published package), skipping')
+  process.exit(0)
+}
+
+fs.rmSync(outDir, { recursive: true, force: true })
+fs.mkdirSync(outDir, { recursive: true })
+
+for (const name of fs.readdirSync(srcDir)) {
+  if (name.endsWith('.c') || name.endsWith('.h')) {
+    fs.copyFileSync(path.join(srcDir, name), path.join(outDir, name))
+  }
+}

package/src/cli.js

@@ -0,0 +1,190 @@
+#!/usr/bin/env node
+
+/**
+ * cli.js — Tranfi CLI for Node.js.
+ *
+ * Usage:
+ *   tranfi 'csv | filter "age > 25" | csv'  < in.csv
+ *   tranfi -f pipeline.tf < in.csv > out.csv
+ *   tranfi profile < data.csv
+ */
+
+const { pipeline, compileDsl, recipes } = require('./index.js')
+const { readFileSync, writeFileSync, existsSync } = require('fs')
+const { resolve, dirname } = require('path')
+const nativeBinding = require('./native.js')
+
+function usage() {
+  process.stderr.write(`Usage: tranfi [OPTIONS] PIPELINE
+       tranfi [OPTIONS] -f FILE
+
+Streaming ETL with a pipe-style DSL.
+
+Examples:
+  tranfi 'csv | csv'                           # passthrough
+  tranfi 'csv | filter "age > 25" | csv'       # filter rows
+  tranfi 'csv | select name,age | csv'         # select columns
+  tranfi 'csv | head 10 | csv'                 # first N rows
+  tranfi 'csv | top-k 10 age | csv'            # bounded top rows by column
+  tranfi 'csv | stats | jsonl'                 # aggregate stats
+  tranfi profile                               # built-in recipe
+
+Options:
+  -f FILE   Read pipeline from file
+  -i FILE   Read input from file instead of stdin
+  -o FILE   Write output to file instead of stdout
+  -j        Compile only, output JSON plan
+  -q        Quiet mode (suppress stats)
+  -v        Show version
+  -R        List built-in recipes
+  -h        Show this help
+`)
+}
+
+function findAppDir () {
+  // Try known locations relative to this file
+  const candidates = [
+    resolve(__dirname, '../../app/dist'),          // dev: js/src/ -> ../../app/dist
+    resolve(__dirname, '../app'),                 // npm package: src/ -> ../app/
+  ]
+  for (const dir of candidates) {
+    if (existsSync(resolve(dir, 'index.html'))) return dir
+  }
+  return null
+}
+
+async function serveCommand (argv) {
+  const { startServer } = require('./server.js')
+  let dataDir = '.'
+  let appDir = null
+  let port = 3000
+
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i]
+    if (arg === '-d' || arg === '--data') dataDir = argv[++i]
+    else if (arg === '-a' || arg === '--app') appDir = argv[++i]
+    else if (arg === '-p' || arg === '--port') port = parseInt(argv[++i], 10)
+    else if (arg === '-h' || arg === '--help') {
+      process.stderr.write(`Usage: tranfi serve [OPTIONS]
+
+Serve the tranfi app with a native backend.
+
+Options:
+  -d, --data DIR   Data directory (default: .)
+  -a, --app DIR    App dist directory (auto-detect)
+  -p, --port PORT  Port (default: 3000)
+  -h, --help       Show this help
+`)
+      process.exit(0)
+    }
+  }
+
+  if (!appDir) appDir = findAppDir()
+  if (!appDir) {
+    process.stderr.write('error: app directory not found. Use --app to specify it.\n')
+    process.exit(1)
+  }
+
+  startServer({ port, dataDir: resolve(dataDir), appDir: resolve(appDir) })
+}
+
+async function main() {
+  const argv = process.argv.slice(2)
+
+  // Check for serve subcommand
+  if (argv[0] === 'serve') {
+    return serveCommand(argv.slice(1))
+  }
+
+  let pipelineFile = null
+  let pipelineText = null
+  let inputFile = null
+  let outputFile = null
+  let jsonMode = false
+  let quiet = false
+
+  // Parse args
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i]
+    if (arg === '-h' || arg === '--help') { usage(); process.exit(0) }
+    else if (arg === '-v' || arg === '--version') {
+      const v = nativeBinding ? nativeBinding.version() : 'unknown'
+      process.stdout.write(`tranfi ${v}\n`)
+      process.exit(0)
+    }
+    else if (arg === '-R' || arg === '--recipes') {
+      const list = await recipes()
+      process.stdout.write(`Built-in recipes (${list.length}):\n\n`)
+      for (const r of list) {
+        process.stdout.write(`  ${r.name.padEnd(12)} ${r.description}\n`)
+        process.stdout.write(`  ${''.padEnd(12)} ${r.dsl}\n\n`)
+      }
+      process.exit(0)
+    }
+    else if (arg === '-j') jsonMode = true
+    else if (arg === '-q') quiet = true
+    else if (arg === '-f') { pipelineFile = argv[++i] }
+    else if (arg === '-i') { inputFile = argv[++i] }
+    else if (arg === '-o') { outputFile = argv[++i] }
+    else if (!arg.startsWith('-')) { pipelineText = arg }
+    else {
+      process.stderr.write(`error: unknown option '${arg}'\n`)
+      process.exit(1)
+    }
+  }
+
+  // Get pipeline text
+  if (pipelineFile) {
+    pipelineText = readFileSync(pipelineFile, 'utf-8')
+  }
+  if (!pipelineText) {
+    process.stderr.write('error: no pipeline specified\n\n')
+    usage()
+    process.exit(1)
+  }
+
+  // JSON mode
+  if (jsonMode) {
+    const json = await compileDsl(pipelineText)
+    process.stdout.write(json + '\n')
+    process.exit(0)
+  }
+
+  // Create and run pipeline
+  const p = pipeline(pipelineText)
+  const input = inputFile ? undefined : await readStdin()
+  const result = await p.run({
+    inputFile: inputFile || undefined,
+    input
+  })
+
+  // Write output
+  if (outputFile) {
+    writeFileSync(outputFile, result.output)
+  } else {
+    process.stdout.write(result.output)
+  }
+
+  // Errors to stderr
+  if (result.errors.length > 0) {
+    process.stderr.write(result.errors)
+  }
+
+  // Stats to stderr (unless quiet)
+  if (!quiet && result.stats.length > 0) {
+    process.stderr.write(result.stats)
+  }
+}
+
+function readStdin() {
+  return new Promise((resolve) => {
+    const chunks = []
+    process.stdin.on('data', chunk => chunks.push(chunk))
+    process.stdin.on('end', () => resolve(Buffer.concat(chunks)))
+  })
+}
+
+main().catch(err => {
+  process.stderr.write(`error: ${err.message}\n`)
+  process.exit(1)
+})

package/src/engines/duckdb.js

@@ -0,0 +1,142 @@
+/**
+ * duckdb.js — DuckDB engine for Node.js.
+ *
+ * Executes tranfi pipelines via SQL transpilation + duckdb.
+ * Install: npm install duckdb
+ */
+
+const { writeFile, unlink } = require('fs/promises')
+const { tmpdir } = require('os')
+const { join } = require('path')
+const { randomBytes } = require('crypto')
+const { PipelineResult } = require('../pipeline.js')
+const { memoryForEngine } = require('../memory_policy.js')
+
+let _compileToSql = null
+
+async function getCompileToSql() {
+  if (_compileToSql) return _compileToSql
+  const { compileToSql } = require('../pipeline.js')
+  _compileToSql = compileToSql
+  return _compileToSql
+}
+
+class DuckDBEngine {
+  async run(dsl, { input, inputFile, memory, spillDir } = {}) {
+    let duckdb
+    try {
+      duckdb = require('duckdb')
+    } catch {
+      throw new Error(
+        "DuckDB engine requires the 'duckdb' package. " +
+        'Install it with: npm install duckdb'
+      )
+    }
+
+    const compileToSql = await getCompileToSql()
+    let sql = await compileToSql(dsl)
+
+    const db = new duckdb.Database(':memory:')
+    const conn = db.connect()
+
+    let tmpPath = null
+    try {
+      const memoryLimit = memoryForEngine(memory)
+      if (memoryLimit) await runSql(conn, `SET memory_limit = '${sqlString(memoryLimit)}'`)
+      if (spillDir) await runSql(conn, `SET temp_directory = '${sqlString(spillDir)}'`)
+
+      if (inputFile) {
+        sql = sql.replaceAll('input_data', `read_csv('${sqlString(inputFile)}')`)
+      } else if (input) {
+        const buf = typeof input === 'string' ? Buffer.from(input, 'utf-8') : input
+        tmpPath = join(tmpdir(), `tranfi_${randomBytes(8).toString('hex')}.csv`)
+        await writeFile(tmpPath, buf)
+        sql = sql.replaceAll('input_data', `read_csv('${sqlString(tmpPath)}')`)
+      } else {
+        throw new Error('Either input or inputFile must be provided')
+      }
+
+      const rows = await queryAll(conn, sql)
+      const output = rowsToCsv(rows)
+
+      return new PipelineResult(
+        Buffer.from(output, 'utf-8'),
+        Buffer.alloc(0),
+        Buffer.alloc(0),
+        Buffer.alloc(0)
+      )
+    } finally {
+      conn.close()
+      db.close()
+      if (tmpPath) {
+        await unlink(tmpPath).catch(() => {})
+      }
+    }
+  }
+}
+
+function sqlString(value) {
+  return String(value).replace(/'/g, "''")
+}
+
+function runSql(conn, sql) {
+  return new Promise((resolve, reject) => {
+    conn.run(sql, err => {
+      if (err) reject(err)
+      else resolve()
+    })
+  })
+}
+
+function queryAll(conn, sql) {
+  return new Promise((resolve, reject) => {
+    conn.all(sql, (err, rows) => {
+      if (err) reject(err)
+      else resolve(rows)
+    })
+  })
+}
+
+function rowsToCsv(rows) {
+  if (!rows || rows.length === 0) return ''
+  const cols = Object.keys(rows[0])
+  const lines = [cols.join(',')]
+  for (const row of rows) {
+    const vals = cols.map(c => {
+      const v = row[c]
+      if (v === null || v === undefined) return ''
+      const s = formatDuckValue(v)
+      if (s.includes(',') || s.includes('"') || s.includes('\n')) {
+        return '"' + s.replace(/"/g, '""') + '"'
+      }
+      return s
+    })
+    lines.push(vals.join(','))
+  }
+  return lines.join('\n') + '\n'
+}
+
+function formatDuckValue(value) {
+  if (typeof value !== 'number') return String(value)
+  if (!Number.isFinite(value)) return String(value)
+  if (Number.isInteger(value) && Math.abs(value) <= Number.MAX_SAFE_INTEGER) {
+    return String(value)
+  }
+  let text = value.toPrecision(17)
+  if (!/[eE]/.test(text) && text.includes('.')) {
+    text = text.replace(/0+$/, '').replace(/\.$/, '')
+  }
+  return text
+}
+
+let _engine = null
+
+function getEngine(name) {
+  if (name === 'duckdb') {
+    if (!_engine) _engine = new DuckDBEngine()
+    return _engine
+  }
+  throw new Error(`Unknown engine: '${name}'. Available: 'native', 'duckdb'`)
+}
+
+module.exports = { getEngine }