tranfi 0.1.2 → 0.2.1

package/napi_transform.h

@@ -0,0 +1,8 @@
+#ifndef TRANFI_NAPI_TRANSFORM_H
+#define TRANFI_NAPI_TRANSFORM_H
+
+#include <node_api.h>
+
+napi_status tranfi_napi_define_transform(napi_env env, napi_value exports);
+
+#endif

package/package.json

@@ -1,15 +1,25 @@
 {
   "name": "tranfi",
-  "version": "0.1.2",
+  "version": "0.2.1",
   "description": "Streaming ETL language + runtime",
-  "type": "module",
+  "type": "commonjs",
   "main": "src/index.js",
   "bin": {
     "tranfi": "src/cli.js"
   },
   "exports": {
-    ".": "./src/index.js",
-    "./wasm": "./wasm/index.js"
+    ".": {
+      "require": "./src/index.js",
+      "default": "./src/index.js"
+    },
+    "./wasm": {
+      "require": "./wasm/index.js",
+      "default": "./wasm/index.js"
+    },
+    "./wasm/worker": {
+      "require": "./wasm/worker.js",
+      "default": "./wasm/worker.js"
+    }
   },
   "files": [
     "src/",
@@ -17,14 +27,19 @@
     "csrc/",
     "app/",
     "napi_api.c",
+    "napi_transform.c",
+    "napi_transform.h",
     "binding.gyp",
-    "README.md"
+    "scripts/",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
   ],
   "scripts": {
-    "prepack": "rm -rf csrc && mkdir -p csrc && cp ../src/*.c ../src/*.h csrc/ && rm -rf app && cp -r ../app/dist app && rm -rf app/wasm app/lib",
-    "install": "node-gyp rebuild 2>/dev/null || true",
-    "build:native": "rm -rf csrc && mkdir -p csrc && cp ../src/*.c ../src/*.h csrc/ && node-gyp rebuild",
-    "test": "node ../test/test_node.js"
+    "prepack": "node scripts/prepack.js",
+    "install": "node scripts/install-native.js",
+    "build:native": "node scripts/sync-csrc.js && node-gyp rebuild",
+    "test": "node --test ../test/test_install_native.js ../test/test_cli_node.js && node ../test/test_node.js && node ../test/test_transform_node.js && node ../test/test_transform_wasm_node.js && node ../test/test_transform_worker_node.js"
   },
   "peerDependencies": {
     "duckdb": ">=1.0.0",
@@ -46,12 +61,13 @@
     "pipeline",
     "transform"
   ],
-  "license": "MIT",
+  "license": "Apache-2.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/tranfi/tranfi"
   },
   "devDependencies": {
     "duckdb": "^1.4.4"
-  }
+  },
+  "author": "Anton Zemlyansky"
 }

package/scripts/install-native.js

@@ -0,0 +1,76 @@
+'use strict'
+
+const { spawnSync } = require('child_process')
+const path = require('path')
+
+function describeFailure(label, result, logger) {
+  if (result && result.error) {
+    logger.error(`tranfi: ${label} failed: ${result.error.message}`)
+  } else if (result && result.signal) {
+    logger.error(`tranfi: ${label} terminated by signal ${result.signal}`)
+  } else {
+    logger.error(`tranfi: ${label} exited with status ${result ? result.status : 'unknown'}`)
+  }
+}
+
+function successful(result) {
+  return Boolean(result) && result.error === undefined && result.signal === null
+    && result.status === 0
+}
+
+function installNative({
+  env = process.env,
+  execPath = process.execPath,
+  platform = process.platform,
+  spawn = spawnSync,
+  logger = console,
+  packageRoot = path.resolve(__dirname, '..')
+} = {}) {
+  if (env.TRANFI_SKIP_NATIVE_BUILD === '1') {
+    logger.warn(
+      'tranfi: skipping the native addon because TRANFI_SKIP_NATIVE_BUILD=1; ' +
+      "use require('tranfi/wasm') explicitly because prepared transforms on " +
+      "require('tranfi') need the native addon"
+    )
+    return 0
+  }
+
+  const syncResult = spawn(execPath, [path.join(__dirname, 'sync-csrc.js')], {
+    cwd: packageRoot,
+    stdio: 'inherit'
+  })
+  if (!successful(syncResult)) {
+    describeFailure('C-source synchronization', syncResult, logger)
+    return Number.isInteger(syncResult && syncResult.status) && syncResult.status !== 0
+      ? syncResult.status
+      : 1
+  }
+
+  const nodeGyp = env.npm_config_node_gyp || 'node-gyp'
+  const nodeGypIsScript = /\.[cm]?js$/i.test(nodeGyp)
+  const buildCommand = nodeGypIsScript ? execPath : nodeGyp
+  const buildArgs = nodeGypIsScript ? [nodeGyp, 'rebuild'] : ['rebuild']
+  const buildResult = spawn(buildCommand, buildArgs, {
+    cwd: packageRoot,
+    stdio: 'inherit',
+    shell: platform === 'win32' && !nodeGypIsScript
+  })
+  if (!successful(buildResult)) {
+    describeFailure('native addon build', buildResult, logger)
+    logger.error(
+      'tranfi: installation requires a working native toolchain for the default ' +
+      "Node entry; set TRANFI_SKIP_NATIVE_BUILD=1 only for explicit 'tranfi/wasm' use"
+    )
+    return Number.isInteger(buildResult && buildResult.status) && buildResult.status !== 0
+      ? buildResult.status
+      : 1
+  }
+
+  return 0
+}
+
+if (require.main === module) {
+  process.exitCode = installNative()
+}
+
+module.exports = { installNative }

package/scripts/prepack.js

@@ -0,0 +1,64 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+const { spawnSync } = require('child_process')
+
+const scriptsDir = __dirname
+const pkgDir = path.resolve(scriptsDir, '..')
+const repoDir = path.resolve(pkgDir, '..')
+
+function run(cmd, args, cwd, extraEnv = {}) {
+  const result = spawnSync(cmd, args, {
+    cwd,
+    stdio: 'inherit',
+    env: { ...process.env, ...extraEnv }
+  })
+  if (result.status !== 0) {
+    process.exit(result.status || 1)
+  }
+}
+
+run(process.execPath, [path.join(scriptsDir, 'sync-csrc.js')], pkgDir)
+
+const wasmFile = path.join(pkgDir, 'wasm', 'tranfi_core.js')
+if (!fs.existsSync(wasmFile)) {
+  const env = { EM_CACHE: process.env.EM_CACHE || path.join(repoDir, 'build-wasm', '.emcache') }
+  if (!process.env.EMSDK_PYTHON && fs.existsSync('/usr/bin/python3')) {
+    env.EMSDK_PYTHON = '/usr/bin/python3'
+  }
+  run('bash', [path.join(repoDir, 'scripts', 'build-wasm.sh')], repoDir, env)
+}
+
+const appDir = path.join(repoDir, 'app')
+const appPublicWasmDir = path.join(appDir, 'public', 'wasm')
+const appSource = path.join(appDir, 'dist')
+const appDest = path.join(pkgDir, 'app')
+
+fs.mkdirSync(appPublicWasmDir, { recursive: true })
+fs.copyFileSync(wasmFile, path.join(appPublicWasmDir, 'tranfi_core.js'))
+
+if (fs.existsSync(path.join(appDir, 'package.json'))) {
+  run('npm', ['run', 'build'], appDir)
+}
+
+if (fs.existsSync(appSource)) {
+  fs.rmSync(appDest, { recursive: true, force: true })
+  fs.cpSync(appSource, appDest, { recursive: true })
+  for (const extra of ['wasm', 'lib']) {
+    fs.rmSync(path.join(appDest, extra), { recursive: true, force: true })
+  }
+}
+
+const required = [
+  path.join(pkgDir, 'csrc'),
+  path.join(pkgDir, 'wasm', 'tranfi_core.js'),
+  path.join(pkgDir, 'app', 'index.html')
+]
+
+for (const target of required) {
+  if (!fs.existsSync(target)) {
+    console.error('prepack: missing required publish artifact: ' + target)
+    process.exit(1)
+  }
+}

package/scripts/sync-csrc.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+
+const pkgDir = path.resolve(__dirname, '..')
+const repoDir = path.resolve(pkgDir, '..')
+const srcDir = path.join(repoDir, 'src')
+const outDir = path.join(pkgDir, 'csrc')
+
+if (!fs.existsSync(srcDir)) {
+  console.log('sync-csrc: src/ not found (published package), skipping')
+  process.exit(0)
+}
+
+fs.rmSync(outDir, { recursive: true, force: true })
+fs.mkdirSync(outDir, { recursive: true })
+
+for (const name of fs.readdirSync(srcDir)) {
+  if (name.endsWith('.c') || name.endsWith('.h')) {
+    fs.copyFileSync(path.join(srcDir, name), path.join(outDir, name))
+  }
+}

package/src/cli.js

@@ -9,13 +9,10 @@
  *   tranfi profile < data.csv
  */
 
-import { pipeline, compileDsl, recipes } from './index.js'
-import { readFileSync, writeFileSync, existsSync } from 'fs'
-import { resolve, dirname } from 'path'
-import { fileURLToPath } from 'url'
-import nativeBinding from './native.js'
-
-const __dirname = dirname(fileURLToPath(import.meta.url))
+const { pipeline, compileDsl, compileToSql, recipes } = require('./index.js')
+const { readFileSync, writeFileSync, existsSync, openSync, closeSync } = require('fs')
+const { resolve, dirname } = require('path')
+const nativeBinding = require('./native.js')
 
 function usage() {
   process.stderr.write(`Usage: tranfi [OPTIONS] PIPELINE
@@ -28,7 +25,7 @@ Examples:
   tranfi 'csv | filter "age > 25" | csv'       # filter rows
   tranfi 'csv | select name,age | csv'         # select columns
   tranfi 'csv | head 10 | csv'                 # first N rows
-  tranfi 'csv | sort age | csv'                # sort by column
+  tranfi 'csv | top-k 10 age | csv'            # bounded top rows by column
   tranfi 'csv | stats | jsonl'                 # aggregate stats
   tranfi profile                               # built-in recipe
 
@@ -37,6 +34,13 @@ Options:
   -i FILE   Read input from file instead of stdin
   -o FILE   Write output to file instead of stdout
   -j        Compile only, output JSON plan
+  --target native|json|sql  Execute or compile a pipeline
+  --dialect NAME           SQL dialect (duckdb)
+  --allow-blocking         Permit full-input operations on known-small data
+  --fail-on-blocking       Reject blocking operations (default)
+  --memory SIZE            Native memory policy, e.g. max:64MB
+  --spill-dir DIR          Existing private spill directory
+  --stats-json FILE        Write stats NDJSON; - means stderr
   -q        Quiet mode (suppress stats)
   -v        Show version
   -R        List built-in recipes
@@ -47,8 +51,8 @@ Options:
 function findAppDir () {
   // Try known locations relative to this file
   const candidates = [
-    resolve(__dirname, '../../app/dist'),          // dev: js/src/ → ../../app/dist
-    resolve(__dirname, '../app'),                 // npm package: src/ → ../app/
+    resolve(__dirname, '../../app/dist'),          // dev: js/src/ -> ../../app/dist
+    resolve(__dirname, '../app'),                 // npm package: src/ -> ../app/
   ]
   for (const dir of candidates) {
     if (existsSync(resolve(dir, 'index.html'))) return dir
@@ -57,7 +61,7 @@ function findAppDir () {
 }
 
 async function serveCommand (argv) {
-  const { startServer } = await import('./server.js')
+  const { startServer } = require('./server.js')
   let dataDir = '.'
   let appDir = null
   let port = 3000
@@ -105,10 +109,21 @@ async function main() {
   let outputFile = null
   let jsonMode = false
   let quiet = false
+  let allowBlocking = false
+  let failOnBlocking = false
+  let memory, spillDir, statsFile
+  let target = 'native'
+  let dialect = 'duckdb'
+  function valueAt(i, inline, flag) {
+    const value = inline === undefined ? argv[i + 1] : inline
+    if (!value || value.startsWith('--')) throw new Error(`${flag} requires a value`)
+    return value
+  }
 
   // Parse args
   for (let i = 0; i < argv.length; i++) {
-    const arg = argv[i]
+    const [arg, ...assigned] = argv[i].split('=')
+    const inline = assigned.length ? assigned.join('=') : undefined
     if (arg === '-h' || arg === '--help') { usage(); process.exit(0) }
     else if (arg === '-v' || arg === '--version') {
       const v = nativeBinding ? nativeBinding.version() : 'unknown'
@@ -124,18 +139,35 @@ async function main() {
       }
       process.exit(0)
     }
-    else if (arg === '-j') jsonMode = true
-    else if (arg === '-q') quiet = true
-    else if (arg === '-f') { pipelineFile = argv[++i] }
-    else if (arg === '-i') { inputFile = argv[++i] }
-    else if (arg === '-o') { outputFile = argv[++i] }
-    else if (!arg.startsWith('-')) { pipelineText = arg }
+    else if (arg === '-j' || arg === '--json') jsonMode = true
+    else if (arg === '-q' || arg === '--quiet') quiet = true
+    else if (arg === '--allow-blocking') allowBlocking = true
+    else if (arg === '--fail-on-blocking') failOnBlocking = true
+    else if (['-f', '--file', '-i', '--input', '-o', '--output', '--memory', '--spill-dir', '--stats-json', '--target', '--dialect'].includes(arg)) {
+      const value = valueAt(i, inline, arg)
+      if (inline === undefined) i++
+      if (arg === '-f' || arg === '--file') pipelineFile = value
+      else if (arg === '-i' || arg === '--input') inputFile = value
+      else if (arg === '-o' || arg === '--output') outputFile = value
+      else if (arg === '--memory') memory = value
+      else if (arg === '--spill-dir') spillDir = value
+      else if (arg === '--stats-json') statsFile = value
+      else if (arg === '--target') target = value
+      else if (arg === '--dialect') dialect = value
+    }
+    else if (!argv[i].startsWith('-')) {
+      if (pipelineText !== null) throw new Error('only one pipeline argument is allowed')
+      pipelineText = argv[i]
+    }
     else {
       process.stderr.write(`error: unknown option '${arg}'\n`)
       process.exit(1)
     }
   }
 
+  if (allowBlocking && failOnBlocking) throw new Error('--allow-blocking conflicts with --fail-on-blocking')
+  if (!['native', 'json', 'sql'].includes(target)) throw new Error('unknown --target; expected native, json or sql')
+
   // Get pipeline text
   if (pipelineFile) {
     pipelineText = readFileSync(pipelineFile, 'utf-8')
@@ -146,26 +178,44 @@ async function main() {
     process.exit(1)
   }
 
-  // JSON mode
-  if (jsonMode) {
+  const named = (await recipes()).find(recipe => recipe.name === pipelineText.trim())
+  if (named) pipelineText = named.dsl
+  if (target === 'sql') {
+    process.stdout.write(await compileToSql(pipelineText, { dialect }) + '\n')
+    return
+  }
+
+  if (jsonMode || target === 'json') {
     const json = await compileDsl(pipelineText)
     process.stdout.write(json + '\n')
     process.exit(0)
   }
 
-  // Create and run pipeline
   const p = pipeline(pipelineText)
-  const input = inputFile ? undefined : await readStdin()
-  const result = await p.run({
+  const options = {
     inputFile: inputFile || undefined,
-    input
-  })
-
-  // Write output
+    inputStream: inputFile ? undefined : process.stdin,
+    allowBlocking, memory, spillDir, allowFs: true, allowRulesFile: true
+  }
+  let result
   if (outputFile) {
-    writeFileSync(outputFile, result.output)
+    let fd
+    try {
+      result = await p.run({
+        ...options,
+        collectOutput: false,
+        onOutput(chunk) {
+          // Open only after plan validation; a rejected plan must not truncate output.
+          if (fd === undefined) fd = openSync(outputFile, 'w')
+          writeFileSync(fd, chunk)
+        }
+      })
+      if (fd === undefined) fd = openSync(outputFile, 'w')
+    } finally {
+      if (fd !== undefined) closeSync(fd)
+    }
   } else {
-    process.stdout.write(result.output)
+    result = await p.writeTo(process.stdout, { ...options, end: false })
   }
 
   // Errors to stderr
@@ -173,18 +223,8 @@ async function main() {
     process.stderr.write(result.errors)
   }
 
-  // Stats to stderr (unless quiet)
-  if (!quiet && result.stats.length > 0) {
-    process.stderr.write(result.stats)
-  }
-}
-
-function readStdin() {
-  return new Promise((resolve) => {
-    const chunks = []
-    process.stdin.on('data', chunk => chunks.push(chunk))
-    process.stdin.on('end', () => resolve(Buffer.concat(chunks)))
-  })
+  if (statsFile && statsFile !== '-') writeFileSync(statsFile, result.stats)
+  else if ((statsFile || !quiet) && result.stats.length > 0) process.stderr.write(result.stats)
 }
 
 main().catch(err => {

package/src/engines/duckdb.js

@@ -5,26 +5,27 @@
  * Install: npm install duckdb
  */
 
-import { writeFile, unlink } from 'fs/promises'
-import { tmpdir } from 'os'
-import { join } from 'path'
-import { randomBytes } from 'crypto'
-import { PipelineResult } from '../pipeline.js'
+const { writeFile, unlink } = require('fs/promises')
+const { tmpdir } = require('os')
+const { join } = require('path')
+const { randomBytes } = require('crypto')
+const { PipelineResult } = require('../pipeline.js')
+const { memoryForEngine } = require('../memory_policy.js')
 
 let _compileToSql = null
 
 async function getCompileToSql() {
   if (_compileToSql) return _compileToSql
-  const { compileToSql } = await import('../pipeline.js')
+  const { compileToSql } = require('../pipeline.js')
   _compileToSql = compileToSql
   return _compileToSql
 }
 
 class DuckDBEngine {
-  async run(dsl, { input, inputFile } = {}) {
+  async run(dsl, { input, inputFile, memory, spillDir } = {}) {
     let duckdb
     try {
-      duckdb = (await import('duckdb')).default || await import('duckdb')
+      duckdb = require('duckdb')
     } catch {
       throw new Error(
         "DuckDB engine requires the 'duckdb' package. " +
@@ -40,13 +41,17 @@ class DuckDBEngine {
 
     let tmpPath = null
     try {
+      const memoryLimit = memoryForEngine(memory)
+      if (memoryLimit) await runSql(conn, `SET memory_limit = '${sqlString(memoryLimit)}'`)
+      if (spillDir) await runSql(conn, `SET temp_directory = '${sqlString(spillDir)}'`)
+
       if (inputFile) {
-        sql = sql.replaceAll('input_data', `read_csv('${inputFile}')`)
+        sql = sql.replaceAll('input_data', `read_csv('${sqlString(inputFile)}')`)
       } else if (input) {
         const buf = typeof input === 'string' ? Buffer.from(input, 'utf-8') : input
         tmpPath = join(tmpdir(), `tranfi_${randomBytes(8).toString('hex')}.csv`)
         await writeFile(tmpPath, buf)
-        sql = sql.replaceAll('input_data', `read_csv('${tmpPath}')`)
+        sql = sql.replaceAll('input_data', `read_csv('${sqlString(tmpPath)}')`)
       } else {
         throw new Error('Either input or inputFile must be provided')
       }
@@ -70,6 +75,19 @@ class DuckDBEngine {
   }
 }
 
+function sqlString(value) {
+  return String(value).replace(/'/g, "''")
+}
+
+function runSql(conn, sql) {
+  return new Promise((resolve, reject) => {
+    conn.run(sql, err => {
+      if (err) reject(err)
+      else resolve()
+    })
+  })
+}
+
 function queryAll(conn, sql) {
   return new Promise((resolve, reject) => {
     conn.all(sql, (err, rows) => {
@@ -87,7 +105,7 @@ function rowsToCsv(rows) {
     const vals = cols.map(c => {
       const v = row[c]
       if (v === null || v === undefined) return ''
-      const s = String(v)
+      const s = formatDuckValue(v)
       if (s.includes(',') || s.includes('"') || s.includes('\n')) {
         return '"' + s.replace(/"/g, '""') + '"'
       }
@@ -98,12 +116,27 @@ function rowsToCsv(rows) {
   return lines.join('\n') + '\n'
 }
 
+function formatDuckValue(value) {
+  if (typeof value !== 'number') return String(value)
+  if (!Number.isFinite(value)) return String(value)
+  if (Number.isInteger(value) && Math.abs(value) <= Number.MAX_SAFE_INTEGER) {
+    return String(value)
+  }
+  let text = value.toPrecision(17)
+  if (!/[eE]/.test(text) && text.includes('.')) {
+    text = text.replace(/0+$/, '').replace(/\.$/, '')
+  }
+  return text
+}
+
 let _engine = null
 
-export function getEngine(name) {
+function getEngine(name) {
   if (name === 'duckdb') {
     if (!_engine) _engine = new DuckDBEngine()
     return _engine
   }
   throw new Error(`Unknown engine: '${name}'. Available: 'native', 'duckdb'`)
 }
+
+module.exports = { getEngine }