tranfi 0.1.2 → 0.2.0

package/napi_transform.h

@@ -0,0 +1,8 @@
+#ifndef TRANFI_NAPI_TRANSFORM_H
+#define TRANFI_NAPI_TRANSFORM_H
+
+#include <node_api.h>
+
+napi_status tranfi_napi_define_transform(napi_env env, napi_value exports);
+
+#endif

package/package.json

@@ -1,15 +1,25 @@
 {
   "name": "tranfi",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "Streaming ETL language + runtime",
-  "type": "module",
+  "type": "commonjs",
   "main": "src/index.js",
   "bin": {
     "tranfi": "src/cli.js"
   },
   "exports": {
-    ".": "./src/index.js",
-    "./wasm": "./wasm/index.js"
+    ".": {
+      "require": "./src/index.js",
+      "default": "./src/index.js"
+    },
+    "./wasm": {
+      "require": "./wasm/index.js",
+      "default": "./wasm/index.js"
+    },
+    "./wasm/worker": {
+      "require": "./wasm/worker.js",
+      "default": "./wasm/worker.js"
+    }
   },
   "files": [
     "src/",
@@ -17,14 +27,19 @@
     "csrc/",
     "app/",
     "napi_api.c",
+    "napi_transform.c",
+    "napi_transform.h",
     "binding.gyp",
-    "README.md"
+    "scripts/",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
   ],
   "scripts": {
-    "prepack": "rm -rf csrc && mkdir -p csrc && cp ../src/*.c ../src/*.h csrc/ && rm -rf app && cp -r ../app/dist app && rm -rf app/wasm app/lib",
-    "install": "node-gyp rebuild 2>/dev/null || true",
-    "build:native": "rm -rf csrc && mkdir -p csrc && cp ../src/*.c ../src/*.h csrc/ && node-gyp rebuild",
-    "test": "node ../test/test_node.js"
+    "prepack": "node scripts/prepack.js",
+    "install": "node scripts/install-native.js",
+    "build:native": "node scripts/sync-csrc.js && node-gyp rebuild",
+    "test": "node --test ../test/test_install_native.js && node ../test/test_node.js && node ../test/test_transform_node.js && node ../test/test_transform_wasm_node.js && node ../test/test_transform_worker_node.js"
   },
   "peerDependencies": {
     "duckdb": ">=1.0.0",
@@ -46,12 +61,13 @@
     "pipeline",
     "transform"
   ],
-  "license": "MIT",
+  "license": "Apache-2.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/tranfi/tranfi"
   },
   "devDependencies": {
     "duckdb": "^1.4.4"
-  }
+  },
+  "author": "Anton Zemlyansky"
 }

package/scripts/install-native.js

@@ -0,0 +1,76 @@
+'use strict'
+
+const { spawnSync } = require('child_process')
+const path = require('path')
+
+function describeFailure(label, result, logger) {
+  if (result && result.error) {
+    logger.error(`tranfi: ${label} failed: ${result.error.message}`)
+  } else if (result && result.signal) {
+    logger.error(`tranfi: ${label} terminated by signal ${result.signal}`)
+  } else {
+    logger.error(`tranfi: ${label} exited with status ${result ? result.status : 'unknown'}`)
+  }
+}
+
+function successful(result) {
+  return Boolean(result) && result.error === undefined && result.signal === null
+    && result.status === 0
+}
+
+function installNative({
+  env = process.env,
+  execPath = process.execPath,
+  platform = process.platform,
+  spawn = spawnSync,
+  logger = console,
+  packageRoot = path.resolve(__dirname, '..')
+} = {}) {
+  if (env.TRANFI_SKIP_NATIVE_BUILD === '1') {
+    logger.warn(
+      'tranfi: skipping the native addon because TRANFI_SKIP_NATIVE_BUILD=1; ' +
+      "use require('tranfi/wasm') explicitly because prepared transforms on " +
+      "require('tranfi') need the native addon"
+    )
+    return 0
+  }
+
+  const syncResult = spawn(execPath, [path.join(__dirname, 'sync-csrc.js')], {
+    cwd: packageRoot,
+    stdio: 'inherit'
+  })
+  if (!successful(syncResult)) {
+    describeFailure('C-source synchronization', syncResult, logger)
+    return Number.isInteger(syncResult && syncResult.status) && syncResult.status !== 0
+      ? syncResult.status
+      : 1
+  }
+
+  const nodeGyp = env.npm_config_node_gyp || 'node-gyp'
+  const nodeGypIsScript = /\.[cm]?js$/i.test(nodeGyp)
+  const buildCommand = nodeGypIsScript ? execPath : nodeGyp
+  const buildArgs = nodeGypIsScript ? [nodeGyp, 'rebuild'] : ['rebuild']
+  const buildResult = spawn(buildCommand, buildArgs, {
+    cwd: packageRoot,
+    stdio: 'inherit',
+    shell: platform === 'win32' && !nodeGypIsScript
+  })
+  if (!successful(buildResult)) {
+    describeFailure('native addon build', buildResult, logger)
+    logger.error(
+      'tranfi: installation requires a working native toolchain for the default ' +
+      "Node entry; set TRANFI_SKIP_NATIVE_BUILD=1 only for explicit 'tranfi/wasm' use"
+    )
+    return Number.isInteger(buildResult && buildResult.status) && buildResult.status !== 0
+      ? buildResult.status
+      : 1
+  }
+
+  return 0
+}
+
+if (require.main === module) {
+  process.exitCode = installNative()
+}
+
+module.exports = { installNative }

package/scripts/prepack.js

@@ -0,0 +1,64 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+const { spawnSync } = require('child_process')
+
+const scriptsDir = __dirname
+const pkgDir = path.resolve(scriptsDir, '..')
+const repoDir = path.resolve(pkgDir, '..')
+
+function run(cmd, args, cwd, extraEnv = {}) {
+  const result = spawnSync(cmd, args, {
+    cwd,
+    stdio: 'inherit',
+    env: { ...process.env, ...extraEnv }
+  })
+  if (result.status !== 0) {
+    process.exit(result.status || 1)
+  }
+}
+
+run(process.execPath, [path.join(scriptsDir, 'sync-csrc.js')], pkgDir)
+
+const wasmFile = path.join(pkgDir, 'wasm', 'tranfi_core.js')
+if (!fs.existsSync(wasmFile)) {
+  const env = { EM_CACHE: process.env.EM_CACHE || path.join(repoDir, 'build-wasm', '.emcache') }
+  if (!process.env.EMSDK_PYTHON && fs.existsSync('/usr/bin/python3')) {
+    env.EMSDK_PYTHON = '/usr/bin/python3'
+  }
+  run('bash', [path.join(repoDir, 'scripts', 'build-wasm.sh')], repoDir, env)
+}
+
+const appDir = path.join(repoDir, 'app')
+const appPublicWasmDir = path.join(appDir, 'public', 'wasm')
+const appSource = path.join(appDir, 'dist')
+const appDest = path.join(pkgDir, 'app')
+
+fs.mkdirSync(appPublicWasmDir, { recursive: true })
+fs.copyFileSync(wasmFile, path.join(appPublicWasmDir, 'tranfi_core.js'))
+
+if (fs.existsSync(path.join(appDir, 'package.json'))) {
+  run('npm', ['run', 'build'], appDir)
+}
+
+if (fs.existsSync(appSource)) {
+  fs.rmSync(appDest, { recursive: true, force: true })
+  fs.cpSync(appSource, appDest, { recursive: true })
+  for (const extra of ['wasm', 'lib']) {
+    fs.rmSync(path.join(appDest, extra), { recursive: true, force: true })
+  }
+}
+
+const required = [
+  path.join(pkgDir, 'csrc'),
+  path.join(pkgDir, 'wasm', 'tranfi_core.js'),
+  path.join(pkgDir, 'app', 'index.html')
+]
+
+for (const target of required) {
+  if (!fs.existsSync(target)) {
+    console.error('prepack: missing required publish artifact: ' + target)
+    process.exit(1)
+  }
+}

package/scripts/sync-csrc.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const fs = require('fs')
+const path = require('path')
+
+const pkgDir = path.resolve(__dirname, '..')
+const repoDir = path.resolve(pkgDir, '..')
+const srcDir = path.join(repoDir, 'src')
+const outDir = path.join(pkgDir, 'csrc')
+
+if (!fs.existsSync(srcDir)) {
+  console.log('sync-csrc: src/ not found (published package), skipping')
+  process.exit(0)
+}
+
+fs.rmSync(outDir, { recursive: true, force: true })
+fs.mkdirSync(outDir, { recursive: true })
+
+for (const name of fs.readdirSync(srcDir)) {
+  if (name.endsWith('.c') || name.endsWith('.h')) {
+    fs.copyFileSync(path.join(srcDir, name), path.join(outDir, name))
+  }
+}

package/src/cli.js

@@ -9,13 +9,10 @@
  *   tranfi profile < data.csv
  */
 
-import { pipeline, compileDsl, recipes } from './index.js'
-import { readFileSync, writeFileSync, existsSync } from 'fs'
-import { resolve, dirname } from 'path'
-import { fileURLToPath } from 'url'
-import nativeBinding from './native.js'
-
-const __dirname = dirname(fileURLToPath(import.meta.url))
+const { pipeline, compileDsl, recipes } = require('./index.js')
+const { readFileSync, writeFileSync, existsSync } = require('fs')
+const { resolve, dirname } = require('path')
+const nativeBinding = require('./native.js')
 
 function usage() {
   process.stderr.write(`Usage: tranfi [OPTIONS] PIPELINE
@@ -28,7 +25,7 @@ Examples:
   tranfi 'csv | filter "age > 25" | csv'       # filter rows
   tranfi 'csv | select name,age | csv'         # select columns
   tranfi 'csv | head 10 | csv'                 # first N rows
-  tranfi 'csv | sort age | csv'                # sort by column
+  tranfi 'csv | top-k 10 age | csv'            # bounded top rows by column
   tranfi 'csv | stats | jsonl'                 # aggregate stats
   tranfi profile                               # built-in recipe
 
@@ -47,8 +44,8 @@ Options:
 function findAppDir () {
   // Try known locations relative to this file
   const candidates = [
-    resolve(__dirname, '../../app/dist'),          // dev: js/src/ → ../../app/dist
-    resolve(__dirname, '../app'),                 // npm package: src/ → ../app/
+    resolve(__dirname, '../../app/dist'),          // dev: js/src/ -> ../../app/dist
+    resolve(__dirname, '../app'),                 // npm package: src/ -> ../app/
   ]
   for (const dir of candidates) {
     if (existsSync(resolve(dir, 'index.html'))) return dir
@@ -57,7 +54,7 @@ function findAppDir () {
 }
 
 async function serveCommand (argv) {
-  const { startServer } = await import('./server.js')
+  const { startServer } = require('./server.js')
   let dataDir = '.'
   let appDir = null
   let port = 3000

package/src/engines/duckdb.js

@@ -5,26 +5,27 @@
  * Install: npm install duckdb
  */
 
-import { writeFile, unlink } from 'fs/promises'
-import { tmpdir } from 'os'
-import { join } from 'path'
-import { randomBytes } from 'crypto'
-import { PipelineResult } from '../pipeline.js'
+const { writeFile, unlink } = require('fs/promises')
+const { tmpdir } = require('os')
+const { join } = require('path')
+const { randomBytes } = require('crypto')
+const { PipelineResult } = require('../pipeline.js')
+const { memoryForEngine } = require('../memory_policy.js')
 
 let _compileToSql = null
 
 async function getCompileToSql() {
   if (_compileToSql) return _compileToSql
-  const { compileToSql } = await import('../pipeline.js')
+  const { compileToSql } = require('../pipeline.js')
   _compileToSql = compileToSql
   return _compileToSql
 }
 
 class DuckDBEngine {
-  async run(dsl, { input, inputFile } = {}) {
+  async run(dsl, { input, inputFile, memory, spillDir } = {}) {
     let duckdb
     try {
-      duckdb = (await import('duckdb')).default || await import('duckdb')
+      duckdb = require('duckdb')
     } catch {
       throw new Error(
         "DuckDB engine requires the 'duckdb' package. " +
@@ -40,13 +41,17 @@ class DuckDBEngine {
 
     let tmpPath = null
     try {
+      const memoryLimit = memoryForEngine(memory)
+      if (memoryLimit) await runSql(conn, `SET memory_limit = '${sqlString(memoryLimit)}'`)
+      if (spillDir) await runSql(conn, `SET temp_directory = '${sqlString(spillDir)}'`)
+
       if (inputFile) {
-        sql = sql.replaceAll('input_data', `read_csv('${inputFile}')`)
+        sql = sql.replaceAll('input_data', `read_csv('${sqlString(inputFile)}')`)
       } else if (input) {
         const buf = typeof input === 'string' ? Buffer.from(input, 'utf-8') : input
         tmpPath = join(tmpdir(), `tranfi_${randomBytes(8).toString('hex')}.csv`)
         await writeFile(tmpPath, buf)
-        sql = sql.replaceAll('input_data', `read_csv('${tmpPath}')`)
+        sql = sql.replaceAll('input_data', `read_csv('${sqlString(tmpPath)}')`)
       } else {
         throw new Error('Either input or inputFile must be provided')
       }
@@ -70,6 +75,19 @@ class DuckDBEngine {
   }
 }
 
+function sqlString(value) {
+  return String(value).replace(/'/g, "''")
+}
+
+function runSql(conn, sql) {
+  return new Promise((resolve, reject) => {
+    conn.run(sql, err => {
+      if (err) reject(err)
+      else resolve()
+    })
+  })
+}
+
 function queryAll(conn, sql) {
   return new Promise((resolve, reject) => {
     conn.all(sql, (err, rows) => {
@@ -87,7 +105,7 @@ function rowsToCsv(rows) {
     const vals = cols.map(c => {
       const v = row[c]
       if (v === null || v === undefined) return ''
-      const s = String(v)
+      const s = formatDuckValue(v)
       if (s.includes(',') || s.includes('"') || s.includes('\n')) {
         return '"' + s.replace(/"/g, '""') + '"'
       }
@@ -98,12 +116,27 @@ function rowsToCsv(rows) {
   return lines.join('\n') + '\n'
 }
 
+function formatDuckValue(value) {
+  if (typeof value !== 'number') return String(value)
+  if (!Number.isFinite(value)) return String(value)
+  if (Number.isInteger(value) && Math.abs(value) <= Number.MAX_SAFE_INTEGER) {
+    return String(value)
+  }
+  let text = value.toPrecision(17)
+  if (!/[eE]/.test(text) && text.includes('.')) {
+    text = text.replace(/0+$/, '').replace(/\.$/, '')
+  }
+  return text
+}
+
 let _engine = null
 
-export function getEngine(name) {
+function getEngine(name) {
   if (name === 'duckdb') {
     if (!_engine) _engine = new DuckDBEngine()
     return _engine
   }
   throw new Error(`Unknown engine: '${name}'. Available: 'native', 'duckdb'`)
 }
+
+module.exports = { getEngine }