tranfi 0.0.2 → 0.1.2

package/src/cli.js

@@ -0,0 +1,193 @@
+#!/usr/bin/env node
+
+/**
+ * cli.js — Tranfi CLI for Node.js.
+ *
+ * Usage:
+ *   tranfi 'csv | filter "age > 25" | csv'  < in.csv
+ *   tranfi -f pipeline.tf < in.csv > out.csv
+ *   tranfi profile < data.csv
+ */
+
+import { pipeline, compileDsl, recipes } from './index.js'
+import { readFileSync, writeFileSync, existsSync } from 'fs'
+import { resolve, dirname } from 'path'
+import { fileURLToPath } from 'url'
+import nativeBinding from './native.js'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+
+function usage() {
+  process.stderr.write(`Usage: tranfi [OPTIONS] PIPELINE
+       tranfi [OPTIONS] -f FILE
+
+Streaming ETL with a pipe-style DSL.
+
+Examples:
+  tranfi 'csv | csv'                           # passthrough
+  tranfi 'csv | filter "age > 25" | csv'       # filter rows
+  tranfi 'csv | select name,age | csv'         # select columns
+  tranfi 'csv | head 10 | csv'                 # first N rows
+  tranfi 'csv | sort age | csv'                # sort by column
+  tranfi 'csv | stats | jsonl'                 # aggregate stats
+  tranfi profile                               # built-in recipe
+
+Options:
+  -f FILE   Read pipeline from file
+  -i FILE   Read input from file instead of stdin
+  -o FILE   Write output to file instead of stdout
+  -j        Compile only, output JSON plan
+  -q        Quiet mode (suppress stats)
+  -v        Show version
+  -R        List built-in recipes
+  -h        Show this help
+`)
+}
+
+function findAppDir () {
+  // Try known locations relative to this file
+  const candidates = [
+    resolve(__dirname, '../../app/dist'),          // dev: js/src/ → ../../app/dist
+    resolve(__dirname, '../app'),                 // npm package: src/ → ../app/
+  ]
+  for (const dir of candidates) {
+    if (existsSync(resolve(dir, 'index.html'))) return dir
+  }
+  return null
+}
+
+async function serveCommand (argv) {
+  const { startServer } = await import('./server.js')
+  let dataDir = '.'
+  let appDir = null
+  let port = 3000
+
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i]
+    if (arg === '-d' || arg === '--data') dataDir = argv[++i]
+    else if (arg === '-a' || arg === '--app') appDir = argv[++i]
+    else if (arg === '-p' || arg === '--port') port = parseInt(argv[++i], 10)
+    else if (arg === '-h' || arg === '--help') {
+      process.stderr.write(`Usage: tranfi serve [OPTIONS]
+
+Serve the tranfi app with a native backend.
+
+Options:
+  -d, --data DIR   Data directory (default: .)
+  -a, --app DIR    App dist directory (auto-detect)
+  -p, --port PORT  Port (default: 3000)
+  -h, --help       Show this help
+`)
+      process.exit(0)
+    }
+  }
+
+  if (!appDir) appDir = findAppDir()
+  if (!appDir) {
+    process.stderr.write('error: app directory not found. Use --app to specify it.\n')
+    process.exit(1)
+  }
+
+  startServer({ port, dataDir: resolve(dataDir), appDir: resolve(appDir) })
+}
+
+async function main() {
+  const argv = process.argv.slice(2)
+
+  // Check for serve subcommand
+  if (argv[0] === 'serve') {
+    return serveCommand(argv.slice(1))
+  }
+
+  let pipelineFile = null
+  let pipelineText = null
+  let inputFile = null
+  let outputFile = null
+  let jsonMode = false
+  let quiet = false
+
+  // Parse args
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i]
+    if (arg === '-h' || arg === '--help') { usage(); process.exit(0) }
+    else if (arg === '-v' || arg === '--version') {
+      const v = nativeBinding ? nativeBinding.version() : 'unknown'
+      process.stdout.write(`tranfi ${v}\n`)
+      process.exit(0)
+    }
+    else if (arg === '-R' || arg === '--recipes') {
+      const list = await recipes()
+      process.stdout.write(`Built-in recipes (${list.length}):\n\n`)
+      for (const r of list) {
+        process.stdout.write(`  ${r.name.padEnd(12)} ${r.description}\n`)
+        process.stdout.write(`  ${''.padEnd(12)} ${r.dsl}\n\n`)
+      }
+      process.exit(0)
+    }
+    else if (arg === '-j') jsonMode = true
+    else if (arg === '-q') quiet = true
+    else if (arg === '-f') { pipelineFile = argv[++i] }
+    else if (arg === '-i') { inputFile = argv[++i] }
+    else if (arg === '-o') { outputFile = argv[++i] }
+    else if (!arg.startsWith('-')) { pipelineText = arg }
+    else {
+      process.stderr.write(`error: unknown option '${arg}'\n`)
+      process.exit(1)
+    }
+  }
+
+  // Get pipeline text
+  if (pipelineFile) {
+    pipelineText = readFileSync(pipelineFile, 'utf-8')
+  }
+  if (!pipelineText) {
+    process.stderr.write('error: no pipeline specified\n\n')
+    usage()
+    process.exit(1)
+  }
+
+  // JSON mode
+  if (jsonMode) {
+    const json = await compileDsl(pipelineText)
+    process.stdout.write(json + '\n')
+    process.exit(0)
+  }
+
+  // Create and run pipeline
+  const p = pipeline(pipelineText)
+  const input = inputFile ? undefined : await readStdin()
+  const result = await p.run({
+    inputFile: inputFile || undefined,
+    input
+  })
+
+  // Write output
+  if (outputFile) {
+    writeFileSync(outputFile, result.output)
+  } else {
+    process.stdout.write(result.output)
+  }
+
+  // Errors to stderr
+  if (result.errors.length > 0) {
+    process.stderr.write(result.errors)
+  }
+
+  // Stats to stderr (unless quiet)
+  if (!quiet && result.stats.length > 0) {
+    process.stderr.write(result.stats)
+  }
+}
+
+function readStdin() {
+  return new Promise((resolve) => {
+    const chunks = []
+    process.stdin.on('data', chunk => chunks.push(chunk))
+    process.stdin.on('end', () => resolve(Buffer.concat(chunks)))
+  })
+}
+
+main().catch(err => {
+  process.stderr.write(`error: ${err.message}\n`)
+  process.exit(1)
+})

package/src/engines/duckdb.js

@@ -0,0 +1,109 @@
+/**
+ * duckdb.js — DuckDB engine for Node.js.
+ *
+ * Executes tranfi pipelines via SQL transpilation + duckdb.
+ * Install: npm install duckdb
+ */
+
+import { writeFile, unlink } from 'fs/promises'
+import { tmpdir } from 'os'
+import { join } from 'path'
+import { randomBytes } from 'crypto'
+import { PipelineResult } from '../pipeline.js'
+
+let _compileToSql = null
+
+async function getCompileToSql() {
+  if (_compileToSql) return _compileToSql
+  const { compileToSql } = await import('../pipeline.js')
+  _compileToSql = compileToSql
+  return _compileToSql
+}
+
+class DuckDBEngine {
+  async run(dsl, { input, inputFile } = {}) {
+    let duckdb
+    try {
+      duckdb = (await import('duckdb')).default || await import('duckdb')
+    } catch {
+      throw new Error(
+        "DuckDB engine requires the 'duckdb' package. " +
+        'Install it with: npm install duckdb'
+      )
+    }
+
+    const compileToSql = await getCompileToSql()
+    let sql = await compileToSql(dsl)
+
+    const db = new duckdb.Database(':memory:')
+    const conn = db.connect()
+
+    let tmpPath = null
+    try {
+      if (inputFile) {
+        sql = sql.replaceAll('input_data', `read_csv('${inputFile}')`)
+      } else if (input) {
+        const buf = typeof input === 'string' ? Buffer.from(input, 'utf-8') : input
+        tmpPath = join(tmpdir(), `tranfi_${randomBytes(8).toString('hex')}.csv`)
+        await writeFile(tmpPath, buf)
+        sql = sql.replaceAll('input_data', `read_csv('${tmpPath}')`)
+      } else {
+        throw new Error('Either input or inputFile must be provided')
+      }
+
+      const rows = await queryAll(conn, sql)
+      const output = rowsToCsv(rows)
+
+      return new PipelineResult(
+        Buffer.from(output, 'utf-8'),
+        Buffer.alloc(0),
+        Buffer.alloc(0),
+        Buffer.alloc(0)
+      )
+    } finally {
+      conn.close()
+      db.close()
+      if (tmpPath) {
+        await unlink(tmpPath).catch(() => {})
+      }
+    }
+  }
+}
+
+function queryAll(conn, sql) {
+  return new Promise((resolve, reject) => {
+    conn.all(sql, (err, rows) => {
+      if (err) reject(err)
+      else resolve(rows)
+    })
+  })
+}
+
+function rowsToCsv(rows) {
+  if (!rows || rows.length === 0) return ''
+  const cols = Object.keys(rows[0])
+  const lines = [cols.join(',')]
+  for (const row of rows) {
+    const vals = cols.map(c => {
+      const v = row[c]
+      if (v === null || v === undefined) return ''
+      const s = String(v)
+      if (s.includes(',') || s.includes('"') || s.includes('\n')) {
+        return '"' + s.replace(/"/g, '""') + '"'
+      }
+      return s
+    })
+    lines.push(vals.join(','))
+  }
+  return lines.join('\n') + '\n'
+}
+
+let _engine = null
+
+export function getEngine(name) {
+  if (name === 'duckdb') {
+    if (!_engine) _engine = new DuckDBEngine()
+    return _engine
+  }
+  throw new Error(`Unknown engine: '${name}'. Available: 'native', 'duckdb'`)
+}

package/src/index.js

@@ -0,0 +1,306 @@
+/**
+ * tranfi — Streaming ETL language + runtime.
+ *
+ * Usage:
+ *   import { pipeline, codec, ops, expr } from 'tranfi'
+ *
+ *   const p = pipeline([
+ *     codec.csv({ delimiter: ',' }),
+ *     ops.filter(expr("col('age') > 25")),
+ *     ops.select(['name', 'age']),
+ *     codec.csvEncode(),
+ *   ])
+ *
+ *   const result = await p.run({ inputFile: './data.csv' })
+ *   console.log(result.outputText)
+ *
+ *   // Or use DSL string:
+ *   const p2 = pipeline('csv | filter "col(\'age\') > 25" | csv')
+ *   const result2 = await p2.run({ inputFile: './data.csv' })
+ */
+
+import { Pipeline, PipelineResult, compileDsl, compileToSql, loadRecipe, saveRecipe, recipes } from './pipeline.js'
+
+export { Pipeline, PipelineResult, compileDsl, compileToSql, loadRecipe, saveRecipe, recipes }
+
+export function pipeline(steps, { engine } = {}) {
+  return new Pipeline(steps, { engine })
+}
+
+export function param(name, defaultValue) {
+  const result = { param: name }
+  if (defaultValue !== undefined) result.default = defaultValue
+  return result
+}
+
+export function expr(text) {
+  return text
+}
+
+export const codec = {
+  csv({ delimiter = ',', header = true, batchSize = 1024, repair = false } = {}) {
+    const args = {}
+    if (delimiter !== ',') args.delimiter = delimiter
+    if (!header) args.header = false
+    if (batchSize !== 1024) args.batch_size = batchSize
+    if (repair) args.repair = true
+    return { op: 'codec.csv.decode', args }
+  },
+
+  csvDecode(opts = {}) {
+    return codec.csv(opts)
+  },
+
+  csvEncode({ delimiter = ',' } = {}) {
+    const args = {}
+    if (delimiter !== ',') args.delimiter = delimiter
+    return { op: 'codec.csv.encode', args }
+  },
+
+  jsonl({ batchSize = 1024 } = {}) {
+    const args = {}
+    if (batchSize !== 1024) args.batch_size = batchSize
+    return { op: 'codec.jsonl.decode', args }
+  },
+
+  jsonlDecode(opts = {}) {
+    return codec.jsonl(opts)
+  },
+
+  jsonlEncode() {
+    return { op: 'codec.jsonl.encode', args: {} }
+  },
+
+  text({ batchSize = 1024 } = {}) {
+    const args = {}
+    if (batchSize !== 1024) args.batch_size = batchSize
+    return { op: 'codec.text.decode', args }
+  },
+
+  textDecode(opts = {}) {
+    return codec.text(opts)
+  },
+
+  textEncode() {
+    return { op: 'codec.text.encode', args: {} }
+  },
+
+  tableEncode({ maxWidth = 40, maxRows = 0 } = {}) {
+    const args = {}
+    if (maxWidth !== 40) args.max_width = maxWidth
+    if (maxRows !== 0) args.max_rows = maxRows
+    return { op: 'codec.table.encode', args }
+  }
+}
+
+export const ops = {
+  filter(expression) {
+    return { op: 'filter', args: { expr: expression } }
+  },
+
+  select(columns) {
+    return { op: 'select', args: { columns } }
+  },
+
+  rename(mapping) {
+    return { op: 'rename', args: { mapping } }
+  },
+
+  head(n) {
+    return { op: 'head', args: { n } }
+  },
+
+  skip(n) {
+    return { op: 'skip', args: { n } }
+  },
+
+  derive(columns) {
+    // columns: { name: expr, ... } or [{ name, expr }, ...]
+    const cols = Array.isArray(columns)
+      ? columns
+      : Object.entries(columns).map(([name, e]) => ({ name, expr: e }))
+    return { op: 'derive', args: { columns: cols } }
+  },
+
+  stats(statsList) {
+    const args = {}
+    if (statsList) args.stats = statsList
+    return { op: 'stats', args }
+  },
+
+  unique(columns) {
+    const args = {}
+    if (columns) args.columns = columns
+    return { op: 'unique', args }
+  },
+
+  sort(columns) {
+    // columns: ['age', '-name'] or [{ name, desc }, ...]
+    const cols = columns.map(c => {
+      if (typeof c === 'string') {
+        const desc = c.startsWith('-')
+        return { name: desc ? c.slice(1) : c, desc }
+      }
+      return c
+    })
+    return { op: 'sort', args: { columns: cols } }
+  },
+
+  tail(n) {
+    return { op: 'tail', args: { n } }
+  },
+
+  validate(expression) {
+    return { op: 'validate', args: { expr: expression } }
+  },
+
+  trim(columns) {
+    const args = {}
+    if (columns) args.columns = columns
+    return { op: 'trim', args }
+  },
+
+  fillNull(mapping) {
+    return { op: 'fill-null', args: { mapping } }
+  },
+
+  cast(mapping) {
+    return { op: 'cast', args: { mapping } }
+  },
+
+  clip(column, { min, max } = {}) {
+    const args = { column }
+    if (min !== undefined) args.min = min
+    if (max !== undefined) args.max = max
+    return { op: 'clip', args }
+  },
+
+  replace(column, pattern, replacement, { regex = false } = {}) {
+    const args = { column, pattern, replacement }
+    if (regex) args.regex = true
+    return { op: 'replace', args }
+  },
+
+  hash(columns) {
+    const args = {}
+    if (columns) args.columns = columns
+    return { op: 'hash', args }
+  },
+
+  bin(column, boundaries) {
+    return { op: 'bin', args: { column, boundaries } }
+  },
+
+  fillDown(columns) {
+    const args = {}
+    if (columns) args.columns = columns
+    return { op: 'fill-down', args }
+  },
+
+  step(column, func, result) {
+    const args = { column, func }
+    if (result) args.result = result
+    return { op: 'step', args }
+  },
+
+  window(column, size, func, result) {
+    const args = { column, size, func }
+    if (result) args.result = result
+    return { op: 'window', args }
+  },
+
+  explode(column, delimiter) {
+    const args = { column }
+    if (delimiter && delimiter !== ',') args.delimiter = delimiter
+    return { op: 'explode', args }
+  },
+
+  split(column, names, delimiter) {
+    const args = { column, names }
+    if (delimiter && delimiter !== ' ') args.delimiter = delimiter
+    return { op: 'split', args }
+  },
+
+  unpivot(columns) {
+    return { op: 'unpivot', args: { columns } }
+  },
+
+  top(n, column, desc = true) {
+    return { op: 'top', args: { n, column, desc } }
+  },
+
+  sample(n) {
+    return { op: 'sample', args: { n } }
+  },
+
+  groupAgg(groupBy, aggs) {
+    return { op: 'group-agg', args: { group_by: groupBy, aggs } }
+  },
+
+  frequency(columns) {
+    const args = {}
+    if (columns) args.columns = columns
+    return { op: 'frequency', args }
+  },
+
+  datetime(column, extract) {
+    const args = { column }
+    if (extract) args.extract = extract
+    return { op: 'datetime', args }
+  },
+
+  reorder(columns) {
+    return { op: 'reorder', args: { columns } }
+  },
+
+  dedup(columns) {
+    const args = {}
+    if (columns) args.columns = columns
+    return { op: 'dedup', args }
+  },
+
+  grep(pattern, { invert = false, column = '_line', regex = false } = {}) {
+    const args = { pattern }
+    if (invert) args.invert = true
+    if (column !== '_line') args.column = column
+    if (regex) args.regex = true
+    return { op: 'grep', args }
+  },
+
+  flatten() {
+    return { op: 'flatten', args: {} }
+  },
+
+  stack(file, { tag, tagValue } = {}) {
+    const args = { file }
+    if (tag) args.tag = tag
+    if (tagValue) args.tag_value = tagValue
+    return { op: 'stack', args }
+  },
+
+  lead(column, { offset = 1, result } = {}) {
+    const args = { column }
+    if (offset !== 1) args.offset = offset
+    if (result) args.result = result
+    return { op: 'lead', args }
+  },
+
+  dateTrunc(column, trunc, { result } = {}) {
+    const args = { column, trunc }
+    if (result) args.result = result
+    return { op: 'date-trunc', args }
+  }
+}
+
+export const io = {
+  read: {
+    file(path) {
+      return { op: 'io.read.file', args: { path } }
+    }
+  },
+  write: {
+    stdout() {
+      return { op: 'io.write.stdout', args: {} }
+    }
+  }
+}

package/src/native.js

@@ -0,0 +1,22 @@
+/**
+ * native.js — Load the N-API native addon.
+ * Returns the binding object or null if not available.
+ */
+
+import { createRequire } from 'module'
+
+let binding = null
+
+try {
+  const require = createRequire(import.meta.url)
+  binding = require('../build/Release/tranfi_napi.node')
+} catch {
+  try {
+    const require = createRequire(import.meta.url)
+    binding = require('../build/Debug/tranfi_napi.node')
+  } catch {
+    binding = null
+  }
+}
+
+export default binding