tranfi 0.0.2 → 0.1.2

package/src/pipeline.js

@@ -0,0 +1,286 @@
+/**
+ * pipeline.js — Pipeline builder and runner.
+ *
+ * Tries native N-API addon first, falls back to WASM.
+ */
+
+import { readFile, writeFile } from 'fs/promises'
+import { existsSync } from 'fs'
+import nativeBinding from './native.js'
+
+// Channel IDs (match tranfi.h)
+const CHAN_MAIN = 0
+const CHAN_ERRORS = 1
+const CHAN_STATS = 2
+const CHAN_SAMPLES = 3
+
+const CHUNK_SIZE = 64 * 1024
+
+/**
+ * Get the backend (native or WASM).
+ * Returns an object with: createPipeline, push, finish, pull, free, version
+ */
+let _backend = null
+
+async function getBackend() {
+  if (_backend) return _backend
+
+  // Try native first
+  if (nativeBinding) {
+    _backend = nativeBinding
+    return _backend
+  }
+
+  // Fall back to WASM
+  const { loadWasm } = await import('./wasm.js')
+  const wasm = await loadWasm()
+
+  _backend = {
+    compileDsl(dsl) {
+      const len = wasm.lengthBytesUTF8(dsl)
+      const ptr = wasm._malloc(len + 1)
+      wasm.stringToUTF8(dsl, ptr, len + 1)
+      const resultPtr = wasm.ccall('wasm_compile_dsl', 'number', ['number', 'number'], [ptr, len])
+      wasm._free(ptr)
+      if (!resultPtr) {
+        const err = wasm.ccall('wasm_pipeline_error', 'string', ['number'], [-1])
+        throw new Error(`DSL parse failed: ${err || 'unknown error'}`)
+      }
+      const json = wasm.UTF8ToString(resultPtr)
+      wasm._free(resultPtr)
+      return json
+    },
+    createPipeline(planJson) {
+      const len = wasm.lengthBytesUTF8(planJson)
+      const ptr = wasm._malloc(len + 1)
+      wasm.stringToUTF8(planJson, ptr, len + 1)
+      const handle = wasm.ccall('wasm_pipeline_create', 'number', ['number', 'number'], [ptr, len])
+      wasm._free(ptr)
+      if (handle < 0) {
+        const err = wasm.ccall('wasm_pipeline_error', 'string', ['number'], [handle])
+        throw new Error(`Failed to create pipeline: ${err || 'unknown error'}`)
+      }
+      return handle
+    },
+    push(handle, buffer) {
+      const len = buffer.length
+      const ptr = wasm._malloc(len)
+      wasm.HEAPU8.set(buffer, ptr)
+      const rc = wasm.ccall('wasm_pipeline_push', 'number', ['number', 'number', 'number'], [handle, ptr, len])
+      wasm._free(ptr)
+      if (rc !== 0) {
+        const err = wasm.ccall('wasm_pipeline_error', 'string', ['number'], [handle])
+        throw new Error(`Push failed: ${err || 'unknown error'}`)
+      }
+    },
+    finish(handle) {
+      const rc = wasm.ccall('wasm_pipeline_finish', 'number', ['number'], [handle])
+      if (rc !== 0) {
+        const err = wasm.ccall('wasm_pipeline_error', 'string', ['number'], [handle])
+        throw new Error(`Finish failed: ${err || 'unknown error'}`)
+      }
+    },
+    pull(handle, channel) {
+      const bufSize = 65536
+      const ptr = wasm._malloc(bufSize)
+      const chunks = []
+      for (;;) {
+        const n = wasm.ccall('wasm_pipeline_pull', 'number',
+          ['number', 'number', 'number', 'number'], [handle, channel, ptr, bufSize])
+        if (n === 0) break
+        chunks.push(new Uint8Array(wasm.HEAPU8.buffer, ptr, n).slice())
+      }
+      wasm._free(ptr)
+      // Concatenate
+      const total = chunks.reduce((s, c) => s + c.length, 0)
+      const result = Buffer.alloc(total)
+      let offset = 0
+      for (const c of chunks) {
+        result.set(c, offset)
+        offset += c.length
+      }
+      return result
+    },
+    free(handle) {
+      wasm.ccall('wasm_pipeline_free', null, ['number'], [handle])
+    },
+    version() {
+      return wasm.ccall('wasm_version', 'string', [], [])
+    },
+    recipeCount() {
+      return wasm.ccall('wasm_recipe_count', 'number', [], [])
+    },
+    recipeName(index) {
+      return wasm.ccall('wasm_recipe_name', 'string', ['number'], [index])
+    },
+    recipeDsl(index) {
+      return wasm.ccall('wasm_recipe_dsl', 'string', ['number'], [index])
+    },
+    recipeDescription(index) {
+      return wasm.ccall('wasm_recipe_description', 'string', ['number'], [index])
+    },
+    recipeFindDsl(name) {
+      const len = wasm.lengthBytesUTF8(name)
+      const ptr = wasm._malloc(len + 1)
+      wasm.stringToUTF8(name, ptr, len + 1)
+      const result = wasm.ccall('wasm_recipe_find_dsl', 'string', ['number'], [ptr])
+      wasm._free(ptr)
+      return result || null
+    },
+    compileToSql(dsl) {
+      const len = wasm.lengthBytesUTF8(dsl)
+      const ptr = wasm._malloc(len + 1)
+      wasm.stringToUTF8(dsl, ptr, len + 1)
+      const resultPtr = wasm.ccall('wasm_compile_to_sql', 'number', ['number', 'number'], [ptr, len])
+      wasm._free(ptr)
+      if (!resultPtr) {
+        const err = wasm.ccall('wasm_pipeline_error', 'string', ['number'], [-1])
+        throw new Error(`SQL compile failed: ${err || 'unknown error'}`)
+      }
+      const sql = wasm.UTF8ToString(resultPtr)
+      wasm._free(resultPtr)
+      return sql
+    }
+  }
+
+  return _backend
+}
+
+
+export class PipelineResult {
+  constructor(output, errors, stats, samples) {
+    this.output = output
+    this.errors = errors
+    this.stats = stats
+    this.samples = samples
+  }
+
+  get outputText() {
+    return this.output.toString('utf-8')
+  }
+
+  get statsText() {
+    return this.stats.toString('utf-8')
+  }
+
+  toString() {
+    return `PipelineResult(output=${this.output.length} bytes, errors=${this.errors.length} bytes, stats=${this.stats.length} bytes)`
+  }
+}
+
+
+export class Pipeline {
+  constructor(steps, { planJson, engine } = {}) {
+    this._engine = engine || null
+    if (planJson) {
+      this._planJson = planJson
+      this._dsl = null
+      this._steps = null
+    } else if (typeof steps === 'string') {
+      this._dsl = steps
+      this._steps = null
+      this._planJson = null
+    } else {
+      this._dsl = null
+      this._steps = steps
+      this._planJson = null
+    }
+  }
+
+  async _toPlanJson(backend) {
+    if (this._planJson) return this._planJson
+    if (this._dsl) {
+      // Check if it's a built-in recipe name (no pipes, no braces)
+      const s = this._dsl.trim()
+      if (!s.includes('|') && !s.startsWith('{')) {
+        const recipeDsl = backend.recipeFindDsl(s)
+        if (recipeDsl) return backend.compileDsl(recipeDsl)
+      }
+      return backend.compileDsl(this._dsl)
+    }
+    return JSON.stringify({ steps: this._steps })
+  }
+
+  async run({ input, inputFile } = {}) {
+    if (this._engine && this._engine !== 'native') {
+      return this._runEngine({ input, inputFile })
+    }
+
+    const backend = await getBackend()
+    const planJson = await this._toPlanJson(backend)
+    const handle = backend.createPipeline(planJson)
+
+    try {
+      if (inputFile) {
+        const data = await readFile(inputFile)
+        // Push in chunks
+        for (let i = 0; i < data.length; i += CHUNK_SIZE) {
+          const chunk = data.subarray(i, Math.min(i + CHUNK_SIZE, data.length))
+          backend.push(handle, chunk)
+        }
+      } else if (input) {
+        const buf = typeof input === 'string' ? Buffer.from(input, 'utf-8') : input
+        backend.push(handle, buf)
+      }
+
+      backend.finish(handle)
+
+      const output = backend.pull(handle, CHAN_MAIN)
+      const errors = backend.pull(handle, CHAN_ERRORS)
+      const stats = backend.pull(handle, CHAN_STATS)
+      const samples = backend.pull(handle, CHAN_SAMPLES)
+
+      return new PipelineResult(output, errors, stats, samples)
+    } finally {
+      backend.free(handle)
+    }
+  }
+
+  async _runEngine({ input, inputFile } = {}) {
+    const { getEngine } = await import('./engines/duckdb.js')
+    const engine = getEngine(this._engine)
+    const dsl = this._dsl
+    if (!dsl) throw new Error('DuckDB engine requires a DSL string pipeline')
+    return engine.run(dsl, { input, inputFile })
+  }
+}
+
+export async function compileDsl(dsl) {
+  const backend = await getBackend()
+  return backend.compileDsl(dsl)
+}
+
+export async function compileToSql(dsl) {
+  const backend = await getBackend()
+  return backend.compileToSql(dsl)
+}
+
+export async function loadRecipe(source) {
+  let planJson
+  if (typeof source === 'object' && source.steps) {
+    planJson = JSON.stringify(source)
+  } else if (typeof source === 'string' && source.trimStart().startsWith('{')) {
+    planJson = source
+  } else {
+    planJson = await readFile(source, 'utf-8')
+  }
+  return new Pipeline(null, { planJson })
+}
+
+export async function saveRecipe(steps, path) {
+  await writeFile(path, JSON.stringify({ steps }))
+}
+
+export async function recipes() {
+  const backend = await getBackend()
+  const n = backend.recipeCount()
+  const result = []
+  for (let i = 0; i < n; i++) {
+    result.push({
+      name: backend.recipeName(i),
+      dsl: backend.recipeDsl(i),
+      description: backend.recipeDescription(i)
+    })
+  }
+  return result
+}

package/src/server.js

@@ -0,0 +1,277 @@
+/**
+ * server.js — HTTP server for tranfi serve mode.
+ *
+ * Serves the app frontend and provides API endpoints backed by
+ * the native C core (N-API). No external dependencies — uses
+ * built-in http, fs, path, url modules only.
+ */
+
+import { createServer } from 'http'
+import { readFileSync, readdirSync, statSync, readFile as readFileCb, existsSync } from 'fs'
+import { join, extname, resolve, basename } from 'path'
+import { URL } from 'url'
+import { pipeline, recipes } from './index.js'
+import nativeBinding from './native.js'
+
+const MIME_TYPES = {
+  '.html': 'text/html; charset=utf-8',
+  '.js': 'application/javascript; charset=utf-8',
+  '.mjs': 'application/javascript; charset=utf-8',
+  '.css': 'text/css; charset=utf-8',
+  '.json': 'application/json; charset=utf-8',
+  '.wasm': 'application/wasm',
+  '.svg': 'image/svg+xml',
+  '.png': 'image/png',
+  '.jpg': 'image/jpeg',
+  '.ico': 'image/x-icon',
+  '.woff': 'font/woff',
+  '.woff2': 'font/woff2',
+  '.ttf': 'font/ttf',
+  '.eot': 'application/vnd.ms-fontobject',
+  '.map': 'application/json'
+}
+
+const DATA_EXTENSIONS = new Set(['.csv', '.jsonl', '.tsv', '.txt', '.json'])
+
+function esc (s) {
+  return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;')
+}
+
+function csvToHtml (csv, label) {
+  if (!csv || !csv.trim()) return ''
+  const lines = csv.trim().split('\n')
+  const header = lines[0].split(',')
+  const body = lines.slice(1)
+  const maxRows = 500
+  const truncated = body.length > maxRows
+  const displayBody = truncated ? body.slice(0, maxRows) : body
+
+  let h = `<div style="font-size:12px;color:#888;margin-bottom:6px">`
+  h += `${esc(label)} (${body.length} rows${truncated ? ', showing first ' + maxRows : ''})</div>`
+  h += '<div style="overflow-x:auto;max-height:500px;overflow-y:auto">'
+  h += '<table style="width:100%;border-collapse:collapse;font-size:13px;font-family:monospace">'
+  h += '<thead><tr>'
+  for (const col of header) {
+    h += '<th style="padding:6px 12px;text-align:left;border-bottom:2px solid #ddd;'
+    h += 'background:#f8f8f8;font-weight:600;position:sticky;top:0">'
+    h += esc(col.trim()) + '</th>'
+  }
+  h += '</tr></thead><tbody>'
+  for (const row of displayBody) {
+    h += '<tr>'
+    for (const cell of row.split(',')) {
+      h += '<td style="padding:4px 12px;text-align:left;border-bottom:1px solid #eee">'
+      h += esc(cell.trim()) + '</td>'
+    }
+    h += '</tr>'
+  }
+  h += '</tbody></table></div>'
+  return h
+}
+
+function jsonBody (req) {
+  return new Promise((resolve, reject) => {
+    const chunks = []
+    req.on('data', c => chunks.push(c))
+    req.on('end', () => {
+      try {
+        resolve(JSON.parse(Buffer.concat(chunks).toString()))
+      } catch (e) {
+        reject(new Error('Invalid JSON body'))
+      }
+    })
+    req.on('error', reject)
+  })
+}
+
+function sendJson (res, data, status = 200) {
+  const body = JSON.stringify(data)
+  res.writeHead(status, {
+    'Content-Type': 'application/json; charset=utf-8',
+    'Access-Control-Allow-Origin': '*'
+  })
+  res.end(body)
+}
+
+function sendError (res, msg, status = 400) {
+  sendJson(res, { error: msg }, status)
+}
+
+function isSafeName (name) {
+  return name && !name.includes('..') && !name.includes('/') && !name.includes('\\')
+}
+
+export function startServer ({ port = 3000, dataDir, appDir }) {
+  dataDir = resolve(dataDir)
+  appDir = resolve(appDir)
+
+  if (!existsSync(appDir)) {
+    throw new Error(`App directory not found: ${appDir}`)
+  }
+  if (!existsSync(dataDir)) {
+    throw new Error(`Data directory not found: ${dataDir}`)
+  }
+
+  // Read and patch index.html once
+  const indexPath = join(appDir, 'index.html')
+  let indexHtml = ''
+  if (existsSync(indexPath)) {
+    indexHtml = readFileSync(indexPath, 'utf-8')
+    const config = `<script>window.__TRANFI_SERVER__={api:'/api'}</script>`
+    indexHtml = indexHtml.replace('</head>', config + '</head>')
+  }
+
+  const server = createServer(async (req, res) => {
+    const url = new URL(req.url, `http://${req.headers.host}`)
+    const pathname = url.pathname
+
+    // CORS preflight
+    if (req.method === 'OPTIONS') {
+      res.writeHead(204, {
+        'Access-Control-Allow-Origin': '*',
+        'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
+        'Access-Control-Allow-Headers': 'Content-Type'
+      })
+      res.end()
+      return
+    }
+
+    try {
+      // API routes
+      if (pathname === '/api/files' && req.method === 'GET') {
+        return handleFiles(res, dataDir)
+      }
+      if (pathname === '/api/file' && req.method === 'GET') {
+        const name = url.searchParams.get('name')
+        const head = parseInt(url.searchParams.get('head') || '20', 10)
+        return handleFilePreview(res, dataDir, name, head)
+      }
+      if (pathname === '/api/run' && req.method === 'POST') {
+        const body = await jsonBody(req)
+        return await handleRun(res, dataDir, body)
+      }
+      if (pathname === '/api/recipes' && req.method === 'GET') {
+        return await handleRecipes(res)
+      }
+      if (pathname === '/api/version' && req.method === 'GET') {
+        const v = nativeBinding ? nativeBinding.version() : 'unknown'
+        return sendJson(res, { version: v })
+      }
+
+      // Static files
+      serveStatic(res, appDir, pathname, indexHtml)
+    } catch (err) {
+      sendError(res, err.message, 500)
+    }
+  })
+
+  server.listen(port, () => {
+    const v = nativeBinding ? nativeBinding.version() : 'unknown'
+    process.stderr.write(`tranfi ${v} server\n`)
+    process.stderr.write(`  App:  ${appDir}\n`)
+    process.stderr.write(`  Data: ${dataDir}\n`)
+    process.stderr.write(`  http://localhost:${port}\n`)
+  })
+
+  return server
+}
+
+function handleFiles (res, dataDir) {
+  const entries = readdirSync(dataDir)
+  const files = []
+  for (const name of entries) {
+    const ext = extname(name).toLowerCase()
+    if (!DATA_EXTENSIONS.has(ext)) continue
+    try {
+      const st = statSync(join(dataDir, name))
+      if (st.isFile()) {
+        files.push({ name, size: st.size })
+      }
+    } catch {}
+  }
+  files.sort((a, b) => a.name.localeCompare(b.name))
+  sendJson(res, { files })
+}
+
+function handleFilePreview (res, dataDir, name, head) {
+  if (!isSafeName(name)) return sendError(res, 'Invalid file name')
+  const filePath = join(dataDir, name)
+  if (!existsSync(filePath)) return sendError(res, 'File not found', 404)
+
+  const content = readFileSync(filePath, 'utf-8')
+  const lines = content.split('\n')
+  const preview = lines.slice(0, head).join('\n')
+  sendJson(res, { preview, lines: lines.length })
+}
+
+async function handleRun (res, dataDir, body) {
+  const { file, dsl } = body
+  if (!dsl) return sendError(res, 'Missing dsl field')
+  if (!file) return sendError(res, 'Missing file field')
+  if (!isSafeName(file)) return sendError(res, 'Invalid file name')
+
+  const filePath = join(dataDir, file)
+  if (!existsSync(filePath)) return sendError(res, `File not found: ${file}`, 404)
+
+  const p = pipeline(dsl)
+  const result = await p.run({ inputFile: filePath })
+
+  const output = result.outputText
+  const stats = result.statsText
+
+  sendJson(res, {
+    output: csvToHtml(output, 'Output'),
+    stats: csvToHtml(stats, 'Stats')
+  })
+}
+
+async function handleRecipes (res) {
+  const list = await recipes()
+  sendJson(res, { recipes: list })
+}
+
+function serveStatic (res, appDir, pathname, indexHtml) {
+  // Normalize path
+  let filePath = join(appDir, pathname === '/' ? 'index.html' : pathname)
+
+  // Security: ensure resolved path is within appDir
+  const resolved = resolve(filePath)
+  if (!resolved.startsWith(appDir)) {
+    res.writeHead(403)
+    res.end('Forbidden')
+    return
+  }
+
+  // SPA fallback: if file doesn't exist, serve index.html
+  if (!existsSync(resolved) || statSync(resolved).isDirectory()) {
+    if (indexHtml) {
+      res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' })
+      res.end(indexHtml)
+    } else {
+      res.writeHead(404)
+      res.end('Not found')
+    }
+    return
+  }
+
+  // Serve index.html with injected config
+  if (basename(resolved) === 'index.html' && indexHtml) {
+    res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' })
+    res.end(indexHtml)
+    return
+  }
+
+  // Serve regular file
+  const ext = extname(resolved).toLowerCase()
+  const contentType = MIME_TYPES[ext] || 'application/octet-stream'
+
+  readFileCb(resolved, (err, data) => {
+    if (err) {
+      res.writeHead(500)
+      res.end('Internal error')
+      return
+    }
+    res.writeHead(200, { 'Content-Type': contentType })
+    res.end(data)
+  })
+}

package/src/wasm.js

@@ -0,0 +1,19 @@
+/**
+ * wasm.js — Load the WASM module.
+ * Lazy-initializes on first use.
+ */
+
+let instance = null
+
+export async function loadWasm() {
+  if (instance) return instance
+
+  // Try to load the WASM module
+  const { default: createTranfi } = await import('../wasm/tranfi_core.js')
+  instance = await createTranfi()
+  return instance
+}
+
+export function getWasm() {
+  return instance
+}