tranfi 0.1.2 → 0.2.0

package/src/pipeline.js

@@ -4,9 +4,13 @@
  * Tries native N-API addon first, falls back to WASM.
  */
 
-import { readFile, writeFile } from 'fs/promises'
-import { existsSync } from 'fs'
-import nativeBinding from './native.js'
+const { readFile, writeFile } = require('fs/promises')
+const { createReadStream } = require('fs')
+const { Readable } = require('stream')
+const { createGunzip } = require('zlib')
+const { finished } = require('stream/promises')
+const nativeBinding = require('./native.js')
+const { prepareNativePlan, validateNativeMemoryPolicy } = require('./memory_policy.js')
 
 // Channel IDs (match tranfi.h)
 const CHAN_MAIN = 0
@@ -16,12 +20,87 @@ const CHAN_SAMPLES = 3
 
 const CHUNK_SIZE = 64 * 1024
 
+const HOSTED_AUDIT_PRIVACY_OPS = new Set([
+  'codec.csv.decode',
+  'filter',
+  'validate',
+  'assert',
+  'schema',
+  'json-schema',
+  'fill-null',
+  'replace',
+  'cast',
+  'normalize',
+  'frequency',
+  'tee',
+  'quarantine'
+])
+
+function normalizeSqlDialect(options) {
+  let dialect = 'duckdb'
+  if (typeof options === 'string') {
+    dialect = options
+  } else if (options && typeof options === 'object') {
+    dialect = options.dialect || options.sqlDialect || dialect
+  } else if (options !== undefined && options !== null) {
+    throw new TypeError('compileToSql options must be an object or dialect string')
+  }
+  if (typeof dialect !== 'string') throw new TypeError('SQL dialect must be a string')
+  const name = dialect.toLowerCase()
+  if (name === 'duckdb') return name
+  if (name === 'sqlite' || name === 'postgres') {
+    throw new Error(`SQL dialect '${name}' is recognized but not implemented yet; only duckdb lowering is available`)
+  }
+  throw new Error(`unknown SQL dialect '${dialect}' (expected duckdb, sqlite, or postgres)`)
+}
+
+function applyHostedAuditDefaults(planJson) {
+  const plan = JSON.parse(planJson)
+  if (!plan || !Array.isArray(plan.steps)) return planJson
+  let changed = false
+  for (const step of plan.steps) {
+    if (!step || typeof step !== 'object' || !HOSTED_AUDIT_PRIVACY_OPS.has(step.op)) continue
+    if (!step.args || typeof step.args !== 'object' || Array.isArray(step.args)) step.args = {}
+    if (!Object.prototype.hasOwnProperty.call(step.args, 'audit_include_row')) {
+      step.args.audit_include_row = false
+      changed = true
+    }
+  }
+  return changed ? JSON.stringify(plan) : planJson
+}
+
 /**
  * Get the backend (native or WASM).
  * Returns an object with: createPipeline, push, finish, pull, free, version
  */
 let _backend = null
 
+function effectiveHostPolicy({ allowFs, allowRulesFile = false, workspaceRoot, spillDir, allowBlocking = true } = {}) {
+  return {
+    allowFs: allowFs === undefined ? Boolean(spillDir) : Boolean(allowFs),
+    allowSpill: Boolean(spillDir),
+    allowRulesFile: Boolean(allowRulesFile),
+    allowBlocking: Boolean(allowBlocking),
+    workspaceRoot
+  }
+}
+
+function planUsesHostPaths(planJson) {
+  try {
+    const plan = JSON.parse(planJson)
+    const steps = Array.isArray(plan.steps) ? plan.steps : []
+    return steps.some((step) => {
+      const args = step && step.args
+      if (!args || typeof args !== 'object') return false
+      return ['file', 'rules_file', 'rulesFile', 'spill_dir', 'spillDir'].some((key) =>
+        typeof args[key] === 'string' && args[key].length > 0
+      )
+    })
+  } catch (_) {
+    return false
+  }
+}
+
 async function getBackend() {
   if (_backend) return _backend
 
@@ -32,10 +111,11 @@ async function getBackend() {
   }
 
   // Fall back to WASM
-  const { loadWasm } = await import('./wasm.js')
+  const { loadWasm } = require('./wasm.js')
   const wasm = await loadWasm()
 
   _backend = {
+    isWasm: true,
     compileDsl(dsl) {
       const len = wasm.lengthBytesUTF8(dsl)
       const ptr = wasm._malloc(len + 1)
@@ -50,12 +130,33 @@ async function getBackend() {
       wasm._free(resultPtr)
       return json
     },
-    createPipeline(planJson) {
+    createPipeline(planJson, options = {}) {
       const len = wasm.lengthBytesUTF8(planJson)
       const ptr = wasm._malloc(len + 1)
       wasm.stringToUTF8(planJson, ptr, len + 1)
-      const handle = wasm.ccall('wasm_pipeline_create', 'number', ['number', 'number'], [ptr, len])
+      let rootPtr = 0
+      if (options.workspaceRoot) {
+        const root = String(options.workspaceRoot)
+        const rootLen = wasm.lengthBytesUTF8(root)
+        rootPtr = wasm._malloc(rootLen + 1)
+        wasm.stringToUTF8(root, rootPtr, rootLen + 1)
+      }
+      const hasPolicyCreate = typeof wasm._wasm_pipeline_create_with_policy === 'function'
+      const needsPolicyCreate = options.allowFs || options.allowSpill || options.allowRulesFile || rootPtr || planUsesHostPaths(planJson)
+      let handle = -1
+      if (hasPolicyCreate) {
+        handle = wasm.ccall('wasm_pipeline_create_with_policy', 'number',
+          ['number', 'number', 'number', 'number', 'number', 'number'],
+          [ptr, len, options.allowFs ? 1 : 0, options.allowSpill ? 1 : 0,
+            options.allowRulesFile ? 1 : 0, rootPtr])
+      } else if (!needsPolicyCreate) {
+        handle = wasm.ccall('wasm_pipeline_create', 'number', ['number', 'number'], [ptr, len])
+      }
       wasm._free(ptr)
+      if (rootPtr) wasm._free(rootPtr)
+      if (!hasPolicyCreate && needsPolicyCreate) {
+        throw new Error('Failed to create pipeline: WASM host policy export unavailable for this plan; rebuild tranfi_core.js')
+      }
       if (handle < 0) {
         const err = wasm.ccall('wasm_pipeline_error', 'string', ['number'], [handle])
         throw new Error(`Failed to create pipeline: ${err || 'unknown error'}`)
@@ -73,6 +174,25 @@ async function getBackend() {
         throw new Error(`Push failed: ${err || 'unknown error'}`)
       }
     },
+    flushInput(handle) {
+      const rc = wasm.ccall('wasm_pipeline_flush_input', 'number', ['number'], [handle])
+      if (rc !== 0) {
+        const err = wasm.ccall('wasm_pipeline_error', 'string', ['number'], [handle])
+        throw new Error(`Input flush failed: ${err || 'unknown error'}`)
+      }
+    },
+    setSourceName(handle, name) {
+      const source = name == null ? '' : String(name)
+      const len = wasm.lengthBytesUTF8(source)
+      const ptr = wasm._malloc(len + 1)
+      wasm.stringToUTF8(source, ptr, len + 1)
+      const rc = wasm.ccall('wasm_pipeline_set_source_name', 'number', ['number', 'number'], [handle, ptr])
+      wasm._free(ptr)
+      if (rc !== 0) {
+        const err = wasm.ccall('wasm_pipeline_error', 'string', ['number'], [handle])
+        throw new Error(`Set source name failed: ${err || 'unknown error'}`)
+      }
+    },
     finish(handle) {
       const rc = wasm.ccall('wasm_pipeline_finish', 'number', ['number'], [handle])
       if (rc !== 0) {
@@ -80,6 +200,14 @@ async function getBackend() {
         throw new Error(`Finish failed: ${err || 'unknown error'}`)
       }
     },
+    finishStep(handle) {
+      const rc = wasm.ccall('wasm_pipeline_finish_step', 'number', ['number'], [handle])
+      if (rc < 0) {
+        const err = wasm.ccall('wasm_pipeline_error', 'string', ['number'], [handle])
+        throw new Error(`Finish failed: ${err || 'unknown error'}`)
+      }
+      return rc === 1
+    },
     pull(handle, channel) {
       const bufSize = 65536
       const ptr = wasm._malloc(bufSize)
@@ -101,6 +229,15 @@ async function getBackend() {
       }
       return result
     },
+    pullChunk(handle, channel, chunkSize = 65536) {
+      if (chunkSize <= 0) throw new Error('chunkSize must be positive')
+      const ptr = wasm._malloc(chunkSize)
+      const n = wasm.ccall('wasm_pipeline_pull', 'number',
+        ['number', 'number', 'number', 'number'], [handle, channel, ptr, chunkSize])
+      const result = Buffer.from(new Uint8Array(wasm.HEAPU8.buffer, ptr, n))
+      wasm._free(ptr)
+      return result
+    },
     free(handle) {
       wasm.ccall('wasm_pipeline_free', null, ['number'], [handle])
     },
@@ -147,7 +284,181 @@ async function getBackend() {
 }
 
 
-export class PipelineResult {
+function asInputBuffer(input) {
+  if (Buffer.isBuffer(input)) return input
+  if (typeof input === 'string') return Buffer.from(input, 'utf-8')
+  if (input instanceof ArrayBuffer) return Buffer.from(new Uint8Array(input))
+  return Buffer.from(input)
+}
+
+function hasInput(value) {
+  return value !== undefined && value !== null
+}
+
+function assertSingleInputSource({ input, inputFile, inputFiles, inputStream }) {
+  const count = (hasInput(input) ? 1 : 0) + (hasInput(inputFile) ? 1 : 0) + (hasInput(inputFiles) ? 1 : 0) + (hasInput(inputStream) ? 1 : 0)
+  if (count > 1) throw new Error('pass only one of input, inputFile, inputFiles, or inputStream')
+}
+
+function normalizeInputFiles(inputFiles) {
+  if (!hasInput(inputFiles)) return null
+  if (typeof inputFiles === 'string' || Buffer.isBuffer(inputFiles)) {
+    throw new TypeError('inputFiles must be an iterable of paths, not a single path')
+  }
+  if (typeof inputFiles[Symbol.iterator] !== 'function') {
+    throw new TypeError('inputFiles must be an iterable of paths')
+  }
+  const files = Array.from(inputFiles)
+  if (files.length === 0) throw new Error('inputFiles must not be empty')
+  return files
+}
+
+function injectSourceNameStep(planJson, sourceColumn) {
+  if (sourceColumn === undefined || sourceColumn === null) return planJson
+  const result = String(sourceColumn)
+  if (!result) throw new Error('sourceColumn must be a non-empty column name')
+  const plan = JSON.parse(planJson)
+  if (!plan || !Array.isArray(plan.steps)) throw new Error('plan must contain a steps array')
+  let insertAt = 0
+  if (plan.steps.length > 0) {
+    const op = plan.steps[0] && plan.steps[0].op
+    if (typeof op === 'string' && op.startsWith('codec.') && op.endsWith('.decode')) insertAt = 1
+  }
+  plan.steps.splice(insertAt, 0, { op: 'source-name', args: { result } })
+  return JSON.stringify(plan)
+}
+
+function normalizeCompression(compression = 'auto') {
+  const mode = compression == null ? 'auto' : String(compression).toLowerCase()
+  if (!['auto', 'none', 'gzip'].includes(mode)) {
+    throw new Error("compression must be 'auto', 'none', or 'gzip'")
+  }
+  return mode
+}
+
+function resolveFileCompression(inputFile, compression) {
+  const mode = normalizeCompression(compression)
+  if (mode === 'auto') {
+    return String(inputFile).toLowerCase().endsWith('.gz') ? 'gzip' : 'none'
+  }
+  return mode
+}
+
+async function *yieldChunkItems(chunk, chunkSize, sourceName) {
+  const buf = asInputBuffer(chunk)
+  for (let i = 0; i < buf.length; i += chunkSize) {
+    yield { chunk: buf.subarray(i, Math.min(i + chunkSize, buf.length)), sourceName }
+  }
+}
+
+async function *yieldFileItems(inputFile, chunkSize, requestedCompression) {
+  const sourceName = String(inputFile)
+  const fileCompression = resolveFileCompression(inputFile, requestedCompression)
+  let stream = createReadStream(inputFile, { highWaterMark: chunkSize })
+  if (fileCompression === 'gzip') stream = stream.pipe(createGunzip())
+  for await (const chunk of stream) {
+    yield * yieldChunkItems(chunk, chunkSize, sourceName)
+  }
+  yield { flushInput: true, sourceName }
+}
+
+async function *iterInputChunks({ input, inputFile, inputFiles, inputStream, chunkSize, compression = 'auto' }) {
+  const files = normalizeInputFiles(inputFiles)
+  assertSingleInputSource({ input, inputFile, inputFiles: files, inputStream })
+  const requestedCompression = normalizeCompression(compression)
+
+  if (files) {
+    for (const file of files) yield * yieldFileItems(file, chunkSize, requestedCompression)
+    return
+  }
+
+  if (hasInput(inputFile)) {
+    yield * yieldFileItems(inputFile, chunkSize, requestedCompression)
+    return
+  }
+
+  if (hasInput(inputStream)) {
+    const isIterable = typeof inputStream[Symbol.asyncIterator] === 'function' || typeof inputStream[Symbol.iterator] === 'function'
+    if (!isIterable || typeof inputStream.pipe !== 'function' && requestedCompression === 'gzip') {
+      throw new TypeError('inputStream must be a Node.js stream or an iterable of chunks')
+    }
+    let stream = inputStream
+    if (requestedCompression === 'gzip') stream = inputStream.pipe(createGunzip())
+    for await (const chunk of stream) {
+      yield * yieldChunkItems(chunk, chunkSize)
+    }
+    return
+  }
+
+  if (requestedCompression === 'gzip') throw new Error('compression=gzip requires inputFile, inputFiles, or inputStream')
+
+  if (hasInput(input)) {
+    const buf = asInputBuffer(input)
+    for (let i = 0; i < buf.length; i += chunkSize) {
+      yield { chunk: buf.subarray(i, Math.min(i + chunkSize, buf.length)) }
+    }
+  }
+}
+
+function concatBuffers(chunks) {
+  if (chunks.length === 0) return Buffer.alloc(0)
+  return Buffer.concat(chunks)
+}
+
+function waitForWritableDrain(writable) {
+  return new Promise((resolve, reject) => {
+    function cleanup() {
+      writable.off('drain', onDrain)
+      writable.off('error', onError)
+    }
+    function onDrain() {
+      cleanup()
+      resolve()
+    }
+    function onError(err) {
+      cleanup()
+      reject(err)
+    }
+    writable.once('drain', onDrain)
+    writable.once('error', onError)
+  })
+}
+
+async function writeWritableChunk(writable, chunk) {
+  if (chunk.length === 0) return
+  if (!writable.write(chunk)) await waitForWritableDrain(writable)
+}
+
+function pullOneChunk(backend, handle, channel, chunkSize) {
+  if (typeof backend.pullChunk === 'function') {
+    return Buffer.from(backend.pullChunk(handle, channel, chunkSize))
+  }
+  return Buffer.from(backend.pull(handle, channel))
+}
+
+async function drainMain(backend, handle, chunkSize, outputChunks, onOutput, collectOutput) {
+  for (;;) {
+    const chunk = pullOneChunk(backend, handle, CHAN_MAIN, chunkSize)
+    if (chunk.length === 0) break
+    if (onOutput) await onOutput(chunk)
+    if (collectOutput) outputChunks.push(chunk)
+  }
+}
+
+async function finishAndDrainMain(backend, handle, chunkSize, outputChunks, onOutput, collectOutput) {
+  if (typeof backend.finishStep === 'function') {
+    for (;;) {
+      const done = backend.finishStep(handle)
+      await drainMain(backend, handle, chunkSize, outputChunks, onOutput, collectOutput)
+      if (done) break
+    }
+  } else {
+    backend.finish(handle)
+    await drainMain(backend, handle, chunkSize, outputChunks, onOutput, collectOutput)
+  }
+}
+
+class PipelineResult {
   constructor(output, errors, stats, samples) {
     this.output = output
     this.errors = errors
@@ -169,7 +480,7 @@ export class PipelineResult {
 }
 
 
-export class Pipeline {
+class Pipeline {
   constructor(steps, { planJson, engine } = {}) {
     this._engine = engine || null
     if (planJson) {
@@ -201,31 +512,48 @@ export class Pipeline {
     return JSON.stringify({ steps: this._steps })
   }
 
-  async run({ input, inputFile } = {}) {
+  async run({ input, inputFile, inputFiles, inputStream, sourceColumn, onOutput, collectOutput = true, chunkSize = CHUNK_SIZE, allowBlocking = false, memory, spillDir, allowFs, allowRulesFile = false, workspaceRoot, compression = 'auto', hostedAuditDefaults = false } = {}) {
+    if (chunkSize <= 0) throw new Error('chunkSize must be positive')
+    if (sourceColumn !== undefined && sourceColumn !== null && !hasInput(inputFile) && !hasInput(inputFiles)) {
+      throw new Error('sourceColumn requires inputFile or inputFiles')
+    }
+    if (onOutput !== undefined && typeof onOutput !== 'function') {
+      throw new TypeError('onOutput must be a function')
+    }
     if (this._engine && this._engine !== 'native') {
-      return this._runEngine({ input, inputFile })
+      if (onOutput || collectOutput === false || hasInput(inputStream) || hasInput(inputFiles) || sourceColumn !== undefined && sourceColumn !== null) {
+        throw new Error('streaming input/output, inputFiles, and sourceColumn require the native or WASM backend')
+      }
+      return this._runEngine({ input, inputFile, memory, spillDir })
     }
 
     const backend = await getBackend()
-    const planJson = await this._toPlanJson(backend)
-    const handle = backend.createPipeline(planJson)
+    if (spillDir && backend.isWasm) {
+      throw new Error('spillDir is not supported by WASM native execution because browser/WASM spill storage would occupy WASM memory; use the Node native addon, CLI/direct C, or DuckDB')
+    }
+    let planJson = await this._toPlanJson(backend)
+    planJson = injectSourceNameStep(planJson, sourceColumn)
+    if (hostedAuditDefaults) planJson = applyHostedAuditDefaults(planJson)
+    planJson = prepareNativePlan(planJson, { allowBlocking, memory, spillDir })
+    const handle = backend.createPipeline(planJson, effectiveHostPolicy({ allowFs, allowRulesFile, workspaceRoot, spillDir, allowBlocking: true }))
+    const outputChunks = []
 
     try {
-      if (inputFile) {
-        const data = await readFile(inputFile)
-        // Push in chunks
-        for (let i = 0; i < data.length; i += CHUNK_SIZE) {
-          const chunk = data.subarray(i, Math.min(i + CHUNK_SIZE, data.length))
-          backend.push(handle, chunk)
+      for await (const item of iterInputChunks({ input, inputFile, inputFiles, inputStream, chunkSize, compression })) {
+        if (item.sourceName !== undefined && typeof backend.setSourceName === 'function') backend.setSourceName(handle, item.sourceName)
+        if (item.flushInput) {
+          if (typeof backend.flushInput !== 'function') throw new Error('inputFiles require backend.flushInput')
+          backend.flushInput(handle)
+          await drainMain(backend, handle, chunkSize, outputChunks, onOutput, collectOutput)
+          continue
         }
-      } else if (input) {
-        const buf = typeof input === 'string' ? Buffer.from(input, 'utf-8') : input
-        backend.push(handle, buf)
+        backend.push(handle, item.chunk)
+        await drainMain(backend, handle, chunkSize, outputChunks, onOutput, collectOutput)
       }
 
-      backend.finish(handle)
+      await finishAndDrainMain(backend, handle, chunkSize, outputChunks, onOutput, collectOutput)
 
-      const output = backend.pull(handle, CHAN_MAIN)
+      const output = collectOutput ? concatBuffers(outputChunks) : Buffer.alloc(0)
       const errors = backend.pull(handle, CHAN_ERRORS)
       const stats = backend.pull(handle, CHAN_STATS)
       const samples = backend.pull(handle, CHAN_SAMPLES)
@@ -236,26 +564,111 @@ export class Pipeline {
     }
   }
 
-  async _runEngine({ input, inputFile } = {}) {
-    const { getEngine } = await import('./engines/duckdb.js')
+  async *iterChunks({ input, inputFile, inputFiles, inputStream, sourceColumn, chunkSize = CHUNK_SIZE, allowBlocking = false, memory, spillDir, allowFs, allowRulesFile = false, workspaceRoot, compression = 'auto' } = {}) {
+    if (chunkSize <= 0) throw new Error('chunkSize must be positive')
+    if (sourceColumn !== undefined && sourceColumn !== null && !hasInput(inputFile) && !hasInput(inputFiles)) {
+      throw new Error('sourceColumn requires inputFile or inputFiles')
+    }
+    if (this._engine && this._engine !== 'native') {
+      throw new Error('streaming chunk iteration requires the native or WASM backend')
+    }
+
+    const backend = await getBackend()
+    if (spillDir && backend.isWasm) {
+      throw new Error('spillDir is not supported by WASM native execution because browser/WASM spill storage would occupy WASM memory; use the Node native addon, CLI/direct C, or DuckDB')
+    }
+    let planJson = await this._toPlanJson(backend)
+    planJson = injectSourceNameStep(planJson, sourceColumn)
+    planJson = prepareNativePlan(planJson, { allowBlocking, memory, spillDir })
+    const handle = backend.createPipeline(planJson, effectiveHostPolicy({ allowFs, allowRulesFile, workspaceRoot, spillDir, allowBlocking: true }))
+
+    try {
+      for await (const item of iterInputChunks({ input, inputFile, inputFiles, inputStream, chunkSize, compression })) {
+        if (item.sourceName !== undefined && typeof backend.setSourceName === 'function') backend.setSourceName(handle, item.sourceName)
+        if (item.flushInput) {
+          if (typeof backend.flushInput !== 'function') throw new Error('inputFiles require backend.flushInput')
+          backend.flushInput(handle)
+        } else {
+          backend.push(handle, item.chunk)
+        }
+        for (;;) {
+          const out = pullOneChunk(backend, handle, CHAN_MAIN, chunkSize)
+          if (out.length === 0) break
+          yield out
+        }
+      }
+
+      if (typeof backend.finishStep === 'function') {
+        for (;;) {
+          const done = backend.finishStep(handle)
+          for (;;) {
+            const out = pullOneChunk(backend, handle, CHAN_MAIN, chunkSize)
+            if (out.length === 0) break
+            yield out
+          }
+          if (done) break
+        }
+      } else {
+        backend.finish(handle)
+        for (;;) {
+          const out = pullOneChunk(backend, handle, CHAN_MAIN, chunkSize)
+          if (out.length === 0) break
+          yield out
+        }
+      }
+    } finally {
+      backend.free(handle)
+    }
+  }
+
+  toReadable({ highWaterMark, ...options } = {}) {
+    const streamOptions = { objectMode: false }
+    if (highWaterMark !== undefined) streamOptions.highWaterMark = highWaterMark
+    return Readable.from(this.iterChunks(options), streamOptions)
+  }
+
+  async writeTo(writable, { end = true, ...options } = {}) {
+    if (!writable || typeof writable.write !== 'function') {
+      throw new TypeError('writable must be a Node.js Writable stream')
+    }
+    const result = await this.run({
+      ...options,
+      collectOutput: false,
+      onOutput: async chunk => {
+        await writeWritableChunk(writable, chunk)
+      }
+    })
+    if (end && typeof writable.end === 'function') {
+      writable.end()
+      await finished(writable)
+    }
+    return result
+  }
+
+  async _runEngine({ input, inputFile, inputFiles, sourceColumn, memory, spillDir } = {}) {
+    if (hasInput(inputFiles) || sourceColumn !== undefined && sourceColumn !== null) {
+      throw new Error('inputFiles and sourceColumn require the native or WASM backend')
+    }
+    const { getEngine } = require('./engines/duckdb.js')
     const engine = getEngine(this._engine)
     const dsl = this._dsl
     if (!dsl) throw new Error('DuckDB engine requires a DSL string pipeline')
-    return engine.run(dsl, { input, inputFile })
+    return engine.run(dsl, { input, inputFile, memory, spillDir })
   }
 }
 
-export async function compileDsl(dsl) {
+async function compileDsl(dsl) {
   const backend = await getBackend()
   return backend.compileDsl(dsl)
 }
 
-export async function compileToSql(dsl) {
+async function compileToSql(dsl, options) {
+  normalizeSqlDialect(options)
   const backend = await getBackend()
   return backend.compileToSql(dsl)
 }
 
-export async function loadRecipe(source) {
+async function loadRecipe(source) {
   let planJson
   if (typeof source === 'object' && source.steps) {
     planJson = JSON.stringify(source)
@@ -267,11 +680,11 @@ export async function loadRecipe(source) {
   return new Pipeline(null, { planJson })
 }
 
-export async function saveRecipe(steps, path) {
+async function saveRecipe(steps, path) {
   await writeFile(path, JSON.stringify({ steps }))
 }
 
-export async function recipes() {
+async function recipes() {
   const backend = await getBackend()
   const n = backend.recipeCount()
   const result = []
@@ -284,3 +697,13 @@ export async function recipes() {
   }
   return result
 }
+
+module.exports = {
+  Pipeline,
+  PipelineResult,
+  compileDsl,
+  compileToSql,
+  loadRecipe,
+  saveRecipe,
+  recipes
+}

package/src/recipe_json.js

@@ -0,0 +1,80 @@
+'use strict'
+
+const { TranfiTransformError } = require('./transform_error.js')
+
+// The replacer runs before JSON.stringify emits each value. Charge the exact
+// UTF-8 wire size there, so neither a giant string nor many small fields can
+// allocate an over-budget JSON result. Getters/toJSON are evaluated only once.
+function stringifyRecipe(recipe, limits) {
+  const limit = Math.min(limits.maxRecipeBytes, limits.maxAllocationBytes - 1)
+  let bytes = 0
+  const parents = []
+  const counts = []
+  function charge(n) {
+    bytes += n
+    if (bytes > limit) {
+      throw new TranfiTransformError(104, 'recipe exceeds its configured byte limit')
+    }
+  }
+  function string(value) {
+    charge(2)
+    for (let i = 0; i < value.length; i++) {
+      const c = value.charCodeAt(i)
+      if (c >= 0xd800 && c <= 0xdbff) {
+        const next = value.charCodeAt(++i)
+        if (!(next >= 0xdc00 && next <= 0xdfff)) {
+          throw new TypeError('recipe must not contain an unpaired UTF-16 surrogate')
+        }
+        charge(4)
+      } else if (c >= 0xdc00 && c <= 0xdfff) {
+        throw new TypeError('recipe must not contain an unpaired UTF-16 surrogate')
+      } else if (c === 34 || c === 92 || (c >= 8 && c <= 10) || c === 12 || c === 13) charge(2)
+      else if (c < 32) charge(6)
+      else charge(c < 128 ? 1 : c < 2048 ? 2 : 3)
+    }
+  }
+  return JSON.stringify(recipe, function(key, value) {
+    while (parents.length && parents[parents.length - 1] !== this) {
+      parents.pop()
+      counts.pop()
+    }
+    // JSON's boxed primitives are scalars. Use its conversion semantics before
+    // counting; returning the scalar prevents a second custom coercion.
+    if (value && typeof value === 'object') {
+      let boxed = false
+      try { Number.prototype.valueOf.call(value); boxed = true } catch (_) {}
+      if (boxed) value = Number(value)
+      else {
+        try { String.prototype.valueOf.call(value); boxed = true } catch (_) {}
+        if (boxed) value = String(value)
+        else {
+          try { value = Boolean.prototype.valueOf.call(value) } catch (_) {}
+        }
+      }
+    }
+    const type = typeof value
+    const omitted = value === undefined || type === 'function' || type === 'symbol'
+    if (omitted && !Array.isArray(this)) return undefined
+    if (parents.length) {
+      const last = parents.length - 1
+      if (counts[last]++) charge(1)
+      if (!Array.isArray(this)) { string(key); charge(1) }
+    }
+    if (value === null || omitted) charge(4)
+    else if (type === 'string') string(value)
+    else if (type === 'number') charge(Number.isFinite(value) ? String(value).length : 4)
+    else if (type === 'boolean') charge(value ? 4 : 5)
+    else if (type === 'object') {
+      if (parents.includes(value)) throw new TypeError('circular recipe')
+      if (parents.length + 1 > limits.maxJsonDepth) {
+        throw new TranfiTransformError(104, 'recipe exceeds its configured depth limit')
+      }
+      charge(2)
+      parents.push(value)
+      counts.push(0)
+    }
+    return value
+  })
+}
+
+module.exports = { stringifyRecipe }