tokenfold 0.3.1

package/README.md

@@ -0,0 +1,19 @@
+# tokenfold for Node.js
+
+Zero-runtime-dependency TypeScript bindings for the tokenfold Rust CLI.
+
+```sh
+npm install tokenfold
+```
+
+```ts
+import { compress } from "tokenfold";
+
+const { payload, report } = await compress(input, {
+  format: "json",
+  mode: "balanced",
+});
+```
+
+Requires Node.js 22 or newer. The matching native CLI is installed through an
+optional platform package; set `TOKENFOLD_BINARY_PATH` to use a custom binary.

package/bin/tokenfold.js

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+import { spawn } from "node:child_process";
+import { binaryPath } from "../dist/index.js";
+
+let child;
+try {
+  child = spawn(binaryPath(), process.argv.slice(2), { shell: false, stdio: "inherit" });
+} catch (error) {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exitCode = 1;
+}
+
+if (child) {
+  for (const signal of ["SIGINT", "SIGTERM"]) {
+    process.on(signal, () => child?.kill(signal));
+  }
+  child.once("error", (error) => {
+    console.error(`Could not run tokenfold: ${error.message}`);
+    process.exitCode = 1;
+  });
+  child.once("close", (code, signal) => {
+    if (signal) {
+      process.removeAllListeners(signal);
+      process.kill(process.pid, signal);
+    } else {
+      process.exitCode = code ?? 1;
+    }
+  });
+}

package/dist/binary.d.ts

@@ -0,0 +1 @@
+export declare function binaryPath(): string;

package/dist/binary.js

@@ -0,0 +1,30 @@
+import { createRequire } from "node:module";
+import { dirname, resolve } from "node:path";
+import { TokenFoldProcessError } from "./errors.js";
+const require = createRequire(import.meta.url);
+const PLATFORM_PACKAGES = {
+    "darwin-x64": ["@tokenfold/cli-darwin-x64", "tokenfold"],
+    "darwin-arm64": ["@tokenfold/cli-darwin-arm64", "tokenfold"],
+    "linux-x64": ["@tokenfold/cli-linux-x64", "tokenfold"],
+    "linux-arm64": ["@tokenfold/cli-linux-arm64", "tokenfold"],
+    "win32-x64": ["@tokenfold/cli-win32-x64", "tokenfold.exe"],
+};
+export function binaryPath() {
+    const override = process.env.TOKENFOLD_BINARY_PATH;
+    if (override)
+        return resolve(override);
+    const key = `${process.platform}-${process.arch}`;
+    const platformPackage = PLATFORM_PACKAGES[key];
+    if (!platformPackage) {
+        throw new TokenFoldProcessError(`Unsupported tokenfold platform: ${key}`, {
+            code: "binary_not_found",
+        });
+    }
+    const [packageName, binaryName] = platformPackage;
+    try {
+        return resolve(dirname(require.resolve(`${packageName}/package.json`)), "bin", binaryName);
+    }
+    catch (cause) {
+        throw new TokenFoldProcessError(`The optional package ${packageName} is not installed for ${key}`, { code: "binary_not_found", cause });
+    }
+}

package/dist/errors.d.ts

@@ -0,0 +1,14 @@
+export type TokenFoldErrorCode = "binary_not_found" | "spawn_failed" | "tokenfold_exit" | "invalid_report";
+export declare class TokenFoldProcessError extends Error {
+    readonly code: TokenFoldErrorCode;
+    readonly exitCode: number | null;
+    readonly signal: NodeJS.Signals | null;
+    readonly stderr: Uint8Array;
+    constructor(message: string, options: {
+        code: TokenFoldErrorCode;
+        exitCode?: number | null;
+        signal?: NodeJS.Signals | null;
+        stderr?: Uint8Array;
+        cause?: unknown;
+    });
+}

package/dist/errors.js

@@ -0,0 +1,14 @@
+export class TokenFoldProcessError extends Error {
+    code;
+    exitCode;
+    signal;
+    stderr;
+    constructor(message, options) {
+        super(message, { cause: options.cause });
+        this.name = "TokenFoldProcessError";
+        this.code = options.code;
+        this.exitCode = options.exitCode ?? null;
+        this.signal = options.signal ?? null;
+        this.stderr = options.stderr ?? new Uint8Array();
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,130 @@
+import { binaryPath } from "./binary.js";
+import { TokenFoldProcessError } from "./errors.js";
+import { run, type Input, type ProcessResult, type RunOptions } from "./process.js";
+export { binaryPath, run, TokenFoldProcessError };
+export type { Input, ProcessResult, RunOptions };
+export type CompressionMode = "conservative" | "balanced" | "aggressive";
+export type TaskScope = "all" | "general" | "code_review" | "change_summary" | "debugging" | "generation" | "api_overview" | "retrieval_qa" | "agent_history";
+export interface CompressionOptions {
+    format?: "auto" | "openai" | "anthropic" | "json" | "text" | "command" | "diff";
+    mode?: CompressionMode;
+    targetTokens?: number;
+    disable?: readonly string[];
+    taskScope?: TaskScope;
+    experimental?: boolean;
+    storeOriginals?: boolean;
+    retrieveNamespace?: string;
+    configPath?: string;
+    signal?: AbortSignal;
+}
+export interface EstimatorInfo {
+    backend: string;
+    model: string | null;
+    is_exact: boolean;
+}
+export interface BudgetReport {
+    target_tokens: number | null;
+    protected_floor: number;
+    achieved_tokens: number;
+}
+export interface QualityReport {
+    eval_profile_id: string;
+    task_scope: string;
+    validated_ratio_band: string | null;
+    quality_retention: number;
+    contrastive_failure_rate: number;
+    gate_passed: boolean;
+}
+export interface Warning {
+    code: string;
+    severity: "info" | "warn" | "critical";
+    transform: string | null;
+    message: string;
+}
+export interface TransformReport {
+    id: string;
+    version: string;
+    tokens_before: number;
+    tokens_after: number;
+    saved_tokens: number;
+    savings_ratio: number;
+    elapsed_micros: number | null;
+    status: "applied" | "no_op" | "skipped" | "rolled_back";
+    skipped_reason: string | null;
+    warnings: readonly Warning[];
+}
+export interface CacheReport {
+    boundary_kind: string | null;
+    protected_bytes: number;
+    prefix_byte_identical: boolean;
+    warnings: readonly Warning[];
+}
+export interface RetrievalReport {
+    store_namespace: string;
+    hash_algorithm: string;
+    marker_count: number;
+    ttl_seconds: number | null;
+    persisted_original_bytes: number;
+    skipped_original_bytes: number;
+}
+export interface OutputSavingsReport {
+    profile: string;
+    estimated_output_tokens_saved: number | null;
+    measured_output_tokens_saved: number | null;
+    provenance: string;
+}
+export interface BypassReport {
+    reason: string;
+    source: string;
+}
+export interface CommandReport {
+    command_family: string | null;
+    child_exit_code: number | null;
+    duration_ms: number;
+    raw_output_bytes: number;
+    stdout_bytes: number;
+    stderr_bytes: number;
+    stderr_mode: string;
+    stderr_truncated: boolean;
+    compressed_output_bytes: number;
+    filter_pack_id: string | null;
+    filter_version: string | null;
+    never_worse_applied: boolean;
+    bypass_reason: string | null;
+}
+export interface LedgerReport {
+    recorded: boolean;
+    scope: string | null;
+    project_hash: string | null;
+    record_id: string | null;
+}
+export interface CompressionReport {
+    schema_version: string;
+    original_tokens: number;
+    compressed_tokens: number;
+    saved_tokens: number;
+    savings_ratio: number;
+    savings_pct: number;
+    estimator: EstimatorInfo;
+    status: "compressed" | "passthrough" | "best_effort" | "unreachable_target";
+    mode: string;
+    format: string;
+    task_scope: string;
+    request_id: string | null;
+    quality: QualityReport | null;
+    budget: BudgetReport | null;
+    cache: CacheReport | null;
+    retrieval: RetrievalReport | null;
+    output_savings: OutputSavingsReport | null;
+    bypass: BypassReport | null;
+    command: CommandReport | null;
+    ledger: LedgerReport | null;
+    transforms: readonly TransformReport[];
+    warnings: readonly Warning[];
+}
+export interface CompressionResult {
+    payload: Uint8Array;
+    report: CompressionReport;
+}
+export declare function compress(input: Input, options?: CompressionOptions): Promise<CompressionResult>;
+export declare function inspect(input: Input, options?: CompressionOptions): Promise<CompressionResult>;

package/dist/index.js

@@ -0,0 +1,69 @@
+import { binaryPath } from "./binary.js";
+import { TokenFoldProcessError } from "./errors.js";
+import { run } from "./process.js";
+export { binaryPath, run, TokenFoldProcessError };
+function argumentsFor(command, options) {
+    const args = [command, "--json"];
+    if (options.format)
+        args.push("--format", options.format);
+    if (options.mode)
+        args.push("--mode", options.mode);
+    if (options.targetTokens !== undefined)
+        args.push("--target-tokens", String(options.targetTokens));
+    if (options.disable?.length && command === "compress")
+        args.push("--disable", options.disable.join(","));
+    if (options.taskScope)
+        args.push("--task-scope", options.taskScope);
+    if (options.experimental)
+        args.push("--experimental");
+    if (options.storeOriginals && command === "compress")
+        args.push("--store-originals");
+    if (options.retrieveNamespace && command === "compress") {
+        args.push("--retrieve-namespace", options.retrieveNamespace);
+    }
+    if (options.configPath)
+        args.push("--config", options.configPath);
+    return args;
+}
+function parseReport(bytes, result) {
+    try {
+        return JSON.parse(Buffer.from(bytes).toString("utf8"));
+    }
+    catch (cause) {
+        throw new TokenFoldProcessError("tokenfold returned an invalid JSON report", {
+            code: "invalid_report",
+            exitCode: result.exitCode,
+            signal: result.signal,
+            stderr: result.stderr,
+            cause,
+        });
+    }
+}
+async function execute(command, input, options) {
+    const runOptions = {
+        stdin: input,
+        env: { TOKENFOLD_ANALYTICS_ENABLED: "false" },
+    };
+    if (options.signal)
+        runOptions.signal = options.signal;
+    const result = await run(argumentsFor(command, options), runOptions);
+    if (result.exitCode !== 0) {
+        throw new TokenFoldProcessError(`tokenfold exited with status ${result.exitCode ?? result.signal}`, {
+            code: "tokenfold_exit",
+            exitCode: result.exitCode,
+            signal: result.signal,
+            stderr: result.stderr,
+        });
+    }
+    const reportBytes = command === "compress" ? result.stderr : result.stdout;
+    return {
+        payload: command === "compress" ? result.stdout : Uint8Array.from(Buffer.from(input)),
+        report: parseReport(reportBytes, result),
+    };
+}
+export function compress(input, options = {}) {
+    return execute("compress", input, options);
+}
+export function inspect(input, options = {}) {
+    return execute("inspect", input, options);
+}

package/dist/process.d.ts

@@ -0,0 +1,14 @@
+export type Input = string | Uint8Array;
+export interface ProcessResult {
+    stdout: Uint8Array;
+    stderr: Uint8Array;
+    exitCode: number | null;
+    signal: NodeJS.Signals | null;
+}
+export interface RunOptions {
+    stdin?: Input;
+    cwd?: string;
+    env?: Readonly<Record<string, string | undefined>>;
+    signal?: AbortSignal;
+}
+export declare function run(args: readonly string[], options?: RunOptions): Promise<ProcessResult>;

package/dist/process.js

@@ -0,0 +1,61 @@
+import { spawn } from "node:child_process";
+import { binaryPath } from "./binary.js";
+import { TokenFoldProcessError } from "./errors.js";
+function environment(overrides) {
+    const env = { ...process.env };
+    for (const [key, value] of Object.entries(overrides ?? {})) {
+        if (value === undefined)
+            delete env[key];
+        else
+            env[key] = value;
+    }
+    return env;
+}
+export function run(args, options = {}) {
+    return new Promise((resolve, reject) => {
+        let child;
+        try {
+            child = spawn(binaryPath(), [...args], {
+                cwd: options.cwd,
+                env: environment(options.env),
+                signal: options.signal,
+                shell: false,
+                stdio: ["pipe", "pipe", "pipe"],
+            });
+        }
+        catch (cause) {
+            reject(new TokenFoldProcessError("Could not start tokenfold", { code: "spawn_failed", cause }));
+            return;
+        }
+        const stdout = [];
+        const stderr = [];
+        let settled = false;
+        child.stdout.on("data", (chunk) => stdout.push(chunk));
+        child.stderr.on("data", (chunk) => stderr.push(chunk));
+        // A process may exit before consuming all input. Its close/error event is
+        // authoritative; avoid turning the resulting broken pipe into an unhandled event.
+        child.stdin.on("error", () => { });
+        child.once("error", (cause) => {
+            if (settled)
+                return;
+            settled = true;
+            reject(new TokenFoldProcessError("Could not run tokenfold", {
+                code: "spawn_failed",
+                stderr: Buffer.concat(stderr),
+                cause,
+            }));
+        });
+        child.once("close", (exitCode, signal) => {
+            if (settled)
+                return;
+            settled = true;
+            resolve({
+                stdout: Buffer.concat(stdout),
+                stderr: Buffer.concat(stderr),
+                exitCode,
+                signal,
+            });
+        });
+        child.stdin.end(options.stdin === undefined ? undefined : Buffer.from(options.stdin));
+    });
+}

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "tokenfold",
+  "version": "0.3.1",
+  "description": "Token-aware compression for LLM payloads, backed by the tokenfold Rust CLI.",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/snchimata/tokenfold.git",
+    "directory": "packages/tokenfold"
+  },
+  "homepage": "https://github.com/snchimata/tokenfold#readme",
+  "bugs": "https://github.com/snchimata/tokenfold/issues",
+  "keywords": [
+    "llm",
+    "tokens",
+    "compression",
+    "cli"
+  ],
+  "type": "module",
+  "engines": {
+    "node": ">=22"
+  },
+  "bin": {
+    "tokenfold": "./bin/tokenfold.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "bin",
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "test": "npm run build && node --test test/*.test.mjs ../../scripts/*.test.mjs",
+    "prepack": "npm run build"
+  },
+  "optionalDependencies": {
+    "@tokenfold/cli-darwin-x64": "0.3.1",
+    "@tokenfold/cli-darwin-arm64": "0.3.1",
+    "@tokenfold/cli-linux-x64": "0.3.1",
+    "@tokenfold/cli-linux-arm64": "0.3.1",
+    "@tokenfold/cli-win32-x64": "0.3.1"
+  },
+  "devDependencies": {
+    "@types/node": "^24.0.0",
+    "typescript": "^5.9.0"
+  },
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  }
+}