tokenfold 0.3.4 → 0.4.1

package/README.md

@@ -15,5 +15,24 @@ const { payload, report } = await compress(input, {
 });
 ```
 
+Opt-in lossy array pruning drops whole array items to hit a token budget instead
+of only restructuring them, replacing each with a `$tf_ref` marker that resolves
+through the local retrieval store. Generic JSON only:
+
+```ts
+import { compress, retrieve } from "tokenfold";
+
+const { payload, report } = await compress(feed, {
+  format: "json",
+  lossy: "heuristic",
+  lossyRatio: 0.35,        // selection hint, not an enforced budget
+  lossyPreserve: ["meta"], // arrays that are never pruned
+});
+const original = await retrieve(hash); // a dropped item's original bytes
+```
+
+Pass the same options to `inspect` to preview the projected savings without
+writing anything to the store.
+
 Requires Node.js 22 or newer. The matching native CLI is installed through an
 optional platform package; set `TOKENFOLD_BINARY_PATH` to use a custom binary.

package/dist/index.d.ts

@@ -5,6 +5,8 @@ export { binaryPath, run, TokenFoldProcessError };
 export type { Input, ProcessResult, RunOptions };
 export type CompressionMode = "conservative" | "balanced" | "aggressive";
 export type TaskScope = "all" | "general" | "code_review" | "change_summary" | "debugging" | "generation" | "api_overview" | "retrieval_qa" | "agent_history";
+/** Selection backend for opt-in lossy pruning. `heuristic` is the only Phase 1 path. */
+export type LossyPath = "heuristic";
 export interface CompressionOptions {
     format?: "auto" | "openai" | "anthropic" | "json" | "text" | "command" | "diff";
     mode?: CompressionMode;
@@ -14,6 +16,22 @@ export interface CompressionOptions {
     experimental?: boolean;
     storeOriginals?: boolean;
     retrieveNamespace?: string;
+    /**
+     * Opt-in LOSSY JSON array-item pruning (`--lossy`): drops whole array items to hit a token
+     * budget instead of only restructuring them. Generic JSON only — on any other format the run
+     * is a no-op and `json_prune` comes back as `status: "skipped"` with
+     * `skipped_reason: "not_applicable_to_format"`, persisting nothing. Dropped items are replaced
+     * by `$tf_ref` markers and stay recoverable via {@link retrieve}; pruning needs a durable
+     * filesystem retrieval store, so pass `configPath` when you want one other than the default.
+     */
+    lossy?: LossyPath;
+    /**
+     * `--lossy-ratio`: a best-effort selection hint (0.0..=1.0), not an enforced budget — the
+     * achieved ratio differs by design. Use `targetTokens` for a real ceiling.
+     */
+    lossyRatio?: number;
+    /** `--lossy-preserve`: dot-separated paths (e.g. `data.results`) whose arrays are never pruned. */
+    lossyPreserve?: readonly string[];
     configPath?: string;
     signal?: AbortSignal;
 }
@@ -128,3 +146,21 @@ export interface CompressionResult {
 }
 export declare function compress(input: Input, options?: CompressionOptions): Promise<CompressionResult>;
 export declare function inspect(input: Input, options?: CompressionOptions): Promise<CompressionResult>;
+export interface RetrieveOptions {
+    /** `--retrieve-namespace`: the namespace the item was stored under. */
+    namespace?: string;
+    configPath?: string;
+    signal?: AbortSignal;
+}
+/**
+ * Restores the original bytes of something a lossy run dropped, or a `storeOriginals` run saved,
+ * mirroring `tokenfold retrieve`. `reference` is either a raw hex SHA-256 hash — a compressed
+ * payload's `$tf_ref.hash` — or a `[tokenfold:retrieve hash=... namespace=...]` text marker,
+ * whose embedded namespace is used when `options.namespace` is omitted. A CompressionReport
+ * path is NOT a valid reference: the current report schema carries no per-entry hash, and the
+ * CLI rejects it rather than guessing.
+ *
+ * Throws `TokenFoldProcessError` (`code: "tokenfold_exit"`) when the hash is unknown, its TTL
+ * has elapsed, or the reference is malformed — the CLI reports all of those as a non-zero exit.
+ */
+export declare function retrieve(reference: string, options?: RetrieveOptions): Promise<Uint8Array>;

package/dist/index.js

@@ -3,24 +3,43 @@ import { TokenFoldProcessError } from "./errors.js";
 import { run } from "./process.js";
 export { binaryPath, run, TokenFoldProcessError };
 function argumentsFor(command, options) {
-    const args = [command, "--json"];
+    // The `inspect` subcommand has no --lossy flags of its own; the CLI's lossy preview is
+    // `compress --dry-run`, which routes to the same code path and, exactly like `inspect --json`,
+    // writes the report to stdout and no payload. So a lossy inspect() becomes that instead.
+    const wantsLossy = options.lossy !== undefined ||
+        options.lossyRatio !== undefined ||
+        (options.lossyPreserve?.length ?? 0) > 0;
+    const previewLossy = command === "inspect" && wantsLossy;
+    const args = previewLossy ? ["compress", "--json", "--dry-run"] : [command, "--json"];
+    const compressing = command === "compress" || previewLossy;
     if (options.format)
         args.push("--format", options.format);
     if (options.mode)
         args.push("--mode", options.mode);
     if (options.targetTokens !== undefined)
         args.push("--target-tokens", String(options.targetTokens));
-    if (options.disable?.length && command === "compress")
+    if (options.disable?.length && compressing)
         args.push("--disable", options.disable.join(","));
     if (options.taskScope)
         args.push("--task-scope", options.taskScope);
     if (options.experimental)
         args.push("--experimental");
-    if (options.storeOriginals && command === "compress")
+    if (options.storeOriginals && compressing)
         args.push("--store-originals");
-    if (options.retrieveNamespace && command === "compress") {
+    if (options.retrieveNamespace && compressing) {
         args.push("--retrieve-namespace", options.retrieveNamespace);
     }
+    if (wantsLossy && compressing) {
+        // `--lossy-ratio`/`--lossy-preserve` are `requires = "lossy"` in the CLI. Forward them even
+        // when `lossy` is unset rather than dropping them: silently ignoring a pruning-aggression
+        // knob is worse than the CLI's own "the following required arguments were not provided".
+        if (options.lossy)
+            args.push("--lossy", options.lossy);
+        if (options.lossyRatio !== undefined)
+            args.push("--lossy-ratio", String(options.lossyRatio));
+        for (const path of options.lossyPreserve ?? [])
+            args.push("--lossy-preserve", path);
+    }
     if (options.configPath)
         args.push("--config", options.configPath);
     return args;
@@ -39,6 +58,16 @@ function parseReport(bytes, result) {
         });
     }
 }
+function throwIfFailed(result) {
+    if (result.exitCode === 0)
+        return;
+    throw new TokenFoldProcessError(`tokenfold exited with status ${result.exitCode ?? result.signal}`, {
+        code: "tokenfold_exit",
+        exitCode: result.exitCode,
+        signal: result.signal,
+        stderr: result.stderr,
+    });
+}
 async function execute(command, input, options) {
     const runOptions = {
         stdin: input,
@@ -47,14 +76,7 @@ async function execute(command, input, options) {
     if (options.signal)
         runOptions.signal = options.signal;
     const result = await run(argumentsFor(command, options), runOptions);
-    if (result.exitCode !== 0) {
-        throw new TokenFoldProcessError(`tokenfold exited with status ${result.exitCode ?? result.signal}`, {
-            code: "tokenfold_exit",
-            exitCode: result.exitCode,
-            signal: result.signal,
-            stderr: result.stderr,
-        });
-    }
+    throwIfFailed(result);
     const reportBytes = command === "compress" ? result.stderr : result.stdout;
     return {
         payload: command === "compress" ? result.stdout : Uint8Array.from(Buffer.from(input)),
@@ -67,3 +89,27 @@ export function compress(input, options = {}) {
 export function inspect(input, options = {}) {
     return execute("inspect", input, options);
 }
+/**
+ * Restores the original bytes of something a lossy run dropped, or a `storeOriginals` run saved,
+ * mirroring `tokenfold retrieve`. `reference` is either a raw hex SHA-256 hash — a compressed
+ * payload's `$tf_ref.hash` — or a `[tokenfold:retrieve hash=... namespace=...]` text marker,
+ * whose embedded namespace is used when `options.namespace` is omitted. A CompressionReport
+ * path is NOT a valid reference: the current report schema carries no per-entry hash, and the
+ * CLI rejects it rather than guessing.
+ *
+ * Throws `TokenFoldProcessError` (`code: "tokenfold_exit"`) when the hash is unknown, its TTL
+ * has elapsed, or the reference is malformed — the CLI reports all of those as a non-zero exit.
+ */
+export async function retrieve(reference, options = {}) {
+    const args = ["retrieve", reference];
+    if (options.namespace)
+        args.push("--retrieve-namespace", options.namespace);
+    if (options.configPath)
+        args.push("--config", options.configPath);
+    const runOptions = { env: { TOKENFOLD_ANALYTICS_ENABLED: "false" } };
+    if (options.signal)
+        runOptions.signal = options.signal;
+    const result = await run(args, runOptions);
+    throwIfFailed(result);
+    return result.stdout;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tokenfold",
-  "version": "0.3.4",
+  "version": "0.4.1",
   "description": "Token-aware compression for LLM payloads, backed by the tokenfold Rust CLI.",
   "license": "Apache-2.0",
   "repository": {
@@ -49,11 +49,11 @@
     "prepack": "npm run build"
   },
   "optionalDependencies": {
-    "@tokenfold/cli-darwin-x64": "0.3.4",
-    "@tokenfold/cli-darwin-arm64": "0.3.4",
-    "@tokenfold/cli-linux-x64": "0.3.4",
-    "@tokenfold/cli-linux-arm64": "0.3.4",
-    "@tokenfold/cli-win32-x64": "0.3.4"
+    "@tokenfold/cli-darwin-x64": "0.4.1",
+    "@tokenfold/cli-darwin-arm64": "0.4.1",
+    "@tokenfold/cli-linux-x64": "0.4.1",
+    "@tokenfold/cli-linux-arm64": "0.4.1",
+    "@tokenfold/cli-win32-x64": "0.4.1"
   },
   "devDependencies": {
     "@types/node": "^24.0.0",