tokenfold 0.4.1__tar.gz → 0.5.0__tar.gz

{tokenfold-0.4.1 → tokenfold-0.5.0}/Cargo.lock

@@ -206,7 +206,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -270,7 +270,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -620,7 +620,7 @@ dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -632,7 +632,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -860,7 +860,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]
@@ -945,6 +945,17 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "syn"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "target-lexicon"
 version = "0.13.5"
@@ -970,7 +981,16 @@ version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
 dependencies = [
- "thiserror-impl",
+ "thiserror-impl 1.0.69",
+]
+
+[[package]]
+name = "thiserror"
+version = "2.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f"
+dependencies = [
+ "thiserror-impl 2.0.20",
 ]
 
 [[package]]
@@ -981,7 +1001,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "2.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.4",
 ]
 
 [[package]]
@@ -1013,7 +1044,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-adapters"
-version = "0.4.1"
+version = "0.5.0"
 dependencies = [
  "serde",
  "serde_json",
@@ -1022,7 +1053,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-admin"
-version = "0.4.1"
+version = "0.5.0"
 dependencies = [
  "ed25519-dalek",
  "serde",
@@ -1032,7 +1063,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-cli"
-version = "0.4.1"
+version = "0.5.0"
 dependencies = [
  "clap",
  "serde",
@@ -1045,39 +1076,40 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-core"
-version = "0.4.1"
+version = "0.5.0"
 dependencies = [
  "proptest",
  "regex",
  "serde",
  "serde_json",
  "sha2",
- "thiserror",
+ "thiserror 1.0.69",
  "tiktoken-rs",
  "toml",
+ "toon-format",
 ]
 
 [[package]]
 name = "tokenfold-image"
-version = "0.4.1"
+version = "0.5.0"
 
 [[package]]
 name = "tokenfold-learn"
-version = "0.4.1"
+version = "0.5.0"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-output"
-version = "0.4.1"
+version = "0.5.0"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-proxy"
-version = "0.4.1"
+version = "0.5.0"
 dependencies = [
  "clap",
  "serde",
@@ -1089,7 +1121,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-py"
-version = "0.4.1"
+version = "0.5.0"
 dependencies = [
  "pyo3",
  "serde_json",
@@ -1098,7 +1130,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-rag"
-version = "0.4.1"
+version = "0.5.0"
 
 [[package]]
 name = "toml"
@@ -1141,6 +1173,18 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
 
+[[package]]
+name = "toon-format"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f89570c1a68d73941f728cca32a4345b2ffca36667ad921af336c60309a3e7e"
+dependencies = [
+ "indexmap",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.20",
+]
+
 [[package]]
 name = "typenum"
 version = "1.20.1"
@@ -1364,7 +1408,7 @@ checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.118",
 ]
 
 [[package]]

{tokenfold-0.4.1 → tokenfold-0.5.0}/Cargo.toml

@@ -3,9 +3,9 @@ resolver = "2"
 members = ["crates/tokenfold-core", "crates/tokenfold-py"]
 
 [workspace.package]
-version = "0.4.1"
+version = "0.5.0"
 edition = "2024"
-rust-version = "1.97"
+rust-version = "1.98"
 license = "Apache-2.0"
 repository = "https://github.com/snchimata/tokenfold"
 
@@ -18,6 +18,7 @@ clap = { version = "4", features = ["derive"] }
 
 tiktoken-rs = "0.7"
 sha2 = "0.10"
+toon-format = { version = "0.5", default-features = false }
 
 proptest = "1"
 toml = "0.8"

{tokenfold-0.4.1 → tokenfold-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tokenfold
-Version: 0.4.1
+Version: 0.5.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
@@ -20,19 +20,19 @@ Keywords: llm,tokens,compression,tokenizer,openai,json,prompt-engineering,cli,to
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 
-<div align="center">
+<div align="center">
 
 # TOKENFOLD
 
 **Send less noise. Fit more context. Pay for fewer input tokens.**
 
-**Up to 92% fewer input tokens · lossless by default · exact receipts**
+**Up to 92% fewer input tokens · lossless by default · exact receipts**
 
-CLI · Python · TypeScript · Rust · proxy · MCP · local-first · provider-neutral
+CLI · Python · TypeScript · Rust · proxy · MCP · local-first · provider-neutral
 
 [![CI](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=tests&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![Coverage](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=coverage&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![GitHub Release](https://img.shields.io/github/v/release/snchimata/tokenfold?logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/releases/latest) [![PyPI](https://img.shields.io/pypi/v/tokenfold?label=PyPI&style=flat-square)](https://pypi.org/project/tokenfold/) [![npm](https://img.shields.io/npm/v/tokenfold?label=npm&logo=npm&style=flat-square)](https://www.npmjs.com/package/tokenfold) [![Rust](https://img.shields.io/crates/v/tokenfold-core?label=Rust&style=flat-square)](https://docs.rs/crate/tokenfold-core/latest) [![License](https://img.shields.io/badge/license-Apache--2.0-blue?style=flat-square)](https://github.com/snchimata/tokenfold/blob/main/LICENSE)
 
-[Measured results](#measured-results) · [Platform](#one-platform-two-engines) · [Quick start](#quick-start) · [Lossy pruning](#recoverable-lossy-pruning) · [Select model](#tokenfold-select) · [Reproduce](#reproduce-the-results)
+[Measured results](#measured-results) · [Platform](#one-platform-two-engines) · [Quick start](#quick-start) · [Lossy pruning](#recoverable-lossy-pruning) · [Select model](#tokenfold-select) · [Reproduce](#reproduce-the-results)
 
 </div>
 
@@ -44,18 +44,21 @@ CLI · Python · TypeScript · Rust · proxy · MCP · local-first · provider-n
 
 | Search / RAG results | Repetitive JSON | API responses | Tool schemas |
 | :---: | :---: | :---: | :---: |
-| **91.7% fewer tokens** | **67.6% fewer tokens** | **61.3% fewer tokens** | **45.6% fewer tokens** |
-| 84,032 → 6,951 | 50-record payload | 30-record payload | 1.8 MB OpenAI fixture |
+| **91.7% fewer tokens** | **67.6% fewer tokens** | **63.9% fewer tokens** | **45.6% fewer tokens** |
+| 84,032 → 6,951 | 50-record payload | 30-record payload | 1.8 MB OpenAI fixture |
 | opt-in, recoverable | lossless | lossless | lossless |
 
-All four use exact `o200k_base` counts in balanced mode. The 91.7% showcase
+All four use exact `o200k_base` counts in balanced preset. The 91.7% showcase
 keeps the planted `503` result inline and makes the other 98 long rows
 retrievable by hash. Run it with `python examples/lossy_pruning.py`.
+The versioned inputs and source provenance for these headline figures live in
+[`tests/fixtures/readme_metrics.json`](tests/fixtures/readme_metrics.json).
 
 ### Fine-tuned relevance ranking
 
-[Tokenfold Select][tokenfold-select] is the optional query-aware model for
+[Tokenfold Select][tokenfold-select] is an optional external query-aware model for
 choosing what fills a tight context window after structural compression ends.
+It is distributed and evaluated separately; Tokenfold Core does not invoke it.
 
 | Token budget kept | Tokenfold Select task success | BM25 reference | Lift |
 | ---: | ---: | ---: | ---: |
@@ -70,6 +73,8 @@ three-seed repeated subsampling over roughly 73,000 training and 24,000
 held-out fixtures per run; the [model card][tokenfold-select] publishes the
 training recipe and full baseline set. Here, task success means the literal
 gold answer survived selection under the stated budget.
+These are source-reported external results, pinned by model revision in the
+[metric manifest](tests/fixtures/readme_metrics.json); Core CI does not reproduce them.
 
 ## One platform, two engines
 
@@ -78,7 +83,7 @@ gold answer survived selection under the stated budget.
 | **Tokenfold Core** | Fast deterministic compression for JSON, schemas, provider requests, logs, diffs, and command output |
 | **Recoverable pruning** | Drop low-signal JSON rows only after storing them locally; fetch any omission with `tokenfold retrieve` |
 | **Tokenfold Select** | LoRA-fine-tuned, query-aware span ranking with up to **15.8% lift** over BM25 at the same budget |
-| **Budget control** | Conservative, balanced, and aggressive modes; nine task scopes; `--target-tokens` with honest best-effort reporting |
+| **Budget control** | Conservative, balanced, and aggressive presets; nine task scopes; `--target-tokens` with honest best-effort reporting |
 | **Exact receipts** | Before/after token counts, applied transforms, warnings, provenance, and final status on every call |
 | **Realized savings** | `gain`, `stats`, and `session` report measured savings; `learn` proposes policy improvements without silently applying them |
 | **Every integration** | CLI, Python, TypeScript, Rust, HTTP proxy, and MCP share the same Core engine and report shape |
@@ -86,11 +91,11 @@ gold answer survived selection under the stated budget.
 
 ### Lossless or recoverable lossy
 
-| | Lossless — default | Recoverable lossy — opt-in |
+| | Lossless — default | Recoverable lossy — opt-in |
 | --- | --- | --- |
 | **What it does** | Minifies, folds repeated keys into columns, and stores repeated values once | Adds deterministic ranking and retrieval markers for selected array rows |
 | **Best on** | Uniform records, schemas, logs, diffs | Search results, mixed event feeds, agent traces, long arrays |
-| **Measured here** | **45.6–67.6%** fewer tokens | **39.3–91.7%** fewer tokens |
+| **Measured here** | **45.6–67.6%** fewer tokens | **51.7–91.7%** fewer tokens |
 | **Recovery** | All data remains in the payload | Every emitted marker resolves through the local retrieval store |
 
 ## Quick start
@@ -104,7 +109,7 @@ cargo add tokenfold-core    # Rust library
 cargo install tokenfold-cli # Rust CLI
 ```
 
-Or download a signed CLI build for Linux, macOS, or Windows from
+Or download a checksummed CLI build for Linux, macOS, or Windows from
 [GitHub Releases](https://github.com/snchimata/tokenfold/releases/latest) and
 verify it with the adjacent `.sha256` file.
 
@@ -121,11 +126,12 @@ Python uses the same Core engine and typed receipt:
 import json
 from pathlib import Path
 
-from tokenfold import CompressionMode, compress_openai_payload
+from tokenfold import InputFormat, Preset, compress
 
-result = compress_openai_payload(
-    Path("request.json").read_text(),
-    mode=CompressionMode.BALANCED,
+result = compress(
+    Path("request.json").read_bytes(),
+    format=InputFormat.OPENAI_JSON,
+    preset=Preset.BALANCED,
 )
 compressed_request = json.loads(result.payload)
 print(f"saved {result.report.saved_tokens} tokens ({result.saved_pct():.1f}%)")
@@ -133,15 +139,18 @@ print(f"saved {result.report.saved_tokens} tokens ({result.saved_pct():.1f}%)")
 
 | Surface | Best for | Start here |
 | --- | --- | --- |
-| CLI | Files, stdin, diffs, and command output | `tokenfold compress`, `inspect`, `diff`, `wrap` |
+| CLI | Files and stdin | `tokenfold compress`, `inspect`, `decode`, `retrieve` |
 | Python | Applications and evaluation pipelines | `pip install tokenfold` |
 | TypeScript | Node.js applications and automation | `npm install tokenfold` |
 | Rust | Native embedding | `cargo add tokenfold-core` |
 | HTTP proxy | Transparent provider-shaped traffic | Build `tokenfold-proxy` |
 | MCP | Agents and editors | `tokenfold mcp serve` |
 
-`tokenfold init --agent <agent>` installs a durable host integration;
-`tokenfold doctor` verifies it. Trusted filters for Git, build, and test output
+`tokenfold init --agent claude-code` merges a project-scoped `.mcp.json` entry without replacing
+other servers; `tokenfold doctor --agent claude-code` verifies it. Codex can use the same stdio
+server with `codex mcp add tokenfold -- tokenfold mcp serve`. See
+[`docs/configuration.md`](docs/configuration.md) for tested MCP JSON/TOML and every environment
+override. Trusted filters for Git, build, and test output
 are available through `tokenfold filters list`.
 
 Runnable examples, one per surface, all under [`examples/`](examples):
@@ -153,49 +162,50 @@ cargo run -p tokenfold-core --example quickstart   # Rust: the embedded core API
 python examples/lossy_pruning.py   # CLI: opt-in recoverable pruning, end to end
 ```
 
-[`examples/quickstart.ipynb`](examples/quickstart.ipynb) is the notebook form of the
-Python quickstart, one operation per cell.
+[`examples/quickstart.ipynb`](examples/quickstart.ipynb) is the guided tour of the whole
+Python surface: everything `quickstart.py` shows, plus previews, budgets and presets,
+provider payloads, recoverable pruning with retrieval, and where Tokenfold Select fits.
 
 ## Recoverable lossy pruning
 
-Lossless folding has a ceiling on heterogeneous arrays. `--lossy` ranks rows,
+Lossless folding has a ceiling on heterogeneous arrays. `--prune` ranks rows,
 keeps the strongest signals, and replaces selected rows with compact
 `{"$tf_ref": {...}}` handles. A row leaves the payload only after the local
 store accepts it.
 
 ```bash
 tokenfold compress examples/incident_feed.json --format json \
-  --lossy heuristic --lossy-ratio 0.35 --output feed.compact.json
+  --prune --keep-ratio 0.35 --output feed.compact.json
 ```
 
 On the bundled 40-event feed:
 
 | Mode | Exact tokens | Reduction | Events kept | Incident kept |
 | --- | ---: | ---: | ---: | :---: |
-| Lossless | 2,840 | 28.1% | 40 | ✅ |
-| `--lossy-ratio 0.35` | 2,399 | **39.3%** | 13 | ✅ |
-| `--lossy-ratio 0.05` | 2,192 | **44.5%** | 2 | ✅ |
+| Lossless | 6,144 | 14.9% | 40 | ✅ |
+| `--keep-ratio 0.35` | 3,485 | **51.7%** | 13 | ✅ |
+| `--keep-ratio 0.05` | 2,294 | **68.2%** | 1 | ✅ |
 
 The planted `503` with `success: false` and `retries: 7` survives every shown
-setting because typed failure signals outrank position and length. Long rows
+setting because typed failure signals outrank position and length — at
+`--keep-ratio 0.05` it is the only row kept. Long rows
 amortize marker overhead further: the 100-result showcase reaches **91.7%**.
 
 Fetch a dropped row:
 
 ```bash
-tokenfold retrieve 978339a8898fedde3c5b0662a213f12ae4ad1b7fe6771f62b3c3d74d87389a4c
+tokenfold retrieve cb13cc59cca0c218c579cd1d4b3cbab58d6dea265eb995cc9c00faf0cd0a6856
 # {"seq":1,"ts":"2026-08-15T00:01:11Z","subsystem":"index-writer",...}
 ```
 
 What the flags mean:
 
-- `--lossy-ratio` is an aggression hint over eligible array items. Lower keeps
+- `--keep-ratio` is an aggression hint over eligible array items. Lower keeps
   fewer rows; it is not a whole-document guarantee.
 - `--target-tokens` is the whole-document goal. Tokenfold stops when it reaches
   the target losslessly and reports `best_effort` when the safe transform set
-  cannot reach it (`unreachable_target` when protected content alone exceeds
-  the target).
-- `--lossy-preserve <path>` protects a named array; nested paths conservatively
+  cannot reach it (`unreachable` when protected content alone exceeds the target).
+- `--preserve <path>` protects a named array; nested paths conservatively
   protect their nearest eligible ancestor.
 - Generic JSON only: lossy pruning does not run on OpenAI or Anthropic message
   payloads.
@@ -205,18 +215,18 @@ What the flags mean:
 Preview the projected savings with no store writes:
 
 ```bash
-tokenfold compress examples/incident_feed.json --format json \
-  --lossy heuristic --lossy-ratio 0.35 --dry-run
+tokenfold inspect examples/incident_feed.json --format json \
+  --prune --keep-ratio 0.35
 ```
 
 The same flags, and the same fail-closed contract, from Python and TypeScript:
 
 ```python
-from tokenfold import CompressionPolicy, InputFormat, LossyPath, compress, retrieve
+from tokenfold import InputFormat, PruningPolicy, compress, retrieve
 
-policy = CompressionPolicy(lossy=LossyPath.HEURISTIC, lossy_ratio=0.35)
-result = compress(feed_bytes, format=InputFormat.JSON, policy=policy)
-original = retrieve(marker["$tf_ref"]["hash"])   # any dropped row, verbatim
+pruning = PruningPolicy(keep_ratio=0.35)
+result = compress(feed_bytes, format=InputFormat.JSON, pruning=pruning)
+original = retrieve(marker["$tf_ref"])   # any dropped row, verbatim
 ```
 
 ```ts
@@ -224,8 +234,7 @@ import { compress, retrieve } from "tokenfold";
 
 const { payload, report } = await compress(feed, {
   format: "json",
-  lossy: "heuristic",
-  lossyRatio: 0.35,
+  pruning: { keepRatio: 0.35 },
 });
 const original = await retrieve(hash); // any dropped row, verbatim
 ```
@@ -234,9 +243,8 @@ const original = await retrieve(hash); // any dropped row, verbatim
 <summary><strong>Current Phase 1 constraints</strong></summary>
 
 Treat `$tf_ref` as reserved and do not enable lossy pruning on documents that
-already contain retrieval markers. A filesystem failure after partial writes
-may also leave unreferenced entries until their configured TTL expires. Both
-require location-based transactional materialization before promotion.
+already contain retrieval markers. Filesystem entries are published under a
+cross-process lock, and a refused batch leaves every candidate inline.
 
 Preview is a projection rather than a filesystem transaction, so a real run may
 keep more rows if storage becomes unavailable.
@@ -250,8 +258,8 @@ keep more rows if storage becomes unavailable.
 Tokenfold Select is an Apache-2.0 LoRA adapter on
 `ibm-granite/granite-embedding-reranker-english-r2`. It scores candidate spans
 against a query; your allocator applies the token budget and force-keeps
-required content. Core remains model-free and deterministic, while Select adds
-relevance when lexical heuristics stop being enough.
+required content. It is a separately distributed companion: Core remains
+model-free and deterministic and neither loads nor invokes Select.
 
 | | Tokenfold Core | Tokenfold Select |
 | --- | --- | --- |
@@ -318,6 +326,12 @@ training data, and limitations.
 - **Local control:** detected secrets are redacted before reports or storage;
   policy learning changes configuration only with `--apply`.
 
+The offline fidelity gate uses deterministic lexical-overlap, critical-token,
+and containment proxies. Those checks catch regressions but do not establish
+semantic equivalence or downstream task success. Runtime `quality` fields are
+absent unless a versioned evaluator has supplied data; applications should run
+their own representative task evaluation before enabling lossy pruning.
+
 ## Reproduce the results
 
 ```bash
@@ -332,8 +346,8 @@ cargo run --release --locked -p tokenfold-cli -- \
   inspect examples/api_response.json --format json
 ```
 
-The small bundled API response reports 382 → 206 tokens, a **46.1% lossless
-reduction**. Benchmark sources and thresholds live in [CHANGELOG.md](https://github.com/snchimata/tokenfold/blob/main/CHANGELOG.md)
+The bundled 30-record API response reports 3,812 → 1,376 tokens, a **63.9%
+lossless reduction**. Benchmark sources and thresholds live in [CHANGELOG.md](https://github.com/snchimata/tokenfold/blob/main/CHANGELOG.md)
 and [`crates/tokenfold-core/benches/THRESHOLDS.toml`](https://github.com/snchimata/tokenfold/blob/main/crates/tokenfold-core/benches/THRESHOLDS.toml).
 
 ## Contributing
@@ -362,7 +376,7 @@ tokens your application can stop sending today.
 pip install tokenfold
 ```
 
-If Tokenfold earns a place in your stack, a ⭐ on
+If Tokenfold earns a place in your stack, a ⭐ on
 [GitHub](https://github.com/snchimata/tokenfold) helps the next team find it.
 
 [tokenfold-select]: https://huggingface.co/snchimata/tokenfold-select