tokenfold 0.5.3__tar.gz → 0.5.4__tar.gz

{tokenfold-0.5.3 → tokenfold-0.5.4}/Cargo.lock

@@ -1109,7 +1109,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-adapters"
-version = "0.5.3"
+version = "0.5.4"
 dependencies = [
  "serde",
  "serde_json",
@@ -1118,7 +1118,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-admin"
-version = "0.5.3"
+version = "0.5.4"
 dependencies = [
  "ed25519-dalek",
  "serde",
@@ -1128,7 +1128,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-cli"
-version = "0.5.3"
+version = "0.5.4"
 dependencies = [
  "clap",
  "serde",
@@ -1143,7 +1143,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-core"
-version = "0.5.3"
+version = "0.5.4"
 dependencies = [
  "proptest",
  "regex",
@@ -1158,25 +1158,25 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-image"
-version = "0.5.3"
+version = "0.5.4"
 
 [[package]]
 name = "tokenfold-learn"
-version = "0.5.3"
+version = "0.5.4"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-output"
-version = "0.5.3"
+version = "0.5.4"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-proxy"
-version = "0.5.3"
+version = "0.5.4"
 dependencies = [
  "clap",
  "ctrlc",
@@ -1190,7 +1190,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-py"
-version = "0.5.3"
+version = "0.5.4"
 dependencies = [
  "pyo3",
  "serde_json",
@@ -1199,7 +1199,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-rag"
-version = "0.5.3"
+version = "0.5.4"
 dependencies = [
  "tokenfold-core",
 ]

{tokenfold-0.5.3 → tokenfold-0.5.4}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/tokenfold-core", "crates/tokenfold-py"]
 
 [workspace.package]
-version = "0.5.3"
+version = "0.5.4"
 edition = "2024"
 rust-version = "1.98"
 license = "Apache-2.0"

{tokenfold-0.5.3 → tokenfold-0.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tokenfold
-Version: 0.5.3
+Version: 0.5.4
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
@@ -47,20 +47,7 @@ Tokenfold Select ranks the context that matters most when token budgets get tigh
 
 </div>
 
-## Table of contents
-
-- [Installation](#installation)
-- [How it works](#how-it-works)
-- [Quick start](#quick-start)
-- [MCP & agents](#coding-agents-and-mcp-integration)
-- [Core](#tokenfold-core)
-- [Select](#tokenfold-select)
-- [Benchmarks](#measured-results)
-- [Extended tooling](#extended-tooling)
-- [Safety and auditability](#safety-and-auditability)
-- [Reproduce the results](#reproduce-the-results)
-- [Contributing](#contributing)
-- [License](#license)
+[Install](#installation) | [Quick start](#quick-start) | [How it works](#how-it-works) | [MCP & agents](#coding-agents-and-mcp-integration) | [Core](#tokenfold-core) | [Select](#tokenfold-select) | [Benchmarks](#measured-results) | [Safety](#safety-and-auditability)
 
 <br />
 
@@ -73,6 +60,9 @@ cargo install tokenfold-cli  # Standalone CLI
 cargo add tokenfold-core     # Rust library
 ```
 
+These installs provide Core. For optional query-aware model scoring, see
+[Select setup](#tokenfold-select); its models and runtime setup are separate.
+
 <table>
   <tr>
     <td align="center" width="33%">
@@ -106,9 +96,18 @@ cargo add tokenfold-core     # Rust library
 
 ## How it works
 
+- **Core (default):** folds repeated structure without removing data. The
+  compact payload decodes to the original JSON value.
+- **[Recoverable pruning](#recoverable-lossy-pruning) (opt-in):** model-free
+  rules rank eligible JSON rows; omissions are stored locally before replacement
+  with `$tf_ref` handles. This is the path behind the **96.3% showcase**, not Select.
+- **[Select](#tokenfold-select) (separate):** a model scores groups against your
+  query; your allocator keeps selected original text under a token budget.
+  Select does not automatically store omissions or create recovery handles.
+
 ### What the model sees
 
-```json
+```jsonc
 // Input (conceptual illustration - 2 rows shown; small inputs like this
 // pass through unchanged under the never-larger guard)
 [{"id":1,"role":"admin"},{"id":2,"role":"member"}]
@@ -143,10 +142,6 @@ object shape again.
   in-memory round trip before emitting; lossy pruning is strictly opt-in and
   recoverable.
 
-For tabular arrays, Tokenfold folds repeated keys into
-header columns (`__tf_cols__`). No model runs in the Core engine, and every
-lossless transform is verified by exact decode.
-
 #### How Tokenfold decides what to fold
 
 ```mermaid
@@ -196,6 +191,35 @@ inputs and provenance live in [`tests/fixtures/readme_metrics.json`](tests/fixtu
 
 ## Quick start
 
+### Python: compress JSON directly
+
+After `pip install tokenfold`, try this self-contained example. Thirty uniform
+records provide enough repeated structure to demonstrate lossless folding:
+
+```python
+import json
+from tokenfold import InputFormat, compress
+
+records = [{"id": i, "role": "member", "status": "active"} for i in range(30)]
+result = compress(json.dumps(records), format=InputFormat.JSON)
+print(f"{result.report.original_tokens} -> {result.report.compressed_tokens} tokens "
+      f"({result.saved_pct():.1f}% saved)")
+```
+
+Send `result.payload` (bytes) to the model. Use `tokenfold.decode` when your
+application needs the original JSON shape again. Small or irregular inputs may
+pass through unchanged under the never-larger guard.
+
+### CLI: inspect and compress a file
+
+Install the CLI with `cargo install tokenfold-cli` or use a
+[checksummed GitHub Release](https://github.com/snchimata/tokenfold/releases/latest), then:
+
+```bash
+tokenfold inspect payload.json --format json
+tokenfold compress payload.json --format json --output payload.compact.json
+```
+
 ### Drop-in proxy for OpenAI-compatible clients
 
 Start the local proxy once, then point an existing client at it. The proxy
@@ -277,28 +301,6 @@ sequenceDiagram
 | Rust | `cargo add tokenfold-core` | Native embedding |
 | MCP | `tokenfold mcp serve` | Agents and editors |
 
-For files and stdin, install the CLI with `cargo install tokenfold-cli` or use a
-[checksummed GitHub Release](https://github.com/snchimata/tokenfold/releases/latest), then:
-
-```bash
-tokenfold inspect payload.json --format json
-tokenfold compress payload.json --format json --output payload.compact.json
-```
-
-For direct Python calls, use the same Core engine and typed receipt:
-
-```python
-from tokenfold import InputFormat, compress
-
-# Accepts raw JSON bytes or str (messages, tools, or schemas)
-request_json_bytes = b'{"messages": [{"role": "user", "content": "hello"}]}'
-result = compress(request_json_bytes, format=InputFormat.OPENAI_JSON)
-
-# For standalone JSON records, use: format=InputFormat.JSON
-compressed_payload = result.payload  # bytes
-print(f"Saved {result.report.saved_tokens} tokens ({result.saved_pct():.1f}%)")
-```
-
 ### Presets
 
 | Preset | Behavior |
@@ -326,11 +328,15 @@ python examples/quickstart.py      # Python: compress messages, request bodies,
 node examples/quickstart.mjs       # TypeScript/Node: compress, inspect, read the receipt
 cargo run -p tokenfold-core --example quickstart   # Rust: the embedded core API
 python examples/lossy_pruning.py   # CLI: opt-in recoverable pruning, end to end
+python examples/contextdoctor_report.py   # source checkout: offline outcome/regression controls
 ```
 
 [`examples/quickstart.ipynb`](examples/quickstart.ipynb) is the guided tour of the whole
 Python surface: everything `quickstart.py` shows, plus previews, budgets and presets,
-provider payloads, recoverable pruning with retrieval, and where Tokenfold Select fits.
+provider payloads, recoverable pruning previews, and the offline ContextDoctor controls.
+Its separately provisioned Select v3 CUDA example is explicitly enabled; set
+`RUN_SELECT_V3 = False` for a core-only tour. See the [evaluation guide](eval/README.md)
+for ContextDoctor's source-only interface and evidence limits.
 
 ### Coding agents and MCP integration
 
@@ -395,13 +401,13 @@ not the v3 architecture or loader.
 | Output | Compressed payload + exact receipt | Group relevance scores; caller-owned allocation |
 
 <details>
-<summary><strong>Released tokenfold-select 0.5.3 package: Select v2 setup</strong></summary>
+<summary><strong>Optional tokenfold-select 0.5.4 SDK: Select v2 setup</strong></summary>
 
-V3 is provisioned separately through its model card. The released 0.5.3 package
+V3 is provisioned separately through its model card. The version-aligned 0.5.4 SDK
 continues to use the v2 backend below; this README update does not change it.
 
 **Select v2 is the default scorer backend in the optional `tokenfold-select`
-0.5.3 package: local FP32 ONNX inference, original text, whole-group ranking.**
+0.5.4 package: local FP32 ONNX inference, original text, whole-group ranking.**
 Provision the separate pinned model artifact, generate explicit runtime approval,
 and invoke `tokenfold --experimental select`. Core remains model-free and
 deterministic; installing or upgrading Tokenfold does not silently load a model
@@ -410,13 +416,13 @@ The separate model bundle also carries Safetensors weights and a custom referenc
 loader; ONNX remains the CLI worker default. V2 is an experimental preview:
 [Both formats are published under Apache-2.0](https://huggingface.co/snchimata/tokenfold-select-v2).
 Synthetic CPU parity is verified, but task-quality and cold/warm performance
-qualification are still open. The [example notebook](examples/quickstart.ipynb#10-tokenfold-select-v2-optional-local-ranking)
-keeps model execution off until you explicitly enable it.
+qualification are still open. The notebook's [separate v3 CUDA example](examples/quickstart.ipynb#10-tokenfold-select-v3-local-cuda-ranking)
+is explicitly enabled and needs a provisioned v3 bundle; it does not change this SDK's defaults.
 
 <details>
 <summary><strong>Python: try Tokenfold Select v2 (ONNX)</strong></summary>
 
-Install `tokenfold-select[onnx]==0.5.3` and `huggingface_hub==1.33.0`, then download
+Install `tokenfold-select[onnx]==0.5.4` and `huggingface_hub==1.33.0`, then download
 only the ONNX profile at the pinned release commit:
 
 ```python

{tokenfold-0.5.3 → tokenfold-0.5.4}/README.md

@@ -25,20 +25,7 @@ Tokenfold Select ranks the context that matters most when token budgets get tigh
 
 </div>
 
-## Table of contents
-
-- [Installation](#installation)
-- [How it works](#how-it-works)
-- [Quick start](#quick-start)
-- [MCP & agents](#coding-agents-and-mcp-integration)
-- [Core](#tokenfold-core)
-- [Select](#tokenfold-select)
-- [Benchmarks](#measured-results)
-- [Extended tooling](#extended-tooling)
-- [Safety and auditability](#safety-and-auditability)
-- [Reproduce the results](#reproduce-the-results)
-- [Contributing](#contributing)
-- [License](#license)
+[Install](#installation) | [Quick start](#quick-start) | [How it works](#how-it-works) | [MCP & agents](#coding-agents-and-mcp-integration) | [Core](#tokenfold-core) | [Select](#tokenfold-select) | [Benchmarks](#measured-results) | [Safety](#safety-and-auditability)
 
 <br />
 
@@ -51,6 +38,9 @@ cargo install tokenfold-cli  # Standalone CLI
 cargo add tokenfold-core     # Rust library
 ```
 
+These installs provide Core. For optional query-aware model scoring, see
+[Select setup](#tokenfold-select); its models and runtime setup are separate.
+
 <table>
   <tr>
     <td align="center" width="33%">
@@ -84,9 +74,18 @@ cargo add tokenfold-core     # Rust library
 
 ## How it works
 
+- **Core (default):** folds repeated structure without removing data. The
+  compact payload decodes to the original JSON value.
+- **[Recoverable pruning](#recoverable-lossy-pruning) (opt-in):** model-free
+  rules rank eligible JSON rows; omissions are stored locally before replacement
+  with `$tf_ref` handles. This is the path behind the **96.3% showcase**, not Select.
+- **[Select](#tokenfold-select) (separate):** a model scores groups against your
+  query; your allocator keeps selected original text under a token budget.
+  Select does not automatically store omissions or create recovery handles.
+
 ### What the model sees
 
-```json
+```jsonc
 // Input (conceptual illustration - 2 rows shown; small inputs like this
 // pass through unchanged under the never-larger guard)
 [{"id":1,"role":"admin"},{"id":2,"role":"member"}]
@@ -121,10 +120,6 @@ object shape again.
   in-memory round trip before emitting; lossy pruning is strictly opt-in and
   recoverable.
 
-For tabular arrays, Tokenfold folds repeated keys into
-header columns (`__tf_cols__`). No model runs in the Core engine, and every
-lossless transform is verified by exact decode.
-
 #### How Tokenfold decides what to fold
 
 ```mermaid
@@ -174,6 +169,35 @@ inputs and provenance live in [`tests/fixtures/readme_metrics.json`](tests/fixtu
 
 ## Quick start
 
+### Python: compress JSON directly
+
+After `pip install tokenfold`, try this self-contained example. Thirty uniform
+records provide enough repeated structure to demonstrate lossless folding:
+
+```python
+import json
+from tokenfold import InputFormat, compress
+
+records = [{"id": i, "role": "member", "status": "active"} for i in range(30)]
+result = compress(json.dumps(records), format=InputFormat.JSON)
+print(f"{result.report.original_tokens} -> {result.report.compressed_tokens} tokens "
+      f"({result.saved_pct():.1f}% saved)")
+```
+
+Send `result.payload` (bytes) to the model. Use `tokenfold.decode` when your
+application needs the original JSON shape again. Small or irregular inputs may
+pass through unchanged under the never-larger guard.
+
+### CLI: inspect and compress a file
+
+Install the CLI with `cargo install tokenfold-cli` or use a
+[checksummed GitHub Release](https://github.com/snchimata/tokenfold/releases/latest), then:
+
+```bash
+tokenfold inspect payload.json --format json
+tokenfold compress payload.json --format json --output payload.compact.json
+```
+
 ### Drop-in proxy for OpenAI-compatible clients
 
 Start the local proxy once, then point an existing client at it. The proxy
@@ -255,28 +279,6 @@ sequenceDiagram
 | Rust | `cargo add tokenfold-core` | Native embedding |
 | MCP | `tokenfold mcp serve` | Agents and editors |
 
-For files and stdin, install the CLI with `cargo install tokenfold-cli` or use a
-[checksummed GitHub Release](https://github.com/snchimata/tokenfold/releases/latest), then:
-
-```bash
-tokenfold inspect payload.json --format json
-tokenfold compress payload.json --format json --output payload.compact.json
-```
-
-For direct Python calls, use the same Core engine and typed receipt:
-
-```python
-from tokenfold import InputFormat, compress
-
-# Accepts raw JSON bytes or str (messages, tools, or schemas)
-request_json_bytes = b'{"messages": [{"role": "user", "content": "hello"}]}'
-result = compress(request_json_bytes, format=InputFormat.OPENAI_JSON)
-
-# For standalone JSON records, use: format=InputFormat.JSON
-compressed_payload = result.payload  # bytes
-print(f"Saved {result.report.saved_tokens} tokens ({result.saved_pct():.1f}%)")
-```
-
 ### Presets
 
 | Preset | Behavior |
@@ -304,11 +306,15 @@ python examples/quickstart.py      # Python: compress messages, request bodies,
 node examples/quickstart.mjs       # TypeScript/Node: compress, inspect, read the receipt
 cargo run -p tokenfold-core --example quickstart   # Rust: the embedded core API
 python examples/lossy_pruning.py   # CLI: opt-in recoverable pruning, end to end
+python examples/contextdoctor_report.py   # source checkout: offline outcome/regression controls
 ```
 
 [`examples/quickstart.ipynb`](examples/quickstart.ipynb) is the guided tour of the whole
 Python surface: everything `quickstart.py` shows, plus previews, budgets and presets,
-provider payloads, recoverable pruning with retrieval, and where Tokenfold Select fits.
+provider payloads, recoverable pruning previews, and the offline ContextDoctor controls.
+Its separately provisioned Select v3 CUDA example is explicitly enabled; set
+`RUN_SELECT_V3 = False` for a core-only tour. See the [evaluation guide](eval/README.md)
+for ContextDoctor's source-only interface and evidence limits.
 
 ### Coding agents and MCP integration
 
@@ -373,13 +379,13 @@ not the v3 architecture or loader.
 | Output | Compressed payload + exact receipt | Group relevance scores; caller-owned allocation |
 
 <details>
-<summary><strong>Released tokenfold-select 0.5.3 package: Select v2 setup</strong></summary>
+<summary><strong>Optional tokenfold-select 0.5.4 SDK: Select v2 setup</strong></summary>
 
-V3 is provisioned separately through its model card. The released 0.5.3 package
+V3 is provisioned separately through its model card. The version-aligned 0.5.4 SDK
 continues to use the v2 backend below; this README update does not change it.
 
 **Select v2 is the default scorer backend in the optional `tokenfold-select`
-0.5.3 package: local FP32 ONNX inference, original text, whole-group ranking.**
+0.5.4 package: local FP32 ONNX inference, original text, whole-group ranking.**
 Provision the separate pinned model artifact, generate explicit runtime approval,
 and invoke `tokenfold --experimental select`. Core remains model-free and
 deterministic; installing or upgrading Tokenfold does not silently load a model
@@ -388,13 +394,13 @@ The separate model bundle also carries Safetensors weights and a custom referenc
 loader; ONNX remains the CLI worker default. V2 is an experimental preview:
 [Both formats are published under Apache-2.0](https://huggingface.co/snchimata/tokenfold-select-v2).
 Synthetic CPU parity is verified, but task-quality and cold/warm performance
-qualification are still open. The [example notebook](examples/quickstart.ipynb#10-tokenfold-select-v2-optional-local-ranking)
-keeps model execution off until you explicitly enable it.
+qualification are still open. The notebook's [separate v3 CUDA example](examples/quickstart.ipynb#10-tokenfold-select-v3-local-cuda-ranking)
+is explicitly enabled and needs a provisioned v3 bundle; it does not change this SDK's defaults.
 
 <details>
 <summary><strong>Python: try Tokenfold Select v2 (ONNX)</strong></summary>
 
-Install `tokenfold-select[onnx]==0.5.3` and `huggingface_hub==1.33.0`, then download
+Install `tokenfold-select[onnx]==0.5.4` and `huggingface_hub==1.33.0`, then download
 only the ONNX profile at the pinned release commit:
 
 ```python

{tokenfold-0.5.3 → tokenfold-0.5.4}/crates/tokenfold-py/Cargo.toml

@@ -16,7 +16,7 @@ name = "tokenfold_py"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-tokenfold-core = { version = "0.5.3", path = "../tokenfold-core" }
+tokenfold-core = { version = "0.5.4", path = "../tokenfold-core" }
 serde_json = { workspace = true }
 pyo3 = { version = "0.29.0", features = ["abi3-py39"] }