tokenfold 0.3.3__tar.gz → 0.4.0__tar.gz

{tokenfold-0.3.3 → tokenfold-0.4.0}/Cargo.lock

@@ -1013,7 +1013,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-adapters"
-version = "0.3.3"
+version = "0.4.0"
 dependencies = [
  "serde",
  "serde_json",
@@ -1022,7 +1022,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-admin"
-version = "0.3.3"
+version = "0.4.0"
 dependencies = [
  "ed25519-dalek",
  "serde",
@@ -1032,7 +1032,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-cli"
-version = "0.3.3"
+version = "0.4.0"
 dependencies = [
  "clap",
  "serde",
@@ -1045,7 +1045,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-core"
-version = "0.3.3"
+version = "0.4.0"
 dependencies = [
  "proptest",
  "regex",
@@ -1059,25 +1059,25 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-image"
-version = "0.3.3"
+version = "0.4.0"
 
 [[package]]
 name = "tokenfold-learn"
-version = "0.3.3"
+version = "0.4.0"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-output"
-version = "0.3.3"
+version = "0.4.0"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-proxy"
-version = "0.3.3"
+version = "0.4.0"
 dependencies = [
  "clap",
  "serde",
@@ -1089,7 +1089,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-py"
-version = "0.3.3"
+version = "0.4.0"
 dependencies = [
  "pyo3",
  "serde_json",
@@ -1098,7 +1098,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-rag"
-version = "0.3.3"
+version = "0.4.0"
 
 [[package]]
 name = "toml"

{tokenfold-0.3.3 → tokenfold-0.4.0}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/tokenfold-core", "crates/tokenfold-py"]
 
 [workspace.package]
-version = "0.3.3"
+version = "0.4.0"
 edition = "2024"
 rust-version = "1.97"
 license = "Apache-2.0"

tokenfold-0.4.0/PKG-INFO

@@ -0,0 +1,344 @@
+Metadata-Version: 2.4
+Name: tokenfold
+Version: 0.4.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing
+License-File: LICENSE
+Summary: Token-compression toolkit for LLM payloads (Python binding over the tokenfold-core Rust crate)
+Keywords: llm,tokens,compression,tokenizer,openai,json,prompt-engineering,cli,token-optimization
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+
+<div align="center">
+
+# TOKENFOLD
+
+**Send less noise. Fit more context. Pay for fewer input tokens.**
+
+**Up to 92% fewer input tokens · lossless by default · exact receipts**
+
+CLI · Python · TypeScript · Rust · proxy · MCP · local-first · provider-neutral
+
+[![CI](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=tests&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![Coverage](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=coverage&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![GitHub Release](https://img.shields.io/github/v/release/snchimata/tokenfold?logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/releases/latest) [![PyPI](https://img.shields.io/pypi/v/tokenfold?label=PyPI&style=flat-square)](https://pypi.org/project/tokenfold/) [![npm](https://img.shields.io/npm/v/tokenfold?label=npm&logo=npm&style=flat-square)](https://www.npmjs.com/package/tokenfold) [![Rust](https://img.shields.io/crates/v/tokenfold-core?label=Rust&style=flat-square)](https://docs.rs/crate/tokenfold-core/latest) [![License](https://img.shields.io/badge/license-Apache--2.0-blue?style=flat-square)](LICENSE)
+
+[Measured results](#measured-results) · [Platform](#one-platform-two-engines) · [Quick start](#quick-start) · [Lossy pruning](#recoverable-lossy-pruning) · [Select model](#tokenfold-select) · [Reproduce](#reproduce-the-results)
+
+</div>
+
+---
+
+## Measured results
+
+### Compression
+
+| Search / RAG results | Repetitive JSON | API responses | Tool schemas |
+| :---: | :---: | :---: | :---: |
+| **91.7% fewer tokens** | **67.6% fewer tokens** | **61.3% fewer tokens** | **45.6% fewer tokens** |
+| 84,032 → 6,951 | 50-record payload | 30-record payload | 1.8 MB OpenAI fixture |
+| opt-in, recoverable | lossless | lossless | lossless |
+
+All four use exact `o200k_base` counts in balanced mode. The 91.7% showcase
+keeps the planted `503` result inline and makes the other 98 long rows
+retrievable by hash. Run it with `python examples/lossy_pruning.py`.
+
+### Fine-tuned relevance ranking
+
+[Tokenfold Select][tokenfold-select] is the optional query-aware model for
+choosing what fills a tight context window after structural compression ends.
+
+| Token budget kept | Tokenfold Select task success | BM25 reference | Lift |
+| ---: | ---: | ---: | ---: |
+| 50% | **86.3%** | 79.4% | **+8.7%** |
+| 25% | **70.3%** | 60.7% | **+15.8%** |
+| 10% | **39.9%** | 37.7% | **+5.8%** |
+
+At a 25% budget, the fine-tuned ranker keeps the answer **70% of the time** and
+beats the strongest free heuristic by **15.8%**. Critical-content survival is
+**100% at every measured budget** through allocator force-keep. Results are
+three-seed repeated subsampling over roughly 73,000 training and 24,000
+held-out fixtures per run; the [model card][tokenfold-select] publishes the
+training recipe and full baseline set. Here, task success means the literal
+gold answer survived selection under the stated budget.
+
+## One platform, two engines
+
+| Capability | What you get |
+| --- | --- |
+| **Tokenfold Core** | Fast deterministic compression for JSON, schemas, provider requests, logs, diffs, and command output |
+| **Recoverable pruning** | Drop low-signal JSON rows only after storing them locally; fetch any omission with `tokenfold retrieve` |
+| **Tokenfold Select** | LoRA-fine-tuned, query-aware span ranking with up to **15.8% lift** over BM25 at the same budget |
+| **Budget control** | Conservative, balanced, and aggressive modes; nine task scopes; `--target-tokens` with honest best-effort reporting |
+| **Exact receipts** | Before/after token counts, applied transforms, warnings, provenance, and final status on every call |
+| **Realized savings** | `gain`, `stats`, and `session` report measured savings; `learn` proposes policy improvements without silently applying them |
+| **Every integration** | CLI, Python, TypeScript, Rust, HTTP proxy, and MCP share the same Core engine and report shape |
+| **Local-first safety** | Secret redaction, protected-content gates, reversible structural transforms, and no hosted data processor |
+
+### Lossless or recoverable lossy
+
+| | Lossless — default | Recoverable lossy — opt-in |
+| --- | --- | --- |
+| **What it does** | Minifies, folds repeated keys into columns, and stores repeated values once | Adds deterministic ranking and retrieval markers for selected array rows |
+| **Best on** | Uniform records, schemas, logs, diffs | Search results, mixed event feeds, agent traces, long arrays |
+| **Measured here** | **45.6–67.6%** fewer tokens | **39.3–91.7%** fewer tokens |
+| **Recovery** | All data remains in the payload | Every emitted marker resolves through the local retrieval store |
+
+## Quick start
+
+Install the surface that fits your stack:
+
+```bash
+pip install tokenfold       # Python 3.9+
+npm install tokenfold       # Node.js 22+
+cargo add tokenfold-core    # Rust library
+cargo install tokenfold-cli # Rust CLI
+```
+
+Or download a signed CLI build for Linux, macOS, or Windows from
+[GitHub Releases](https://github.com/snchimata/tokenfold/releases/latest) and
+verify it with the adjacent `.sha256` file.
+
+Preview first, then compress:
+
+```bash
+tokenfold inspect payload.json --format json
+tokenfold compress payload.json --format json --output payload.compact.json
+```
+
+Python uses the same Core engine and typed receipt:
+
+```python
+import json
+from pathlib import Path
+
+from tokenfold import CompressionMode, compress_openai_payload
+
+result = compress_openai_payload(
+    Path("request.json").read_text(),
+    mode=CompressionMode.BALANCED,
+)
+compressed_request = json.loads(result.payload)
+print(f"saved {result.report.saved_tokens} tokens ({result.saved_pct():.1f}%)")
+```
+
+| Surface | Best for | Start here |
+| --- | --- | --- |
+| CLI | Files, stdin, diffs, and command output | `tokenfold compress`, `inspect`, `diff`, `wrap` |
+| Python | Applications and evaluation pipelines | `pip install tokenfold` |
+| TypeScript | Node.js applications and automation | `npm install tokenfold` |
+| Rust | Native embedding | `cargo add tokenfold-core` |
+| HTTP proxy | Transparent provider-shaped traffic | Build `tokenfold-proxy` |
+| MCP | Agents and editors | `tokenfold mcp serve` |
+
+`tokenfold init --agent <agent>` installs a durable host integration;
+`tokenfold doctor` verifies it. Trusted filters for Git, build, and test output
+are available through `tokenfold filters list`.
+
+Runnable examples:
+
+```bash
+python examples/quickstart.py
+python examples/lossy_pruning.py
+```
+
+## Recoverable lossy pruning
+
+Lossless folding has a ceiling on heterogeneous arrays. `--lossy` ranks rows,
+keeps the strongest signals, and replaces selected rows with compact
+`{"$tf_ref": {...}}` handles. A row leaves the payload only after the local
+store accepts it.
+
+```bash
+tokenfold compress examples/incident_feed.json --format json \
+  --lossy heuristic --lossy-ratio 0.35 --output feed.compact.json
+```
+
+On the bundled 40-event feed:
+
+| Mode | Exact tokens | Reduction | Events kept | Incident kept |
+| --- | ---: | ---: | ---: | :---: |
+| Lossless | 2,840 | 28.1% | 40 | ✅ |
+| `--lossy-ratio 0.35` | 2,399 | **39.3%** | 13 | ✅ |
+| `--lossy-ratio 0.05` | 2,192 | **44.5%** | 2 | ✅ |
+
+The planted `503` with `success: false` and `retries: 7` survives every shown
+setting because typed failure signals outrank position and length. Long rows
+amortize marker overhead further: the 100-result showcase reaches **91.7%**.
+
+Fetch a dropped row:
+
+```bash
+tokenfold retrieve 978339a8898fedde3c5b0662a213f12ae4ad1b7fe6771f62b3c3d74d87389a4c
+# {"seq":1,"ts":"2026-08-15T00:01:11Z","subsystem":"index-writer",...}
+```
+
+What the flags mean:
+
+- `--lossy-ratio` is an aggression hint over eligible array items. Lower keeps
+  fewer rows; it is not a whole-document guarantee.
+- `--target-tokens` is the whole-document goal. Tokenfold stops when it reaches
+  the target losslessly and reports `best_effort` when the safe transform set
+  cannot reach it (`unreachable_target` when protected content alone exceeds
+  the target).
+- `--lossy-preserve <path>` protects a named array; nested paths conservatively
+  protect their nearest eligible ancestor.
+- Generic JSON only: lossy pruning does not run on OpenAI or Anthropic message
+  payloads.
+- Storage is fail-closed: refused rows stay inline, and detected secret-shaped
+  bytes are never persisted.
+
+Preview the projected savings with no store writes:
+
+```bash
+tokenfold compress examples/incident_feed.json --format json \
+  --lossy heuristic --lossy-ratio 0.35 --dry-run
+```
+
+<details>
+<summary><strong>Current Phase 1 constraints</strong></summary>
+
+Treat `$tf_ref` as reserved and do not enable lossy pruning on documents that
+already contain retrieval markers. A filesystem failure after partial writes
+may also leave unreferenced entries until their configured TTL expires. Both
+require location-based transactional materialization before promotion.
+
+Lossy pruning is CLI-only today; Python and TypeScript remain lossless. Preview
+is a projection rather than a filesystem transaction, so a real run may keep
+more rows if storage becomes unavailable.
+
+</details>
+
+## Tokenfold Select
+
+**When structure ends, rank what matters.**
+
+Tokenfold Select is an Apache-2.0 LoRA adapter on
+`ibm-granite/granite-embedding-reranker-english-r2`. It scores candidate spans
+against a query; your allocator applies the token budget and force-keeps
+required content. Core remains model-free and deterministic, while Select adds
+relevance when lexical heuristics stop being enough.
+
+| | Tokenfold Core | Tokenfold Select |
+| --- | --- | --- |
+| Best at | Structural compression | Query-conditioned span ranking |
+| Runtime | Static Rust binary | Granite reranker + LoRA adapter |
+| Output | Compressed payload + exact receipt | Ranking logits |
+
+<details>
+<summary><strong>Python: load the model and score spans</strong></summary>
+
+```python
+from pathlib import Path
+
+import torch
+from huggingface_hub import snapshot_download
+from peft import PeftModel
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+base_id = "ibm-granite/granite-embedding-reranker-english-r2"
+repo_dir = Path(snapshot_download("snchimata/tokenfold-select"))
+adapter_dir = repo_dir / "adapter"
+tokenizer = AutoTokenizer.from_pretrained(adapter_dir)
+base = AutoModelForSequenceClassification.from_pretrained(
+    base_id,
+    dtype=torch.float32,
+)
+model = PeftModel.from_pretrained(base, adapter_dir).eval()
+
+def score(query: str, spans: list[str]) -> list[float]:
+    if not spans:
+        return []
+    encoded = tokenizer(
+        [query] * len(spans),
+        spans,
+        padding=True,
+        truncation=True,
+        max_length=8192,
+        return_tensors="pt",
+    )
+    with torch.no_grad():
+        output = model(
+            input_ids=encoded["input_ids"],
+            attention_mask=encoded["attention_mask"],
+        )
+    return output.logits.view(-1).float().tolist()
+```
+
+</details>
+
+See the [Tokenfold Select model card][tokenfold-select] for setup, evaluation,
+training data, and limitations.
+
+## Safety and auditability
+
+- **Never larger:** Core keeps a transform only when exact recounting shows a
+  reduction; a lossy branch must also beat the lossless result.
+- **Reversible structure:** JSON folds must pass an exact round trip.
+- **Protected content:** provider system messages and latest-user content are
+  held behind format-aware safety gates.
+- **Clear provenance:** exact tokenizer counts, heuristics, and extrapolations
+  are labeled separately.
+- **Actionable receipts:** every result lists savings, transforms, warnings,
+  retrieval state, and final status.
+- **Local control:** detected secrets are redacted before reports or storage;
+  policy learning changes configuration only with `--apply`.
+
+## Reproduce the results
+
+```bash
+# 91.7% maximum-compression showcase, mixed-feed curve, retrieval, and preserve
+python examples/lossy_pruning.py
+
+# Lossless transform benchmarks
+cargo bench -p tokenfold-core
+
+# Small exact-token CLI example
+cargo run --release --locked -p tokenfold-cli -- \
+  inspect examples/api_response.json --format json
+```
+
+The small bundled API response reports 382 → 206 tokens, a **46.1% lossless
+reduction**. Benchmark sources and thresholds live in [CHANGELOG.md](CHANGELOG.md)
+and [`crates/tokenfold-core/benches/THRESHOLDS.toml`](crates/tokenfold-core/benches/THRESHOLDS.toml).
+
+## Contributing
+
+Issues and pull requests are welcome. Run the relevant checks before opening a
+PR:
+
+```bash
+cargo fmt --all --check
+cargo clippy --workspace --all-targets -- -D warnings
+cargo test --workspace --locked
+python eval/run_fidelity.py --gate --profile smoke-first-consumer
+cd packages/tokenfold && npm ci && npm test
+```
+
+## License
+
+[Apache-2.0](LICENSE)
+
+---
+
+Start with one representative payload, inspect the receipt, and see how many
+tokens your application can stop sending today.
+
+```bash
+pip install tokenfold
+```
+
+If Tokenfold earns a place in your stack, a ⭐ on
+[GitHub](https://github.com/snchimata/tokenfold) helps the next team find it.
+
+[tokenfold-select]: https://huggingface.co/snchimata/tokenfold-select
+