tokenfold 0.3.3__tar.gz → 0.3.4__tar.gz

{tokenfold-0.3.3 → tokenfold-0.3.4}/Cargo.lock

@@ -1013,7 +1013,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-adapters"
-version = "0.3.3"
+version = "0.3.4"
 dependencies = [
  "serde",
  "serde_json",
@@ -1022,7 +1022,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-admin"
-version = "0.3.3"
+version = "0.3.4"
 dependencies = [
  "ed25519-dalek",
  "serde",
@@ -1032,7 +1032,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-cli"
-version = "0.3.3"
+version = "0.3.4"
 dependencies = [
  "clap",
  "serde",
@@ -1045,7 +1045,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-core"
-version = "0.3.3"
+version = "0.3.4"
 dependencies = [
  "proptest",
  "regex",
@@ -1059,25 +1059,25 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-image"
-version = "0.3.3"
+version = "0.3.4"
 
 [[package]]
 name = "tokenfold-learn"
-version = "0.3.3"
+version = "0.3.4"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-output"
-version = "0.3.3"
+version = "0.3.4"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-proxy"
-version = "0.3.3"
+version = "0.3.4"
 dependencies = [
  "clap",
  "serde",
@@ -1089,7 +1089,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-py"
-version = "0.3.3"
+version = "0.3.4"
 dependencies = [
  "pyo3",
  "serde_json",
@@ -1098,7 +1098,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-rag"
-version = "0.3.3"
+version = "0.3.4"
 
 [[package]]
 name = "toml"

{tokenfold-0.3.3 → tokenfold-0.3.4}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/tokenfold-core", "crates/tokenfold-py"]
 
 [workspace.package]
-version = "0.3.3"
+version = "0.3.4"
 edition = "2024"
 rust-version = "1.97"
 license = "Apache-2.0"

tokenfold-0.3.3/README.md → tokenfold-0.3.4/PKG-INFO

@@ -1,14 +1,38 @@
+Metadata-Version: 2.4
+Name: tokenfold
+Version: 0.3.4
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing
+License-File: LICENSE
+Summary: Token-compression toolkit for LLM payloads (Python binding over the tokenfold-core Rust crate)
+Keywords: llm,tokens,compression,tokenizer,openai,json,prompt-engineering,cli,token-optimization
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+
 <div align="center">
 
 # TOKENFOLD
 
 **Send less noise. Fit more context. Pay for fewer input tokens.**
 
-Local, provider-neutral compression for prompts, tool schemas, JSON, logs, and diffs.
+**46–68% fewer tokens on JSON & schemas · deterministic · reversible · every call audited**
+
+CLI · Python · TypeScript · proxy · MCP · local-first · provider-neutral
 
 [![CI](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=tests&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![Coverage](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=coverage&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![GitHub Release](https://img.shields.io/github/v/release/snchimata/tokenfold?logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/releases/latest) [![PyPI](https://img.shields.io/pypi/v/tokenfold?label=PyPI&style=flat-square)](https://pypi.org/project/tokenfold/) [![npm](https://img.shields.io/npm/v/tokenfold?label=npm&logo=npm&style=flat-square)](https://www.npmjs.com/package/tokenfold) [![Rust](https://img.shields.io/crates/v/tokenfold-core?label=Rust&style=flat-square)](https://docs.rs/crate/tokenfold-core/latest) [![License](https://img.shields.io/badge/license-Apache--2.0-blue?style=flat-square)](LICENSE)
 
-[Quick start](#quick-start) · [Why tokenfold](#why-tokenfold) · [Integrations](#pick-your-integration) · [Benchmarks](#reproduce-the-numbers)
+[Quick start](#quick-start) · [Why tokenfold](#why-tokenfold) · [Integrations](#pick-your-integration) · [Benchmarks](#reproduce-the-numbers) · [Select model](#tokenfold-select)
 
 </div>
 
@@ -26,6 +50,22 @@ JSON-data results are lossless. The schema benchmark preserves required
 fields and descriptions while trimming redundant examples. Repetitive,
 structured data benefits most.
 
+## What it does
+
+- **CLI** — `tokenfold compress` / `inspect` files or stdin, and compare
+  payloads with `tokenfold diff`; `tokenfold wrap -- <command>` compresses a
+  command's output in place.
+- **Library** — the same Rust engine and receipt shape from `pip install tokenfold`
+  or `npm install tokenfold`, so behavior never drifts between languages.
+- **Proxy** — `tokenfold-proxy` sits in front of your provider, compresses
+  requests, and streams responses through untouched.
+- **MCP server** — `tokenfold mcp serve` exposes compress/inspect/retrieve/stats
+  to any MCP-compatible agent or editor.
+- **Receipts, not guesses** — every call returns exact token counts, the
+  transforms it applied, and any warnings, so you can audit what changed.
+- **Local and lightweight** — one static Rust binary runs on your machine or
+  infrastructure, with no hosted service or additional data processor.
+
 ## Quick start
 
 Install the interface that fits your stack:
@@ -41,10 +81,12 @@ Or download the CLI for Linux, macOS, or Windows from
 [GitHub Releases](https://github.com/snchimata/tokenfold/releases/latest),
 then verify it with the adjacent `.sha256` file.
 
-Or install the Python package:
+Preview the savings without changing the input, then write the compressed
+payload when you are ready:
 
 ```bash
-pip install tokenfold
+tokenfold inspect payload.json --format json
+tokenfold compress payload.json --format json --output payload.compact.json
 ```
 
 Compress an OpenAI-style request before sending it to your provider:
@@ -129,6 +171,25 @@ messages · schemas · JSON · logs · diffs
                     └───────────▶ compressed payload + receipt
 ```
 
+## When to use tokenfold · when to look elsewhere
+
+**Good fit if you...**
+
+- want an exact, auditable receipt for every call instead of an estimate
+- need lossless guarantees on structured data — JSON, schemas, tool
+  arguments — before it reaches a billing meter
+- want a portable static binary with deterministic behavior and a small
+  operational footprint
+- are shrinking logs, diffs, and repetitive tool output before they hit an
+  LLM, a log store, or a queue
+
+**Look elsewhere if you...**
+
+- need query-aware summarization of unstructured prose — that calls for a
+  model, and tokenfold stays deterministic on purpose
+- want a hosted API to call — tokenfold runs entirely on your machine or
+  your infrastructure; there is nothing to sign up for
+
 ## What it improves
 
 | Workload | User benefit |
@@ -232,6 +293,61 @@ cargo run --release --locked -p tokenfold-cli -- \
 The sample reports 382 → 206 estimated tokens, a **46.1% reduction**. Ragged
 or compact inputs may save little; Tokenfold reports that result honestly.
 
+## Tokenfold Select
+
+**When structural compression ends, rank what matters.**
+
+Tokenfold Core removes structural waste. [Tokenfold Select][tokenfold-select]
+is its optional, query-aware companion: a LoRA-fine-tuned model that ranks
+text spans by relevance before you assemble a smaller context.
+
+| | Tokenfold Core | Tokenfold Select |
+| --- | --- | --- |
+| Best at | JSON, schemas, logs, and diffs | Query-conditioned span ranking |
+| Runtime | Static Rust binary | Granite reranker + LoRA adapter |
+| Contract | Compressed payload + auditable receipt | Ranking logits only |
+
+Select is deliberately separate from the CLI and libraries. That separation
+lets each tool do one job well: Core provides deterministic transforms and
+auditable receipts; Select adds query-aware ranking when relevance matters.
+Your allocator still owns required-content retention and the hard token
+budget.
+
+<details>
+<summary><strong>Python: load the model and score spans</strong></summary>
+
+```python
+from pathlib import Path
+
+import torch
+from huggingface_hub import snapshot_download
+from peft import PeftModel
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+base_id = "ibm-granite/granite-embedding-reranker-english-r2"
+repo_dir = Path(snapshot_download("snchimata/tokenfold-select"))
+adapter_dir = repo_dir / "adapter"
+tok = AutoTokenizer.from_pretrained(adapter_dir)
+base = AutoModelForSequenceClassification.from_pretrained(base_id, dtype=torch.float32)
+model = PeftModel.from_pretrained(base, adapter_dir).eval()
+
+def score(query: str, spans: list[str]) -> list[float]:
+    if not spans:
+        return []
+    enc = tok([query] * len(spans), spans, padding=True, truncation=True,
+              max_length=8192, return_tensors="pt")
+    with torch.no_grad():
+        out = model(input_ids=enc["input_ids"], attention_mask=enc["attention_mask"])
+        return out.logits.view(-1).float().tolist()
+```
+
+</details>
+
+See the [Tokenfold Select model card][tokenfold-select] for setup, evaluation,
+training data, and limitations.
+
+[tokenfold-select]: https://huggingface.co/snchimata/tokenfold-select
+
 ## Contributing
 
 Issues and pull requests are welcome. Run the core checks before opening a PR:
@@ -256,3 +372,8 @@ receipt, and see how many tokens your application can stop sending today.
 ```bash
 pip install tokenfold
 ```
+
+If tokenfold earns a place in your stack, a ⭐ on
+[GitHub](https://github.com/snchimata/tokenfold) helps the next team
+find it.
+

tokenfold-0.3.3/PKG-INFO → tokenfold-0.3.4/README.md

@@ -1,26 +1,16 @@
-Metadata-Version: 2.4
-Name: tokenfold
-Version: 0.3.3
-Classifier: Programming Language :: Rust
-Classifier: Programming Language :: Python :: Implementation :: CPython
-Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.9
-License-File: LICENSE
-Summary: Token-compression toolkit for LLM payloads (Python binding over the tokenfold-core Rust crate)
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
-
 <div align="center">
 
 # TOKENFOLD
 
 **Send less noise. Fit more context. Pay for fewer input tokens.**
 
-Local, provider-neutral compression for prompts, tool schemas, JSON, logs, and diffs.
+**46–68% fewer tokens on JSON & schemas · deterministic · reversible · every call audited**
+
+CLI · Python · TypeScript · proxy · MCP · local-first · provider-neutral
 
 [![CI](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=tests&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![Coverage](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=coverage&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![GitHub Release](https://img.shields.io/github/v/release/snchimata/tokenfold?logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/releases/latest) [![PyPI](https://img.shields.io/pypi/v/tokenfold?label=PyPI&style=flat-square)](https://pypi.org/project/tokenfold/) [![npm](https://img.shields.io/npm/v/tokenfold?label=npm&logo=npm&style=flat-square)](https://www.npmjs.com/package/tokenfold) [![Rust](https://img.shields.io/crates/v/tokenfold-core?label=Rust&style=flat-square)](https://docs.rs/crate/tokenfold-core/latest) [![License](https://img.shields.io/badge/license-Apache--2.0-blue?style=flat-square)](LICENSE)
 
-[Quick start](#quick-start) · [Why tokenfold](#why-tokenfold) · [Integrations](#pick-your-integration) · [Benchmarks](#reproduce-the-numbers)
+[Quick start](#quick-start) · [Why tokenfold](#why-tokenfold) · [Integrations](#pick-your-integration) · [Benchmarks](#reproduce-the-numbers) · [Select model](#tokenfold-select)
 
 </div>
 
@@ -38,6 +28,22 @@ JSON-data results are lossless. The schema benchmark preserves required
 fields and descriptions while trimming redundant examples. Repetitive,
 structured data benefits most.
 
+## What it does
+
+- **CLI** — `tokenfold compress` / `inspect` files or stdin, and compare
+  payloads with `tokenfold diff`; `tokenfold wrap -- <command>` compresses a
+  command's output in place.
+- **Library** — the same Rust engine and receipt shape from `pip install tokenfold`
+  or `npm install tokenfold`, so behavior never drifts between languages.
+- **Proxy** — `tokenfold-proxy` sits in front of your provider, compresses
+  requests, and streams responses through untouched.
+- **MCP server** — `tokenfold mcp serve` exposes compress/inspect/retrieve/stats
+  to any MCP-compatible agent or editor.
+- **Receipts, not guesses** — every call returns exact token counts, the
+  transforms it applied, and any warnings, so you can audit what changed.
+- **Local and lightweight** — one static Rust binary runs on your machine or
+  infrastructure, with no hosted service or additional data processor.
+
 ## Quick start
 
 Install the interface that fits your stack:
@@ -53,10 +59,12 @@ Or download the CLI for Linux, macOS, or Windows from
 [GitHub Releases](https://github.com/snchimata/tokenfold/releases/latest),
 then verify it with the adjacent `.sha256` file.
 
-Or install the Python package:
+Preview the savings without changing the input, then write the compressed
+payload when you are ready:
 
 ```bash
-pip install tokenfold
+tokenfold inspect payload.json --format json
+tokenfold compress payload.json --format json --output payload.compact.json
 ```
 
 Compress an OpenAI-style request before sending it to your provider:
@@ -141,6 +149,25 @@ messages · schemas · JSON · logs · diffs
                     └───────────▶ compressed payload + receipt
 ```
 
+## When to use tokenfold · when to look elsewhere
+
+**Good fit if you...**
+
+- want an exact, auditable receipt for every call instead of an estimate
+- need lossless guarantees on structured data — JSON, schemas, tool
+  arguments — before it reaches a billing meter
+- want a portable static binary with deterministic behavior and a small
+  operational footprint
+- are shrinking logs, diffs, and repetitive tool output before they hit an
+  LLM, a log store, or a queue
+
+**Look elsewhere if you...**
+
+- need query-aware summarization of unstructured prose — that calls for a
+  model, and tokenfold stays deterministic on purpose
+- want a hosted API to call — tokenfold runs entirely on your machine or
+  your infrastructure; there is nothing to sign up for
+
 ## What it improves
 
 | Workload | User benefit |
@@ -244,6 +271,61 @@ cargo run --release --locked -p tokenfold-cli -- \
 The sample reports 382 → 206 estimated tokens, a **46.1% reduction**. Ragged
 or compact inputs may save little; Tokenfold reports that result honestly.
 
+## Tokenfold Select
+
+**When structural compression ends, rank what matters.**
+
+Tokenfold Core removes structural waste. [Tokenfold Select][tokenfold-select]
+is its optional, query-aware companion: a LoRA-fine-tuned model that ranks
+text spans by relevance before you assemble a smaller context.
+
+| | Tokenfold Core | Tokenfold Select |
+| --- | --- | --- |
+| Best at | JSON, schemas, logs, and diffs | Query-conditioned span ranking |
+| Runtime | Static Rust binary | Granite reranker + LoRA adapter |
+| Contract | Compressed payload + auditable receipt | Ranking logits only |
+
+Select is deliberately separate from the CLI and libraries. That separation
+lets each tool do one job well: Core provides deterministic transforms and
+auditable receipts; Select adds query-aware ranking when relevance matters.
+Your allocator still owns required-content retention and the hard token
+budget.
+
+<details>
+<summary><strong>Python: load the model and score spans</strong></summary>
+
+```python
+from pathlib import Path
+
+import torch
+from huggingface_hub import snapshot_download
+from peft import PeftModel
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+base_id = "ibm-granite/granite-embedding-reranker-english-r2"
+repo_dir = Path(snapshot_download("snchimata/tokenfold-select"))
+adapter_dir = repo_dir / "adapter"
+tok = AutoTokenizer.from_pretrained(adapter_dir)
+base = AutoModelForSequenceClassification.from_pretrained(base_id, dtype=torch.float32)
+model = PeftModel.from_pretrained(base, adapter_dir).eval()
+
+def score(query: str, spans: list[str]) -> list[float]:
+    if not spans:
+        return []
+    enc = tok([query] * len(spans), spans, padding=True, truncation=True,
+              max_length=8192, return_tensors="pt")
+    with torch.no_grad():
+        out = model(input_ids=enc["input_ids"], attention_mask=enc["attention_mask"])
+        return out.logits.view(-1).float().tolist()
+```
+
+</details>
+
+See the [Tokenfold Select model card][tokenfold-select] for setup, evaluation,
+training data, and limitations.
+
+[tokenfold-select]: https://huggingface.co/snchimata/tokenfold-select
+
 ## Contributing
 
 Issues and pull requests are welcome. Run the core checks before opening a PR:
@@ -269,3 +351,6 @@ receipt, and see how many tokens your application can stop sending today.
 pip install tokenfold
 ```
 
+If tokenfold earns a place in your stack, a ⭐ on
+[GitHub](https://github.com/snchimata/tokenfold) helps the next team
+find it.

{tokenfold-0.3.3 → tokenfold-0.3.4}/crates/tokenfold-core/Cargo.toml

@@ -6,7 +6,7 @@ rust-version.workspace = true
 license.workspace = true
 repository.workspace = true
 description = "Token-aware compression for LLM payloads: shrink JSON tool-call bodies, command output, and diffs with exact tiktoken accounting and a typed safety report."
-keywords = ["llm", "tokenizer", "compression", "tiktoken", "openai"]
+keywords = ["llm", "tokenizer", "compression", "tiktoken", "token-optimization"]
 categories = ["compression", "text-processing"]
 
 [dependencies]

{tokenfold-0.3.3 → tokenfold-0.3.4}/crates/tokenfold-py/Cargo.toml

@@ -16,7 +16,7 @@ name = "tokenfold_py"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-tokenfold-core = { version = "0.3.3", path = "../tokenfold-core" }
+tokenfold-core = { version = "0.3.4", path = "../tokenfold-core" }
 serde_json = { workspace = true }
 pyo3 = { version = "0.29.0", features = ["abi3-py39"] }
 

{tokenfold-0.3.3 → tokenfold-0.3.4}/pyproject.toml

@@ -8,11 +8,21 @@ description = "Token-compression toolkit for LLM payloads (Python binding over t
 readme = "README.md"
 requires-python = ">=3.9"
 license = { file = "LICENSE" }
+keywords = ["llm", "tokens", "compression", "tokenizer", "openai", "json", "prompt-engineering", "cli", "token-optimization"]
 classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
     "Programming Language :: Rust",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: 3 :: Only",
     "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Text Processing",
 ]
 dynamic = ["version"]