tokenfold 0.3.2__tar.gz → 0.3.4__tar.gz

{tokenfold-0.3.2 → tokenfold-0.3.4}/Cargo.lock

@@ -1013,7 +1013,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-adapters"
-version = "0.3.2"
+version = "0.3.4"
 dependencies = [
  "serde",
  "serde_json",
@@ -1022,7 +1022,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-admin"
-version = "0.3.2"
+version = "0.3.4"
 dependencies = [
  "ed25519-dalek",
  "serde",
@@ -1032,7 +1032,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-cli"
-version = "0.3.2"
+version = "0.3.4"
 dependencies = [
  "clap",
  "serde",
@@ -1045,7 +1045,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-core"
-version = "0.3.2"
+version = "0.3.4"
 dependencies = [
  "proptest",
  "regex",
@@ -1059,25 +1059,25 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-image"
-version = "0.3.2"
+version = "0.3.4"
 
 [[package]]
 name = "tokenfold-learn"
-version = "0.3.2"
+version = "0.3.4"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-output"
-version = "0.3.2"
+version = "0.3.4"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-proxy"
-version = "0.3.2"
+version = "0.3.4"
 dependencies = [
  "clap",
  "serde",
@@ -1089,7 +1089,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-py"
-version = "0.3.2"
+version = "0.3.4"
 dependencies = [
  "pyo3",
  "serde_json",
@@ -1098,7 +1098,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-rag"
-version = "0.3.2"
+version = "0.3.4"
 
 [[package]]
 name = "toml"

{tokenfold-0.3.2 → tokenfold-0.3.4}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/tokenfold-core", "crates/tokenfold-py"]
 
 [workspace.package]
-version = "0.3.2"
+version = "0.3.4"
 edition = "2024"
 rust-version = "1.97"
 license = "Apache-2.0"

tokenfold-0.3.2/README.md → tokenfold-0.3.4/PKG-INFO

@@ -1,14 +1,38 @@
+Metadata-Version: 2.4
+Name: tokenfold
+Version: 0.3.4
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing
+License-File: LICENSE
+Summary: Token-compression toolkit for LLM payloads (Python binding over the tokenfold-core Rust crate)
+Keywords: llm,tokens,compression,tokenizer,openai,json,prompt-engineering,cli,token-optimization
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+
 <div align="center">
 
 # TOKENFOLD
 
 **Send less noise. Fit more context. Pay for fewer input tokens.**
 
-Local, provider-neutral compression for prompts, tool schemas, JSON, logs, and diffs.
+**46–68% fewer tokens on JSON & schemas · deterministic · reversible · every call audited**
+
+CLI · Python · TypeScript · proxy · MCP · local-first · provider-neutral
 
 [![CI](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=tests&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![Coverage](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=coverage&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![GitHub Release](https://img.shields.io/github/v/release/snchimata/tokenfold?logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/releases/latest) [![PyPI](https://img.shields.io/pypi/v/tokenfold?label=PyPI&style=flat-square)](https://pypi.org/project/tokenfold/) [![npm](https://img.shields.io/npm/v/tokenfold?label=npm&logo=npm&style=flat-square)](https://www.npmjs.com/package/tokenfold) [![Rust](https://img.shields.io/crates/v/tokenfold-core?label=Rust&style=flat-square)](https://docs.rs/crate/tokenfold-core/latest) [![License](https://img.shields.io/badge/license-Apache--2.0-blue?style=flat-square)](LICENSE)
 
-[Quick start](#quick-start) · [Why tokenfold](#why-tokenfold) · [Integrations](#pick-your-integration) · [Benchmarks](#reproduce-the-numbers)
+[Quick start](#quick-start) · [Why tokenfold](#why-tokenfold) · [Integrations](#pick-your-integration) · [Benchmarks](#reproduce-the-numbers) · [Select model](#tokenfold-select)
 
 </div>
 
@@ -26,6 +50,22 @@ JSON-data results are lossless. The schema benchmark preserves required
 fields and descriptions while trimming redundant examples. Repetitive,
 structured data benefits most.
 
+## What it does
+
+- **CLI** — `tokenfold compress` / `inspect` files or stdin, and compare
+  payloads with `tokenfold diff`; `tokenfold wrap -- <command>` compresses a
+  command's output in place.
+- **Library** — the same Rust engine and receipt shape from `pip install tokenfold`
+  or `npm install tokenfold`, so behavior never drifts between languages.
+- **Proxy** — `tokenfold-proxy` sits in front of your provider, compresses
+  requests, and streams responses through untouched.
+- **MCP server** — `tokenfold mcp serve` exposes compress/inspect/retrieve/stats
+  to any MCP-compatible agent or editor.
+- **Receipts, not guesses** — every call returns exact token counts, the
+  transforms it applied, and any warnings, so you can audit what changed.
+- **Local and lightweight** — one static Rust binary runs on your machine or
+  infrastructure, with no hosted service or additional data processor.
+
 ## Quick start
 
 Install the interface that fits your stack:
@@ -37,14 +77,16 @@ cargo add tokenfold-core    # Rust library
 cargo install tokenfold-cli # Rust CLI
 ```
 
-Or download the `v0.3.1` CLI for Linux, macOS, or Windows from
-[GitHub Releases](https://github.com/snchimata/tokenfold/releases/tag/v0.3.1),
+Or download the CLI for Linux, macOS, or Windows from
+[GitHub Releases](https://github.com/snchimata/tokenfold/releases/latest),
 then verify it with the adjacent `.sha256` file.
 
-Or install the Python package:
+Preview the savings without changing the input, then write the compressed
+payload when you are ready:
 
 ```bash
-pip install tokenfold
+tokenfold inspect payload.json --format json
+tokenfold compress payload.json --format json --output payload.compact.json
 ```
 
 Compress an OpenAI-style request before sending it to your provider:
@@ -129,6 +171,25 @@ messages · schemas · JSON · logs · diffs
                     └───────────▶ compressed payload + receipt
 ```
 
+## When to use tokenfold · when to look elsewhere
+
+**Good fit if you...**
+
+- want an exact, auditable receipt for every call instead of an estimate
+- need lossless guarantees on structured data — JSON, schemas, tool
+  arguments — before it reaches a billing meter
+- want a portable static binary with deterministic behavior and a small
+  operational footprint
+- are shrinking logs, diffs, and repetitive tool output before they hit an
+  LLM, a log store, or a queue
+
+**Look elsewhere if you...**
+
+- need query-aware summarization of unstructured prose — that calls for a
+  model, and tokenfold stays deterministic on purpose
+- want a hosted API to call — tokenfold runs entirely on your machine or
+  your infrastructure; there is nothing to sign up for
+
 ## What it improves
 
 | Workload | User benefit |
@@ -232,19 +293,60 @@ cargo run --release --locked -p tokenfold-cli -- \
 The sample reports 382 → 206 estimated tokens, a **46.1% reduction**. Ragged
 or compact inputs may save little; Tokenfold reports that result honestly.
 
-## Status
+## Tokenfold Select
 
-Version **0.3.1** is live on every supported distribution channel:
+**When structural compression ends, rank what matters.**
 
-| Ecosystem | Published packages | Artifacts |
+Tokenfold Core removes structural waste. [Tokenfold Select][tokenfold-select]
+is its optional, query-aware companion: a LoRA-fine-tuned model that ranks
+text spans by relevance before you assemble a smaller context.
+
+| | Tokenfold Core | Tokenfold Select |
 | --- | --- | --- |
-| Python | [`tokenfold==0.3.1`](https://pypi.org/project/tokenfold/0.3.1/) | Source distribution and ABI3 wheels for Linux x64/ARM64, macOS x64/ARM64, and Windows x64 |
-| TypeScript | [`tokenfold@0.3.1`](https://www.npmjs.com/package/tokenfold/v/0.3.1) | Typed Node.js API plus the matching `@tokenfold/cli-*` native package, with no post-install download or local Rust build |
-| Rust | [`tokenfold-core`](https://crates.io/crates/tokenfold-core/0.3.1), [`tokenfold-output`](https://crates.io/crates/tokenfold-output/0.3.1), [`tokenfold-learn`](https://crates.io/crates/tokenfold-learn/0.3.1), and [`tokenfold-cli`](https://crates.io/crates/tokenfold-cli/0.3.1) | Source packages published in dependency order |
+| Best at | JSON, schemas, logs, and diffs | Query-conditioned span ranking |
+| Runtime | Static Rust binary | Granite reranker + LoRA adapter |
+| Contract | Compressed payload + auditable receipt | Ranking logits only |
+
+Select is deliberately separate from the CLI and libraries. That separation
+lets each tool do one job well: Core provides deterministic transforms and
+auditable receipts; Select adds query-aware ranking when relevance matters.
+Your allocator still owns required-content retention and the hard token
+budget.
 
-[GitHub release `v0.3.1`](https://github.com/snchimata/tokenfold/releases/tag/v0.3.1)
-provides SHA-256-checksummed (not signed) CLI binaries for Linux, macOS, and
-Windows, plus CycloneDX SBOMs.
+<details>
+<summary><strong>Python: load the model and score spans</strong></summary>
+
+```python
+from pathlib import Path
+
+import torch
+from huggingface_hub import snapshot_download
+from peft import PeftModel
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+base_id = "ibm-granite/granite-embedding-reranker-english-r2"
+repo_dir = Path(snapshot_download("snchimata/tokenfold-select"))
+adapter_dir = repo_dir / "adapter"
+tok = AutoTokenizer.from_pretrained(adapter_dir)
+base = AutoModelForSequenceClassification.from_pretrained(base_id, dtype=torch.float32)
+model = PeftModel.from_pretrained(base, adapter_dir).eval()
+
+def score(query: str, spans: list[str]) -> list[float]:
+    if not spans:
+        return []
+    enc = tok([query] * len(spans), spans, padding=True, truncation=True,
+              max_length=8192, return_tensors="pt")
+    with torch.no_grad():
+        out = model(input_ids=enc["input_ids"], attention_mask=enc["attention_mask"])
+        return out.logits.view(-1).float().tolist()
+```
+
+</details>
+
+See the [Tokenfold Select model card][tokenfold-select] for setup, evaluation,
+training data, and limitations.
+
+[tokenfold-select]: https://huggingface.co/snchimata/tokenfold-select
 
 ## Contributing
 
@@ -270,3 +372,8 @@ receipt, and see how many tokens your application can stop sending today.
 ```bash
 pip install tokenfold
 ```
+
+If tokenfold earns a place in your stack, a ⭐ on
+[GitHub](https://github.com/snchimata/tokenfold) helps the next team
+find it.
+

tokenfold-0.3.2/PKG-INFO → tokenfold-0.3.4/README.md

@@ -1,26 +1,16 @@
-Metadata-Version: 2.4
-Name: tokenfold
-Version: 0.3.2
-Classifier: Programming Language :: Rust
-Classifier: Programming Language :: Python :: Implementation :: CPython
-Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.9
-License-File: LICENSE
-Summary: Token-compression toolkit for LLM payloads (Python binding over the tokenfold-core Rust crate)
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
-
 <div align="center">
 
 # TOKENFOLD
 
 **Send less noise. Fit more context. Pay for fewer input tokens.**
 
-Local, provider-neutral compression for prompts, tool schemas, JSON, logs, and diffs.
+**46–68% fewer tokens on JSON & schemas · deterministic · reversible · every call audited**
+
+CLI · Python · TypeScript · proxy · MCP · local-first · provider-neutral
 
 [![CI](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=tests&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![Coverage](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=coverage&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![GitHub Release](https://img.shields.io/github/v/release/snchimata/tokenfold?logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/releases/latest) [![PyPI](https://img.shields.io/pypi/v/tokenfold?label=PyPI&style=flat-square)](https://pypi.org/project/tokenfold/) [![npm](https://img.shields.io/npm/v/tokenfold?label=npm&logo=npm&style=flat-square)](https://www.npmjs.com/package/tokenfold) [![Rust](https://img.shields.io/crates/v/tokenfold-core?label=Rust&style=flat-square)](https://docs.rs/crate/tokenfold-core/latest) [![License](https://img.shields.io/badge/license-Apache--2.0-blue?style=flat-square)](LICENSE)
 
-[Quick start](#quick-start) · [Why tokenfold](#why-tokenfold) · [Integrations](#pick-your-integration) · [Benchmarks](#reproduce-the-numbers)
+[Quick start](#quick-start) · [Why tokenfold](#why-tokenfold) · [Integrations](#pick-your-integration) · [Benchmarks](#reproduce-the-numbers) · [Select model](#tokenfold-select)
 
 </div>
 
@@ -38,6 +28,22 @@ JSON-data results are lossless. The schema benchmark preserves required
 fields and descriptions while trimming redundant examples. Repetitive,
 structured data benefits most.
 
+## What it does
+
+- **CLI** — `tokenfold compress` / `inspect` files or stdin, and compare
+  payloads with `tokenfold diff`; `tokenfold wrap -- <command>` compresses a
+  command's output in place.
+- **Library** — the same Rust engine and receipt shape from `pip install tokenfold`
+  or `npm install tokenfold`, so behavior never drifts between languages.
+- **Proxy** — `tokenfold-proxy` sits in front of your provider, compresses
+  requests, and streams responses through untouched.
+- **MCP server** — `tokenfold mcp serve` exposes compress/inspect/retrieve/stats
+  to any MCP-compatible agent or editor.
+- **Receipts, not guesses** — every call returns exact token counts, the
+  transforms it applied, and any warnings, so you can audit what changed.
+- **Local and lightweight** — one static Rust binary runs on your machine or
+  infrastructure, with no hosted service or additional data processor.
+
 ## Quick start
 
 Install the interface that fits your stack:
@@ -49,14 +55,16 @@ cargo add tokenfold-core    # Rust library
 cargo install tokenfold-cli # Rust CLI
 ```
 
-Or download the `v0.3.1` CLI for Linux, macOS, or Windows from
-[GitHub Releases](https://github.com/snchimata/tokenfold/releases/tag/v0.3.1),
+Or download the CLI for Linux, macOS, or Windows from
+[GitHub Releases](https://github.com/snchimata/tokenfold/releases/latest),
 then verify it with the adjacent `.sha256` file.
 
-Or install the Python package:
+Preview the savings without changing the input, then write the compressed
+payload when you are ready:
 
 ```bash
-pip install tokenfold
+tokenfold inspect payload.json --format json
+tokenfold compress payload.json --format json --output payload.compact.json
 ```
 
 Compress an OpenAI-style request before sending it to your provider:
@@ -141,6 +149,25 @@ messages · schemas · JSON · logs · diffs
                     └───────────▶ compressed payload + receipt
 ```
 
+## When to use tokenfold · when to look elsewhere
+
+**Good fit if you...**
+
+- want an exact, auditable receipt for every call instead of an estimate
+- need lossless guarantees on structured data — JSON, schemas, tool
+  arguments — before it reaches a billing meter
+- want a portable static binary with deterministic behavior and a small
+  operational footprint
+- are shrinking logs, diffs, and repetitive tool output before they hit an
+  LLM, a log store, or a queue
+
+**Look elsewhere if you...**
+
+- need query-aware summarization of unstructured prose — that calls for a
+  model, and tokenfold stays deterministic on purpose
+- want a hosted API to call — tokenfold runs entirely on your machine or
+  your infrastructure; there is nothing to sign up for
+
 ## What it improves
 
 | Workload | User benefit |
@@ -244,19 +271,60 @@ cargo run --release --locked -p tokenfold-cli -- \
 The sample reports 382 → 206 estimated tokens, a **46.1% reduction**. Ragged
 or compact inputs may save little; Tokenfold reports that result honestly.
 
-## Status
+## Tokenfold Select
+
+**When structural compression ends, rank what matters.**
 
-Version **0.3.1** is live on every supported distribution channel:
+Tokenfold Core removes structural waste. [Tokenfold Select][tokenfold-select]
+is its optional, query-aware companion: a LoRA-fine-tuned model that ranks
+text spans by relevance before you assemble a smaller context.
 
-| Ecosystem | Published packages | Artifacts |
+| | Tokenfold Core | Tokenfold Select |
 | --- | --- | --- |
-| Python | [`tokenfold==0.3.1`](https://pypi.org/project/tokenfold/0.3.1/) | Source distribution and ABI3 wheels for Linux x64/ARM64, macOS x64/ARM64, and Windows x64 |
-| TypeScript | [`tokenfold@0.3.1`](https://www.npmjs.com/package/tokenfold/v/0.3.1) | Typed Node.js API plus the matching `@tokenfold/cli-*` native package, with no post-install download or local Rust build |
-| Rust | [`tokenfold-core`](https://crates.io/crates/tokenfold-core/0.3.1), [`tokenfold-output`](https://crates.io/crates/tokenfold-output/0.3.1), [`tokenfold-learn`](https://crates.io/crates/tokenfold-learn/0.3.1), and [`tokenfold-cli`](https://crates.io/crates/tokenfold-cli/0.3.1) | Source packages published in dependency order |
+| Best at | JSON, schemas, logs, and diffs | Query-conditioned span ranking |
+| Runtime | Static Rust binary | Granite reranker + LoRA adapter |
+| Contract | Compressed payload + auditable receipt | Ranking logits only |
+
+Select is deliberately separate from the CLI and libraries. That separation
+lets each tool do one job well: Core provides deterministic transforms and
+auditable receipts; Select adds query-aware ranking when relevance matters.
+Your allocator still owns required-content retention and the hard token
+budget.
+
+<details>
+<summary><strong>Python: load the model and score spans</strong></summary>
+
+```python
+from pathlib import Path
+
+import torch
+from huggingface_hub import snapshot_download
+from peft import PeftModel
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+base_id = "ibm-granite/granite-embedding-reranker-english-r2"
+repo_dir = Path(snapshot_download("snchimata/tokenfold-select"))
+adapter_dir = repo_dir / "adapter"
+tok = AutoTokenizer.from_pretrained(adapter_dir)
+base = AutoModelForSequenceClassification.from_pretrained(base_id, dtype=torch.float32)
+model = PeftModel.from_pretrained(base, adapter_dir).eval()
+
+def score(query: str, spans: list[str]) -> list[float]:
+    if not spans:
+        return []
+    enc = tok([query] * len(spans), spans, padding=True, truncation=True,
+              max_length=8192, return_tensors="pt")
+    with torch.no_grad():
+        out = model(input_ids=enc["input_ids"], attention_mask=enc["attention_mask"])
+        return out.logits.view(-1).float().tolist()
+```
+
+</details>
+
+See the [Tokenfold Select model card][tokenfold-select] for setup, evaluation,
+training data, and limitations.
 
-[GitHub release `v0.3.1`](https://github.com/snchimata/tokenfold/releases/tag/v0.3.1)
-provides SHA-256-checksummed (not signed) CLI binaries for Linux, macOS, and
-Windows, plus CycloneDX SBOMs.
+[tokenfold-select]: https://huggingface.co/snchimata/tokenfold-select
 
 ## Contributing
 
@@ -283,3 +351,6 @@ receipt, and see how many tokens your application can stop sending today.
 pip install tokenfold
 ```
 
+If tokenfold earns a place in your stack, a ⭐ on
+[GitHub](https://github.com/snchimata/tokenfold) helps the next team
+find it.

{tokenfold-0.3.2 → tokenfold-0.3.4}/crates/tokenfold-core/Cargo.toml

@@ -6,7 +6,7 @@ rust-version.workspace = true
 license.workspace = true
 repository.workspace = true
 description = "Token-aware compression for LLM payloads: shrink JSON tool-call bodies, command output, and diffs with exact tiktoken accounting and a typed safety report."
-keywords = ["llm", "tokenizer", "compression", "tiktoken", "openai"]
+keywords = ["llm", "tokenizer", "compression", "tiktoken", "token-optimization"]
 categories = ["compression", "text-processing"]
 
 [dependencies]

{tokenfold-0.3.2 → tokenfold-0.3.4}/crates/tokenfold-core/src/modes.rs

@@ -15,6 +15,7 @@ pub enum TransformId {
     JsonFieldFold,
     JsonValueDict,
     SchemaCompaction,
+    LogFieldFold,
     LogCompaction,
     DiffCompaction,
 }
@@ -26,6 +27,7 @@ impl TransformId {
             TransformId::JsonFieldFold => "json_field_fold",
             TransformId::JsonValueDict => "json_value_dict",
             TransformId::SchemaCompaction => "schema_compaction",
+            TransformId::LogFieldFold => "log_field_fold",
             TransformId::LogCompaction => "log_compaction",
             TransformId::DiffCompaction => "diff_compaction",
         }
@@ -140,6 +142,25 @@ pub static ALL_ENTRIES: &[ModeEntry] = &[
         task_scopes: &[TaskScope::All],
         applicable_formats: &[InputFormat::OpenAiJson, InputFormat::AnthropicJson],
     },
+    // log_field_fold (v0.4): reversible columnar fold of TEMPLATED log lines — the log-line
+    // analogue of json_field_fold (emit each shared line skeleton once + per-line captured fields).
+    // Lossless (round-trip gated in the pipeline), so max_ratio is unrestricted (1.0), but like
+    // log_compaction it restructures what the model sees, so it stays out of Conservative and ships
+    // behind --experimental until its fidelity gate is green (the same path json_field_fold and
+    // log_compaction took). Runs before the lossy log_compaction (lossless-before-lossy ordering).
+    ModeEntry {
+        transform_id: TransformId::LogFieldFold,
+        version: "1.0.0",
+        conservative_enabled: false,
+        balanced_enabled: false,
+        aggressive_enabled: false,
+        experimental: true,
+        max_ratio_conservative: 0.0,
+        max_ratio_balanced: 1.0,
+        max_ratio_aggressive: 1.0,
+        task_scopes: &[TaskScope::All],
+        applicable_formats: &[InputFormat::PlainText, InputFormat::CommandOutput],
+    },
     ModeEntry {
         transform_id: TransformId::LogCompaction,
         version: "1.0.0",

{tokenfold-0.3.2 → tokenfold-0.3.4}/crates/tokenfold-core/src/pipeline.rs

@@ -366,6 +366,10 @@ fn apply_single_transform(
             // not that Phase 2 ship distinct values per mode).
             transforms::schema::compact_schema(bytes, 1).map_err(|e| e.to_string())
         }
+        TransformId::LogFieldFold => {
+            let text = std::str::from_utf8(bytes).map_err(|e| e.to_string())?;
+            Ok(transforms::log_fold::fold_log(text).into_bytes())
+        }
         TransformId::LogCompaction => {
             let text = std::str::from_utf8(bytes).map_err(|e| e.to_string())?;
             Ok(transforms::logs::compact(text, false).into_bytes())
@@ -423,6 +427,13 @@ fn validate_safety(
                 }
             }
         }
+        // log_field_fold restructures templated log lines into a columnar form, so its safety
+        // invariant is exact reversibility: unfolding the output must reproduce the input bytes.
+        TransformId::LogFieldFold => {
+            if !transforms::log_fold::round_trips(before, after) {
+                return false;
+            }
+        }
         TransformId::LogCompaction | TransformId::DiffCompaction => {}
     }
     safety::protected_segments_present(protected, after)

tokenfold-0.3.4/crates/tokenfold-core/src/transforms/log_fold.rs

@@ -0,0 +1,296 @@
+//! `log_field_fold` transform (canonical id `"log_field_fold"`, v1.0.0).
+//!
+//! Content-aware, **losslessly reversible** structural compression for line-oriented text
+//! (`InputFormat::PlainText` / `CommandOutput`). It is the log-line analogue of
+//! [`json_field_fold`](super::json_fold): where that folds arrays of homogeneous *objects* by
+//! emitting each repeated key once, this folds runs of *templated* log lines by emitting each
+//! repeated line **template** once.
+//!
+//! Most log lines share a fixed skeleton and vary only in a few fields (timestamps, ids, counts):
+//!
+//! ```text
+//! 2026-07-01T10:05:11Z req=req-0311 status=200 ms=41
+//! 2026-07-01T10:05:12Z req=req-0312 status=200 ms=44
+//! ```
+//!
+//! Replacing each variable token (a run of digits, or a `0x…` hex literal) with a placeholder
+//! yields one shared template `2026-\x00-\x00T\x00:\x00:\x00Z req=req-\x00 status=\x00 ms=\x00`
+//! plus a per-line tuple of the captured values. The fold emits every distinct template once and
+//! a compact row per original line (`template_id` + its captured values), so the skeleton text is
+//! paid for once instead of per line — a large win on repetitive logs, unlike
+//! [`log_compaction`](super::logs) which only collapses *identical adjacent* lines.
+//!
+//! It is a pure structural rewrite: `unfold_log` reconstructs the original bytes exactly, and the
+//! pipeline gates adoption on that round-trip ([`round_trips`]) — a fold that would ever lose data
+//! (or a genuine input that happens to look like our framing) is rolled back rather than emitted.
+
+/// Canonical transform id, as registered with the pipeline.
+pub const TRANSFORM_ID: &str = "log_field_fold";
+
+/// Semantic version of this transform's output behavior.
+pub const TRANSFORM_VERSION: &str = "1.0.0";
+
+/// First line of the folded blob. Collision-unlikely in real logs; any actual collision is caught
+/// by the pipeline's round-trip safety gate, so it never corrupts data.
+const HEADER: &str = "__tf_logfold1__";
+
+/// Placeholder standing in for a captured variable token inside a template.
+const PH: char = '\u{0}';
+
+/// Minimum number of lines worth folding, and the maximum distinct-template fraction below which a
+/// fold can pay off. Both are cheap early-outs; the pipeline also rolls back any net token
+/// regression, so these are heuristics, not the correctness boundary.
+const MIN_LINES: usize = 3;
+
+use regex::Regex;
+use std::sync::OnceLock;
+
+/// Matches a maximal variable token: a `0x…` hex literal or a run of decimal digits. Linear-time
+/// (no backtracking); see `deny.toml`.
+fn var_pattern() -> &'static Regex {
+    static RE: OnceLock<Regex> = OnceLock::new();
+    RE.get_or_init(|| Regex::new(r"0x[0-9a-fA-F]+|[0-9]+").expect("var_pattern is a valid literal"))
+}
+
+/// Replaces each variable token in `segment` with [`PH`], returning the template and the captured
+/// tokens in left-to-right order. `caps` never contain [`PH`], `\n`, or spaces (the pattern only
+/// matches `[0-9a-fA-F]`/`x`), which keeps the row serialization below unambiguous.
+fn templatize(segment: &str) -> (String, Vec<&str>) {
+    let re = var_pattern();
+    let mut template = String::with_capacity(segment.len());
+    let mut caps = Vec::new();
+    let mut last = 0;
+    for m in re.find_iter(segment) {
+        template.push_str(&segment[last..m.start()]);
+        template.push(PH);
+        caps.push(m.as_str());
+        last = m.end();
+    }
+    template.push_str(&segment[last..]);
+    (template, caps)
+}
+
+/// Folds runs of templated lines in `input` into a header + a JSON array of distinct templates +
+/// one compact row (`template_id` then captured values) per original line. Returns `input`
+/// unchanged when folding cannot help (too few lines, no shared templates, or a line contains the
+/// placeholder char). Never panics.
+pub fn fold_log(input: &str) -> String {
+    if input.is_empty() || input.contains(PH) {
+        return input.to_string();
+    }
+    // split_inclusive keeps each line's trailing '\n' as part of the segment, so reassembly is
+    // byte-exact (CRLF, blank lines, and a missing final newline are all preserved).
+    let segments: Vec<&str> = input.split_inclusive('\n').collect();
+    if segments.len() < MIN_LINES {
+        return input.to_string();
+    }
+
+    let mut templates: Vec<String> = Vec::new();
+    let mut ids: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
+    let mut rows: Vec<(usize, Vec<&str>)> = Vec::with_capacity(segments.len());
+    for seg in &segments {
+        let (template, caps) = templatize(seg);
+        let id = *ids.entry(template.clone()).or_insert_with(|| {
+            templates.push(template);
+            templates.len() - 1
+        });
+        rows.push((id, caps));
+    }
+
+    // Only worth it if templates are actually shared (skeleton text amortizes). If every line is
+    // its own template there is nothing to save; let the pipeline keep the original.
+    if templates.len() >= segments.len() {
+        return input.to_string();
+    }
+
+    let mut out = String::with_capacity(input.len() / 2);
+    out.push_str(HEADER);
+    out.push('\n');
+    out.push_str(&serde_json::to_string(&templates).expect("Vec<String> always serializes"));
+    for (id, caps) in &rows {
+        out.push('\n');
+        out.push_str(&id.to_string());
+        for cap in caps {
+            out.push(' ');
+            out.push_str(cap);
+        }
+    }
+    out
+}
+
+/// Inverse of [`fold_log`]: expands a folded blob back to the original text. Returns `input`
+/// unchanged if it is not a well-formed folded blob (so a genuine log that merely starts with the
+/// header is not mangled — the pipeline's [`round_trips`] gate makes that safe either way).
+pub fn unfold_log(input: &str) -> String {
+    match try_unfold(input) {
+        Some(s) => s,
+        None => input.to_string(),
+    }
+}
+
+fn try_unfold(input: &str) -> Option<String> {
+    let mut lines = input.split('\n');
+    if lines.next()? != HEADER {
+        return None;
+    }
+    let templates: Vec<String> = serde_json::from_str(lines.next()?).ok()?;
+    let mut out = String::with_capacity(input.len() * 2);
+    for row in lines {
+        let mut fields = row.split(' ');
+        let id: usize = fields.next()?.parse().ok()?;
+        let template = templates.get(id)?;
+        let mut caps = fields;
+        // Interleave the template's constant segments with the captured values. The number of
+        // placeholders must equal the number of captures, else this isn't our framing.
+        let mut parts = template.split(PH);
+        out.push_str(parts.next()?);
+        for part in parts {
+            out.push_str(caps.next()?);
+            out.push_str(part);
+        }
+        if caps.next().is_some() {
+            return None; // more captures than placeholders → malformed
+        }
+    }
+    Some(out)
+}
+
+/// True iff unfolding `after` reproduces `before` exactly. The pipeline's safety gate for this
+/// transform — folding is only adopted when this holds.
+pub fn round_trips(before: &[u8], after: &[u8]) -> bool {
+    let (Ok(before_s), Ok(after_s)) = (std::str::from_utf8(before), std::str::from_utf8(after))
+    else {
+        return false;
+    };
+    unfold_log(after_s) == before_s
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn assert_lossless(input: &str) {
+        let folded = fold_log(input);
+        assert!(
+            round_trips(input.as_bytes(), folded.as_bytes()),
+            "round_trips() rejected the fold of {input:?}"
+        );
+        assert_eq!(
+            unfold_log(&folded),
+            input,
+            "unfold != original for {input:?}"
+        );
+    }
+
+    #[test]
+    fn folds_templated_log_and_emits_skeleton_once() {
+        // A realistically-sized run of one skeleton, so the shared template amortizes: the header +
+        // template JSON is paid once, the per-line skeleton text is not.
+        let input: String = (0..40)
+            .map(|i| format!("req=req-{i:04} status=200 ms={}\n", 30 + i % 20))
+            .collect();
+        let folded = fold_log(&input);
+        assert!(
+            folded.starts_with(HEADER),
+            "expected folded form, got {folded:?}"
+        );
+        // the shared skeleton word "status" is emitted once (in the single template), not per line.
+        assert_eq!(folded.matches("status").count(), 1);
+        assert!(
+            folded.len() < input.len(),
+            "fold ({}) not smaller than input ({})",
+            folded.len(),
+            input.len()
+        );
+        assert_lossless(&input);
+    }
+
+    #[test]
+    fn preserves_crlf_blank_lines_and_missing_final_newline() {
+        assert_lossless("a=1\r\na=2\r\na=3\r\n");
+        assert_lossless("x=1\n\nx=2\n\nx=3\n");
+        assert_lossless("x=1\nx=2\nx=3"); // no trailing newline
+    }
+
+    #[test]
+    fn no_shared_templates_is_left_unchanged() {
+        // every line a distinct skeleton → nothing to fold.
+        let input = "alpha\nbeta gamma\ndelta epsilon zeta\n";
+        assert_eq!(fold_log(input), input);
+    }
+
+    #[test]
+    fn fewer_than_min_lines_is_left_unchanged() {
+        let input = "req=1 ok\nreq=2 ok\n";
+        assert_eq!(fold_log(input), input);
+    }
+
+    #[test]
+    fn empty_input_is_a_noop() {
+        assert_eq!(fold_log(""), "");
+        assert_eq!(unfold_log(""), "");
+    }
+
+    #[test]
+    fn lines_with_no_variable_tokens_still_fold_when_identical() {
+        // three identical constant lines share one (capture-less) template.
+        let input = "heartbeat ok\nheartbeat ok\nheartbeat ok\n";
+        assert_lossless(input);
+    }
+
+    #[test]
+    fn hex_and_decimal_tokens_both_captured() {
+        let input = "addr=0x1f val=10\naddr=0x2a val=20\naddr=0x3b val=30\n";
+        assert_lossless(input);
+        // the "addr=" / "val=" skeleton is shared → one template.
+        assert!(fold_log(input).starts_with(HEADER));
+    }
+
+    #[test]
+    fn input_containing_the_placeholder_char_is_not_folded() {
+        let input = "a\u{0}b\na\u{0}c\na\u{0}d\n";
+        assert_eq!(fold_log(input), input);
+    }
+
+    #[test]
+    fn genuine_input_shaped_like_the_header_round_trips_safely() {
+        // If real text starts with our header, unfold must not silently corrupt it: fold_log of a
+        // 2-line input is a no-op (< MIN_LINES), and round_trips gates the pipeline regardless.
+        let input = "__tf_logfold1__\n[\"x\"]\n0 boom";
+        let folded = fold_log(input);
+        // Either it wasn't folded (identity) or, if it were, the pipeline would only adopt it when
+        // round_trips holds — so the original is always recoverable either way.
+        assert_eq!(unfold_log(&folded), input);
+    }
+
+    use proptest::prelude::*;
+
+    // Lines built from a small alphabet of skeletons + digit/hex fields, so templates recur (the
+    // regime the fold targets) while still exercising CRLF, blanks, and missing final newlines.
+    fn arb_log() -> impl Strategy<Value = String> {
+        let line = prop_oneof![
+            (0u32..999u32).prop_map(|n| format!("req={n} status=200")),
+            (0u32..999u32).prop_map(|n| format!("addr=0x{n:x} ok")),
+            Just("heartbeat".to_string()),
+            "[a-z ]{0,8}".prop_map(|s| s),
+        ];
+        (prop::collection::vec(line, 0..40), any::<bool>()).prop_map(|(lines, trailing)| {
+            let mut s = lines.join("\n");
+            if trailing && !s.is_empty() {
+                s.push('\n');
+            }
+            s
+        })
+    }
+
+    proptest! {
+        // Core safety guarantee: folding never loses data. For ANY log-ish input, unfolding the
+        // folded form reproduces it exactly, and round_trips() (the pipeline's gate) agrees.
+        #[test]
+        fn fold_then_unfold_is_the_identity(input in arb_log()) {
+            let folded = fold_log(&input);
+            prop_assert!(round_trips(input.as_bytes(), folded.as_bytes()));
+            prop_assert_eq!(unfold_log(&folded), input);
+        }
+    }
+}

{tokenfold-0.3.2 → tokenfold-0.3.4}/crates/tokenfold-core/src/transforms/mod.rs

@@ -6,6 +6,7 @@ pub mod diff;
 pub mod json;
 pub mod json_dict;
 pub mod json_fold;
+pub mod log_fold;
 pub mod logs;
 pub mod redaction;
 pub mod schema;

{tokenfold-0.3.2 → tokenfold-0.3.4}/crates/tokenfold-py/Cargo.toml

@@ -16,7 +16,7 @@ name = "tokenfold_py"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-tokenfold-core = { version = "0.3.2", path = "../tokenfold-core" }
+tokenfold-core = { version = "0.3.4", path = "../tokenfold-core" }
 serde_json = { workspace = true }
 pyo3 = { version = "0.29.0", features = ["abi3-py39"] }
 

{tokenfold-0.3.2 → tokenfold-0.3.4}/pyproject.toml

@@ -8,11 +8,21 @@ description = "Token-compression toolkit for LLM payloads (Python binding over t
 readme = "README.md"
 requires-python = ">=3.9"
 license = { file = "LICENSE" }
+keywords = ["llm", "tokens", "compression", "tokenizer", "openai", "json", "prompt-engineering", "cli", "token-optimization"]
 classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
     "Programming Language :: Rust",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: 3 :: Only",
     "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Text Processing",
 ]
 dynamic = ["version"]