tokenfold 0.3.1__tar.gz → 0.3.2__tar.gz

{tokenfold-0.3.1 → tokenfold-0.3.2}/Cargo.lock

@@ -1013,7 +1013,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-adapters"
-version = "0.3.1"
+version = "0.3.2"
 dependencies = [
  "serde",
  "serde_json",
@@ -1022,7 +1022,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-admin"
-version = "0.3.1"
+version = "0.3.2"
 dependencies = [
  "ed25519-dalek",
  "serde",
@@ -1032,7 +1032,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-cli"
-version = "0.3.1"
+version = "0.3.2"
 dependencies = [
  "clap",
  "serde",
@@ -1045,7 +1045,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-core"
-version = "0.3.1"
+version = "0.3.2"
 dependencies = [
  "proptest",
  "regex",
@@ -1059,25 +1059,25 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-image"
-version = "0.3.1"
+version = "0.3.2"
 
 [[package]]
 name = "tokenfold-learn"
-version = "0.3.1"
+version = "0.3.2"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-output"
-version = "0.3.1"
+version = "0.3.2"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-proxy"
-version = "0.3.1"
+version = "0.3.2"
 dependencies = [
  "clap",
  "serde",
@@ -1089,7 +1089,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-py"
-version = "0.3.1"
+version = "0.3.2"
 dependencies = [
  "pyo3",
  "serde_json",
@@ -1098,7 +1098,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-rag"
-version = "0.3.1"
+version = "0.3.2"
 
 [[package]]
 name = "toml"

{tokenfold-0.3.1 → tokenfold-0.3.2}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/tokenfold-core", "crates/tokenfold-py"]
 
 [workspace.package]
-version = "0.3.1"
+version = "0.3.2"
 edition = "2024"
 rust-version = "1.97"
 license = "Apache-2.0"

{tokenfold-0.3.1 → tokenfold-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tokenfold
-Version: 0.3.1
+Version: 0.3.2
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: 3 :: Only
@@ -10,7 +10,9 @@ Summary: Token-compression toolkit for LLM payloads (Python binding over the tok
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 
-# tokenfold
+<div align="center">
+
+# TOKENFOLD
 
 **Send less noise. Fit more context. Pay for fewer input tokens.**
 
@@ -20,6 +22,8 @@ Local, provider-neutral compression for prompts, tool schemas, JSON, logs, and d
 
 [Quick start](#quick-start) · [Why tokenfold](#why-tokenfold) · [Integrations](#pick-your-integration) · [Benchmarks](#reproduce-the-numbers)
 
+</div>
+
 ---
 
 ## Proven compression, not projections
@@ -79,12 +83,21 @@ the canonical compression receipt:
 ```typescript
 import { compress } from "tokenfold";
 
+const input = new TextEncoder().encode(JSON.stringify({
+  results: [
+    { id: 101, region: "us-east-1", plan: "pro" },
+    { id: 102, region: "us-east-1", plan: "pro" },
+    { id: 103, region: "us-east-1", plan: "pro" },
+  ],
+}, null, 2));
+
 const { payload, report } = await compress(input, {
   format: "json",
   mode: "balanced",
 });
 
 console.log(`saved ${report.saved_tokens} tokens`);
+console.log(new TextDecoder().decode(payload));
 ```
 
 Want to try the CLI from source? Inspect the bundled request without changing it:
@@ -96,10 +109,12 @@ cargo run --release --locked -p tokenfold-cli -- \
   inspect examples/openai_payload.json --format openai
 ```
 
+Across 100 requests with the same payload shape, those savings add up to:
+
 ```text
-json_minify          346 → 229   saved 117
-schema_compaction    229 → 213   saved  16
-TOTAL                346 → 213   saved 133 (38.4% reduction, estimated)
+json_minify          34,600 → 22,900   saved 11,700
+schema_compaction    22,900 → 21,300   saved  1,600
+TOTAL                34,600 → 21,300   saved 13,300 (38.4% reduction, estimated)
 ```
 
 ## Why tokenfold
@@ -231,13 +246,17 @@ or compact inputs may save little; Tokenfold reports that result honestly.
 
 ## Status
 
-Version **0.3.1** provides SHA-256-checksummed (not signed) CLI binaries for
-Linux, macOS, and Windows from [GitHub Releases](https://github.com/snchimata/tokenfold/releases/tag/v0.3.1),
-plus CycloneDX SBOMs. Registry packages are published to
-[PyPI](https://pypi.org/project/tokenfold/), [npm](https://www.npmjs.com/package/tokenfold),
-and [crates.io](https://crates.io/crates/tokenfold-core). npm installs the
-matching native binary package without a post-install download or local Rust
-build.
+Version **0.3.1** is live on every supported distribution channel:
+
+| Ecosystem | Published packages | Artifacts |
+| --- | --- | --- |
+| Python | [`tokenfold==0.3.1`](https://pypi.org/project/tokenfold/0.3.1/) | Source distribution and ABI3 wheels for Linux x64/ARM64, macOS x64/ARM64, and Windows x64 |
+| TypeScript | [`tokenfold@0.3.1`](https://www.npmjs.com/package/tokenfold/v/0.3.1) | Typed Node.js API plus the matching `@tokenfold/cli-*` native package, with no post-install download or local Rust build |
+| Rust | [`tokenfold-core`](https://crates.io/crates/tokenfold-core/0.3.1), [`tokenfold-output`](https://crates.io/crates/tokenfold-output/0.3.1), [`tokenfold-learn`](https://crates.io/crates/tokenfold-learn/0.3.1), and [`tokenfold-cli`](https://crates.io/crates/tokenfold-cli/0.3.1) | Source packages published in dependency order |
+
+[GitHub release `v0.3.1`](https://github.com/snchimata/tokenfold/releases/tag/v0.3.1)
+provides SHA-256-checksummed (not signed) CLI binaries for Linux, macOS, and
+Windows, plus CycloneDX SBOMs.
 
 ## Contributing
 

{tokenfold-0.3.1 → tokenfold-0.3.2}/README.md

@@ -1,4 +1,6 @@
-# tokenfold
+<div align="center">
+
+# TOKENFOLD
 
 **Send less noise. Fit more context. Pay for fewer input tokens.**
 
@@ -8,6 +10,8 @@ Local, provider-neutral compression for prompts, tool schemas, JSON, logs, and d
 
 [Quick start](#quick-start) · [Why tokenfold](#why-tokenfold) · [Integrations](#pick-your-integration) · [Benchmarks](#reproduce-the-numbers)
 
+</div>
+
 ---
 
 ## Proven compression, not projections
@@ -67,12 +71,21 @@ the canonical compression receipt:
 ```typescript
 import { compress } from "tokenfold";
 
+const input = new TextEncoder().encode(JSON.stringify({
+  results: [
+    { id: 101, region: "us-east-1", plan: "pro" },
+    { id: 102, region: "us-east-1", plan: "pro" },
+    { id: 103, region: "us-east-1", plan: "pro" },
+  ],
+}, null, 2));
+
 const { payload, report } = await compress(input, {
   format: "json",
   mode: "balanced",
 });
 
 console.log(`saved ${report.saved_tokens} tokens`);
+console.log(new TextDecoder().decode(payload));
 ```
 
 Want to try the CLI from source? Inspect the bundled request without changing it:
@@ -84,10 +97,12 @@ cargo run --release --locked -p tokenfold-cli -- \
   inspect examples/openai_payload.json --format openai
 ```
 
+Across 100 requests with the same payload shape, those savings add up to:
+
 ```text
-json_minify          346 → 229   saved 117
-schema_compaction    229 → 213   saved  16
-TOTAL                346 → 213   saved 133 (38.4% reduction, estimated)
+json_minify          34,600 → 22,900   saved 11,700
+schema_compaction    22,900 → 21,300   saved  1,600
+TOTAL                34,600 → 21,300   saved 13,300 (38.4% reduction, estimated)
 ```
 
 ## Why tokenfold
@@ -219,13 +234,17 @@ or compact inputs may save little; Tokenfold reports that result honestly.
 
 ## Status
 
-Version **0.3.1** provides SHA-256-checksummed (not signed) CLI binaries for
-Linux, macOS, and Windows from [GitHub Releases](https://github.com/snchimata/tokenfold/releases/tag/v0.3.1),
-plus CycloneDX SBOMs. Registry packages are published to
-[PyPI](https://pypi.org/project/tokenfold/), [npm](https://www.npmjs.com/package/tokenfold),
-and [crates.io](https://crates.io/crates/tokenfold-core). npm installs the
-matching native binary package without a post-install download or local Rust
-build.
+Version **0.3.1** is live on every supported distribution channel:
+
+| Ecosystem | Published packages | Artifacts |
+| --- | --- | --- |
+| Python | [`tokenfold==0.3.1`](https://pypi.org/project/tokenfold/0.3.1/) | Source distribution and ABI3 wheels for Linux x64/ARM64, macOS x64/ARM64, and Windows x64 |
+| TypeScript | [`tokenfold@0.3.1`](https://www.npmjs.com/package/tokenfold/v/0.3.1) | Typed Node.js API plus the matching `@tokenfold/cli-*` native package, with no post-install download or local Rust build |
+| Rust | [`tokenfold-core`](https://crates.io/crates/tokenfold-core/0.3.1), [`tokenfold-output`](https://crates.io/crates/tokenfold-output/0.3.1), [`tokenfold-learn`](https://crates.io/crates/tokenfold-learn/0.3.1), and [`tokenfold-cli`](https://crates.io/crates/tokenfold-cli/0.3.1) | Source packages published in dependency order |
+
+[GitHub release `v0.3.1`](https://github.com/snchimata/tokenfold/releases/tag/v0.3.1)
+provides SHA-256-checksummed (not signed) CLI binaries for Linux, macOS, and
+Windows, plus CycloneDX SBOMs.
 
 ## Contributing
 

{tokenfold-0.3.1 → tokenfold-0.3.2}/crates/tokenfold-core/src/report.rs

@@ -16,6 +16,10 @@ pub struct CompressionReport {
     pub format: String,
     pub task_scope: String,
     pub request_id: Option<String>,
+    /// F-055: staged `raw -> RTK -> tokenfold` accounting. `None` for the common
+    /// single-stage path; populated only by RTK-composed `wrap --rtk` runs.
+    #[serde(default)]
+    pub pipeline: Option<PipelineReport>,
     pub quality: Option<QualityReport>,
     pub budget: Option<BudgetReport>,
     pub cache: Option<CacheReport>,
@@ -61,6 +65,7 @@ impl CompressionReport {
             format,
             task_scope,
             request_id: None,
+            pipeline: None,
             quality: None,
             budget: None,
             cache: None,
@@ -82,6 +87,53 @@ pub struct EstimatorInfo {
     pub is_exact: bool,
 }
 
+/// F-055: separates savings and recoverability across composed stages (RTK then
+/// tokenfold) so RTK's savings are never credited to tokenfold. The top-level
+/// `original_tokens` keeps its v1 meaning — tokens *entering* `tokenfold_core`,
+/// which is the post-RTK count when composed. See INTERFACES.md §1.9 / §2.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct PipelineReport {
+    /// Pre-RTK byte count. `Some` only when a complete raw capture was observed.
+    pub raw_input_bytes: Option<usize>,
+    /// Pre-RTK token count. `Some` only when raw capture is complete.
+    pub raw_input_tokens: Option<usize>,
+    pub final_output_bytes: usize,
+    /// Equals top-level `compressed_tokens`.
+    pub final_output_tokens: usize,
+    /// Populated only when raw and final counts use the same estimator.
+    pub total_saved_tokens: Option<usize>,
+    /// `"complete"`, `"partial"`, `"unavailable"`, or `"not_applicable"`.
+    pub raw_capture: String,
+    /// `"full"`, `"tokenfold_only"`, `"none"`, or `"not_applicable"`.
+    pub upstream_recoverability: String,
+    pub stages: Vec<PipelineStageReport>,
+}
+
+/// F-055: one composed stage. Count fields are nullable because an unavailable
+/// stage or missing pre-stage capture cannot be measured honestly.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct PipelineStageReport {
+    /// `"rtk"` or `"tokenfold"`.
+    pub id: String,
+    pub version: Option<String>,
+    pub input_bytes: Option<usize>,
+    pub output_bytes: Option<usize>,
+    pub saved_bytes: Option<usize>,
+    pub input_tokens: Option<usize>,
+    pub output_tokens: Option<usize>,
+    pub saved_tokens: Option<usize>,
+    pub estimator: Option<EstimatorInfo>,
+    /// `"applied"`, `"passthrough"`, `"unavailable"`, `"incompatible"`, or `"failed"`.
+    pub status: String,
+    pub duration_ms: Option<f64>,
+    pub bypass_reason: Option<String>,
+    /// e.g. `"external:rtk@0.4.1"` or `"tokenfold_core"`.
+    pub provenance: String,
+    /// `"full"`, `"partial"`, `"none"`, or `"not_applicable"`.
+    pub recoverability: String,
+    pub evidence_ref: Option<String>,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct BudgetReport {
     pub target_tokens: Option<usize>,

{tokenfold-0.3.1 → tokenfold-0.3.2}/crates/tokenfold-py/Cargo.toml

@@ -16,7 +16,7 @@ name = "tokenfold_py"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-tokenfold-core = { version = "0.3.1", path = "../tokenfold-core" }
+tokenfold-core = { version = "0.3.2", path = "../tokenfold-core" }
 serde_json = { workspace = true }
 pyo3 = { version = "0.29.0", features = ["abi3-py39"] }