tokenfold 0.4.0__tar.gz → 0.4.1__tar.gz

{tokenfold-0.4.0 → tokenfold-0.4.1}/Cargo.lock

@@ -1013,7 +1013,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-adapters"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "serde",
  "serde_json",
@@ -1022,7 +1022,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-admin"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "ed25519-dalek",
  "serde",
@@ -1032,7 +1032,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-cli"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "clap",
  "serde",
@@ -1045,7 +1045,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-core"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "proptest",
  "regex",
@@ -1059,25 +1059,25 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-image"
-version = "0.4.0"
+version = "0.4.1"
 
 [[package]]
 name = "tokenfold-learn"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-output"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-proxy"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "clap",
  "serde",
@@ -1089,7 +1089,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-py"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "pyo3",
  "serde_json",
@@ -1098,7 +1098,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-rag"
-version = "0.4.0"
+version = "0.4.1"
 
 [[package]]
 name = "toml"

{tokenfold-0.4.0 → tokenfold-0.4.1}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/tokenfold-core", "crates/tokenfold-py"]
 
 [workspace.package]
-version = "0.4.0"
+version = "0.4.1"
 edition = "2024"
 rust-version = "1.97"
 license = "Apache-2.0"

{tokenfold-0.4.0 → tokenfold-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tokenfold
-Version: 0.4.0
+Version: 0.4.1
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
@@ -30,7 +30,7 @@ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 
 CLI · Python · TypeScript · Rust · proxy · MCP · local-first · provider-neutral
 
-[![CI](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=tests&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![Coverage](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=coverage&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![GitHub Release](https://img.shields.io/github/v/release/snchimata/tokenfold?logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/releases/latest) [![PyPI](https://img.shields.io/pypi/v/tokenfold?label=PyPI&style=flat-square)](https://pypi.org/project/tokenfold/) [![npm](https://img.shields.io/npm/v/tokenfold?label=npm&logo=npm&style=flat-square)](https://www.npmjs.com/package/tokenfold) [![Rust](https://img.shields.io/crates/v/tokenfold-core?label=Rust&style=flat-square)](https://docs.rs/crate/tokenfold-core/latest) [![License](https://img.shields.io/badge/license-Apache--2.0-blue?style=flat-square)](LICENSE)
+[![CI](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=tests&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![Coverage](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=coverage&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![GitHub Release](https://img.shields.io/github/v/release/snchimata/tokenfold?logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/releases/latest) [![PyPI](https://img.shields.io/pypi/v/tokenfold?label=PyPI&style=flat-square)](https://pypi.org/project/tokenfold/) [![npm](https://img.shields.io/npm/v/tokenfold?label=npm&logo=npm&style=flat-square)](https://www.npmjs.com/package/tokenfold) [![Rust](https://img.shields.io/crates/v/tokenfold-core?label=Rust&style=flat-square)](https://docs.rs/crate/tokenfold-core/latest) [![License](https://img.shields.io/badge/license-Apache--2.0-blue?style=flat-square)](https://github.com/snchimata/tokenfold/blob/main/LICENSE)
 
 [Measured results](#measured-results) · [Platform](#one-platform-two-engines) · [Quick start](#quick-start) · [Lossy pruning](#recoverable-lossy-pruning) · [Select model](#tokenfold-select) · [Reproduce](#reproduce-the-results)
 
@@ -144,13 +144,18 @@ print(f"saved {result.report.saved_tokens} tokens ({result.saved_pct():.1f}%)")
 `tokenfold doctor` verifies it. Trusted filters for Git, build, and test output
 are available through `tokenfold filters list`.
 
-Runnable examples:
+Runnable examples, one per surface, all under [`examples/`](examples):
 
 ```bash
-python examples/quickstart.py
-python examples/lossy_pruning.py
+python examples/quickstart.py      # Python: compress messages, request bodies, JSON data
+node examples/quickstart.mjs       # TypeScript/Node: compress, inspect, read the receipt
+cargo run -p tokenfold-core --example quickstart   # Rust: the embedded core API
+python examples/lossy_pruning.py   # CLI: opt-in recoverable pruning, end to end
 ```
 
+[`examples/quickstart.ipynb`](examples/quickstart.ipynb) is the notebook form of the
+Python quickstart, one operation per cell.
+
 ## Recoverable lossy pruning
 
 Lossless folding has a ceiling on heterogeneous arrays. `--lossy` ranks rows,
@@ -204,6 +209,27 @@ tokenfold compress examples/incident_feed.json --format json \
   --lossy heuristic --lossy-ratio 0.35 --dry-run
 ```
 
+The same flags, and the same fail-closed contract, from Python and TypeScript:
+
+```python
+from tokenfold import CompressionPolicy, InputFormat, LossyPath, compress, retrieve
+
+policy = CompressionPolicy(lossy=LossyPath.HEURISTIC, lossy_ratio=0.35)
+result = compress(feed_bytes, format=InputFormat.JSON, policy=policy)
+original = retrieve(marker["$tf_ref"]["hash"])   # any dropped row, verbatim
+```
+
+```ts
+import { compress, retrieve } from "tokenfold";
+
+const { payload, report } = await compress(feed, {
+  format: "json",
+  lossy: "heuristic",
+  lossyRatio: 0.35,
+});
+const original = await retrieve(hash); // any dropped row, verbatim
+```
+
 <details>
 <summary><strong>Current Phase 1 constraints</strong></summary>
 
@@ -212,9 +238,8 @@ already contain retrieval markers. A filesystem failure after partial writes
 may also leave unreferenced entries until their configured TTL expires. Both
 require location-based transactional materialization before promotion.
 
-Lossy pruning is CLI-only today; Python and TypeScript remain lossless. Preview
-is a projection rather than a filesystem transaction, so a real run may keep
-more rows if storage becomes unavailable.
+Preview is a projection rather than a filesystem transaction, so a real run may
+keep more rows if storage becomes unavailable.
 
 </details>
 
@@ -308,8 +333,8 @@ cargo run --release --locked -p tokenfold-cli -- \
 ```
 
 The small bundled API response reports 382 → 206 tokens, a **46.1% lossless
-reduction**. Benchmark sources and thresholds live in [CHANGELOG.md](CHANGELOG.md)
-and [`crates/tokenfold-core/benches/THRESHOLDS.toml`](crates/tokenfold-core/benches/THRESHOLDS.toml).
+reduction**. Benchmark sources and thresholds live in [CHANGELOG.md](https://github.com/snchimata/tokenfold/blob/main/CHANGELOG.md)
+and [`crates/tokenfold-core/benches/THRESHOLDS.toml`](https://github.com/snchimata/tokenfold/blob/main/crates/tokenfold-core/benches/THRESHOLDS.toml).
 
 ## Contributing
 
@@ -326,7 +351,7 @@ cd packages/tokenfold && npm ci && npm test
 
 ## License
 
-[Apache-2.0](LICENSE)
+[Apache-2.0](https://github.com/snchimata/tokenfold/blob/main/LICENSE)
 
 ---
 

{tokenfold-0.4.0 → tokenfold-0.4.1}/README.md

@@ -8,7 +8,7 @@
 
 CLI · Python · TypeScript · Rust · proxy · MCP · local-first · provider-neutral
 
-[![CI](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=tests&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![Coverage](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=coverage&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![GitHub Release](https://img.shields.io/github/v/release/snchimata/tokenfold?logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/releases/latest) [![PyPI](https://img.shields.io/pypi/v/tokenfold?label=PyPI&style=flat-square)](https://pypi.org/project/tokenfold/) [![npm](https://img.shields.io/npm/v/tokenfold?label=npm&logo=npm&style=flat-square)](https://www.npmjs.com/package/tokenfold) [![Rust](https://img.shields.io/crates/v/tokenfold-core?label=Rust&style=flat-square)](https://docs.rs/crate/tokenfold-core/latest) [![License](https://img.shields.io/badge/license-Apache--2.0-blue?style=flat-square)](LICENSE)
+[![CI](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=tests&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![Coverage](https://img.shields.io/github/actions/workflow/status/snchimata/tokenfold/ci.yml?branch=main&label=coverage&logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/actions/workflows/ci.yml) [![GitHub Release](https://img.shields.io/github/v/release/snchimata/tokenfold?logo=github&style=flat-square)](https://github.com/snchimata/tokenfold/releases/latest) [![PyPI](https://img.shields.io/pypi/v/tokenfold?label=PyPI&style=flat-square)](https://pypi.org/project/tokenfold/) [![npm](https://img.shields.io/npm/v/tokenfold?label=npm&logo=npm&style=flat-square)](https://www.npmjs.com/package/tokenfold) [![Rust](https://img.shields.io/crates/v/tokenfold-core?label=Rust&style=flat-square)](https://docs.rs/crate/tokenfold-core/latest) [![License](https://img.shields.io/badge/license-Apache--2.0-blue?style=flat-square)](https://github.com/snchimata/tokenfold/blob/main/LICENSE)
 
 [Measured results](#measured-results) · [Platform](#one-platform-two-engines) · [Quick start](#quick-start) · [Lossy pruning](#recoverable-lossy-pruning) · [Select model](#tokenfold-select) · [Reproduce](#reproduce-the-results)
 
@@ -122,13 +122,18 @@ print(f"saved {result.report.saved_tokens} tokens ({result.saved_pct():.1f}%)")
 `tokenfold doctor` verifies it. Trusted filters for Git, build, and test output
 are available through `tokenfold filters list`.
 
-Runnable examples:
+Runnable examples, one per surface, all under [`examples/`](examples):
 
 ```bash
-python examples/quickstart.py
-python examples/lossy_pruning.py
+python examples/quickstart.py      # Python: compress messages, request bodies, JSON data
+node examples/quickstart.mjs       # TypeScript/Node: compress, inspect, read the receipt
+cargo run -p tokenfold-core --example quickstart   # Rust: the embedded core API
+python examples/lossy_pruning.py   # CLI: opt-in recoverable pruning, end to end
 ```
 
+[`examples/quickstart.ipynb`](examples/quickstart.ipynb) is the notebook form of the
+Python quickstart, one operation per cell.
+
 ## Recoverable lossy pruning
 
 Lossless folding has a ceiling on heterogeneous arrays. `--lossy` ranks rows,
@@ -182,6 +187,27 @@ tokenfold compress examples/incident_feed.json --format json \
   --lossy heuristic --lossy-ratio 0.35 --dry-run
 ```
 
+The same flags, and the same fail-closed contract, from Python and TypeScript:
+
+```python
+from tokenfold import CompressionPolicy, InputFormat, LossyPath, compress, retrieve
+
+policy = CompressionPolicy(lossy=LossyPath.HEURISTIC, lossy_ratio=0.35)
+result = compress(feed_bytes, format=InputFormat.JSON, policy=policy)
+original = retrieve(marker["$tf_ref"]["hash"])   # any dropped row, verbatim
+```
+
+```ts
+import { compress, retrieve } from "tokenfold";
+
+const { payload, report } = await compress(feed, {
+  format: "json",
+  lossy: "heuristic",
+  lossyRatio: 0.35,
+});
+const original = await retrieve(hash); // any dropped row, verbatim
+```
+
 <details>
 <summary><strong>Current Phase 1 constraints</strong></summary>
 
@@ -190,9 +216,8 @@ already contain retrieval markers. A filesystem failure after partial writes
 may also leave unreferenced entries until their configured TTL expires. Both
 require location-based transactional materialization before promotion.
 
-Lossy pruning is CLI-only today; Python and TypeScript remain lossless. Preview
-is a projection rather than a filesystem transaction, so a real run may keep
-more rows if storage becomes unavailable.
+Preview is a projection rather than a filesystem transaction, so a real run may
+keep more rows if storage becomes unavailable.
 
 </details>
 
@@ -286,8 +311,8 @@ cargo run --release --locked -p tokenfold-cli -- \
 ```
 
 The small bundled API response reports 382 → 206 tokens, a **46.1% lossless
-reduction**. Benchmark sources and thresholds live in [CHANGELOG.md](CHANGELOG.md)
-and [`crates/tokenfold-core/benches/THRESHOLDS.toml`](crates/tokenfold-core/benches/THRESHOLDS.toml).
+reduction**. Benchmark sources and thresholds live in [CHANGELOG.md](https://github.com/snchimata/tokenfold/blob/main/CHANGELOG.md)
+and [`crates/tokenfold-core/benches/THRESHOLDS.toml`](https://github.com/snchimata/tokenfold/blob/main/crates/tokenfold-core/benches/THRESHOLDS.toml).
 
 ## Contributing
 
@@ -304,7 +329,7 @@ cd packages/tokenfold && npm ci && npm test
 
 ## License
 
-[Apache-2.0](LICENSE)
+[Apache-2.0](https://github.com/snchimata/tokenfold/blob/main/LICENSE)
 
 ---
 

{tokenfold-0.4.0 → tokenfold-0.4.1}/crates/tokenfold-core/benches/THRESHOLDS.toml

@@ -2,7 +2,7 @@
 # Measured 2026-07-11 on AMD Ryzen 7 9800X3D (8c/16t), 31GB RAM, Windows, rustc 1.97.0,
 # `cargo bench -p tokenfold-core` release profile. See CHANGELOG.md for the full reproducibility
 # record. Thresholds carry ~1.3-1.5x headroom over the measured baseline for runner variance;
-# tighten once `bench-regression` has run on a dedicated CI runner (R-014).
+# tighten once `bench-regression` has run on a dedicated, fixed-spec CI runner.
 #
 # p95 latency — dominated by TiktokenEstimator::o200k_base() reloading BPE ranks on every
 # `compress()` call (the public API doesn't cache the estimator across calls) plus tiktoken-rs's

{tokenfold-0.4.0 → tokenfold-0.4.1}/crates/tokenfold-core/benches/compression_bench.rs

@@ -1,6 +1,6 @@
-//! Task 7 benchmark suite (ROADMAP.md Phase 4 / ENGINEERING.md "Benchmark Tests").
+//! Benchmark suite for `compress`: a p95-latency and bytes-allocated regression gate.
 //!
-//! ponytail: ENGINEERING.md names `criterion` + `divan` as the tools here. Both pull in real
+//! ponytail: the conventional picks here are `criterion` + `divan`. Both pull in real
 //! dependency trees (criterion: plotting/serde chain; divan: its own harness+macros) for what
 //! this project actually needs: a p95-latency and bytes-allocated regression gate against a
 //! checked-in threshold file. `std::time::Instant` + a custom counting `GlobalAlloc` do the same

{tokenfold-0.4.0 → tokenfold-0.4.1}/crates/tokenfold-core/src/budget.rs

@@ -5,9 +5,9 @@ use crate::input::{CompressionInput, InputFormat};
 use crate::token_estimator::TokenEstimator;
 
 /// Placeholder floor for `retrieval_ttl_seconds` whenever `lossy` is set — see
-/// `CompressionPolicyBuilder::build`'s lossy validation. The design doc lists the exact policy
-/// as an open owner decision (docs/solution-design/lossy-json-compression.md §8), not yet
-/// confirmed; this value only needs to be "clearly more than instant," not final.
+/// `CompressionPolicyBuilder::build`'s lossy validation. The exact retention policy is still an
+/// open owner decision, not yet confirmed; this value only needs to be "clearly more than
+/// instant," not final.
 const MIN_LOSSY_TTL_SECONDS: u64 = 86_400;
 
 // NOT Eq: `lossy_ratio` holds an f64 (same precedent as `CompressionOutput`/`CompressionReport`).
@@ -29,30 +29,30 @@ pub struct CompressionPolicy {
     /// entry doesn't enable it for the current mode. Still requires `experimental` for any
     /// transform whose `ModeEntry.experimental == true` (see `modes::pipeline_for`).
     pub enable: Vec<String>,
-    /// F-045: when true, and the full pre-transform input contains no secret-shaped content,
+    /// When true, and the full pre-transform input contains no secret-shaped content,
     /// `pipeline::compress_with_estimator` persists it to the reversible evidence store
     /// (`retrieval_backend`/`retrieval_store_path`) under its SHA-256 hash.
     pub store_originals: bool,
-    /// F-045: the namespace stored-original entries are keyed under (see
+    /// The evidence-store namespace stored-original entries are keyed under (see
     /// `retrieval_store::RetrievalStore::store`).
     pub retrieval_namespace: String,
-    /// F-045: TTL passed to `RetrievalStore::store` for newly stored originals. `None` means
+    /// TTL passed to `RetrievalStore::store` for newly stored originals. `None` means
     /// "use `retrieval_store::DEFAULT_TTL_SECONDS`" (this is a *default*, not "never expire" —
     /// that per-entry meaning belongs to `RetrievalStore::store`'s own `ttl_seconds` parameter).
     pub retrieval_ttl_seconds: Option<u64>,
-    /// F-045: backend name passed to `RetrievalStore::open` ("memory" | "filesystem" |
+    /// Backend name passed to `RetrievalStore::open` ("memory" | "filesystem" |
     /// "sqlite" — the latter fails clearly, handled as best-effort skip, see
     /// `pipeline::maybe_store_originals`).
     pub retrieval_backend: String,
-    /// F-045: filesystem backend root override. `None` means
+    /// Filesystem backend root override. `None` means
     /// `retrieval_store::default_store_path()`.
     pub retrieval_store_path: Option<PathBuf>,
-    /// Opt-in lossy JSON array-item selection (`docs/solution-design/lossy-json-compression.md`,
-    /// local-only). `None` (the default) means the lossless pipeline is untouched — this field
-    /// is set only by an explicit CLI flag, never derived from `mode`/`experimental`, and is
-    /// deliberately NOT part of `modes.rs`/`ALL_ENTRIES`: it is a fundamentally different
-    /// category (data-lossy, not just structurally-lossy-but-reversible) from every other
-    /// transform in this crate. See `pipeline::apply_lossy_reduction`.
+    /// Opt-in lossy JSON array-item selection: array items are dropped and replaced by a
+    /// recoverable `$tf_ref` marker. `None` (the default) means the lossless pipeline is
+    /// untouched — this field is set only by an explicit CLI flag, never derived from
+    /// `mode`/`experimental`, and is deliberately NOT part of `modes.rs`/`ALL_ENTRIES`: it is a
+    /// fundamentally different category (data-lossy, not just structurally-lossy-but-reversible)
+    /// from every other transform in this crate. See `pipeline::apply_lossy_reduction`.
     pub lossy: Option<LossyPath>,
     /// BEST-EFFORT selection hint, not an enforced budget: how aggressively to prune, as the
     /// fraction (0.0..=1.0) of the prunable pool's own estimated token cost to keep when `lossy`
@@ -422,8 +422,8 @@ fn message_content_bytes(message: &serde_json::Value) -> Option<Vec<u8>> {
     }
 }
 
-/// Keeps file names and hunk headers, matching the `diff_compaction` (F-013) contract of what
-/// must survive compaction. Each kept line is its own segment.
+/// Keeps file names and hunk headers, matching the `diff_compaction` contract of what must
+/// survive compaction. Each kept line is its own segment.
 fn extract_diff_protected(bytes: &[u8]) -> Vec<Vec<u8>> {
     let text = String::from_utf8_lossy(bytes);
     let mut segments = Vec::new();

{tokenfold-0.4.0 → tokenfold-0.4.1}/crates/tokenfold-core/src/errors.rs

@@ -17,7 +17,12 @@ pub enum TokenFoldError {
 }
 
 impl TokenFoldError {
-    /// CLI exit code per the Error Taxonomy table (PLAN.md / INTERFACES.md §1.4).
+    /// Stable CLI exit code for this error: 2 invalid input, 3 safety/redaction violation,
+    /// 4 estimator failure, 5 bad configuration, 6 internal or I/O failure. Every successful
+    /// [`crate::Status`] outcome — including `UnreachableTarget`, which is a typed result and
+    /// not an error — exits 0. The proxy's HTTP status mapping and the Python binding's
+    /// exception hierarchy are derived from these same buckets, so the numbering is a public
+    /// contract; `exit_codes_match_error_taxonomy_table` below pins it.
     pub fn exit_code(&self) -> i32 {
         match self {
             TokenFoldError::InvalidInput(_) => 2,

{tokenfold-0.4.0 → tokenfold-0.4.1}/crates/tokenfold-core/src/filters.rs

@@ -1,5 +1,5 @@
-//! F-047: declarative command-output filter registry (`roadmap.md` F-047, `interfaces.md`
-//! §7.2 "Filter Pack Contract" and §7.3 "Filter Trust Contract").
+//! Declarative command-output filter registry: the filter-pack schema, its stage-execution
+//! engine, and the trust store that gates project- and user-authored packs.
 //!
 //! Filter packs are TOML documents describing an ordered list of pure text-transform "stages"
 //! (`strip_ansi`, `replace`, `keep_lines`, `strip_lines`, `head`, `tail`, `max_lines`,
@@ -218,7 +218,7 @@ impl FixtureCheck {
 }
 
 // ---------------------------------------------------------------------------------------------
-// Regex complexity guard (R-017 defense-in-depth; see module doc)
+// Regex complexity guard (defense-in-depth against hostile filter patterns; see module doc)
 // ---------------------------------------------------------------------------------------------
 
 fn check_pattern_safety(pattern: &str) -> Result<(), TokenFoldError> {
@@ -407,7 +407,7 @@ pub fn never_worse(raw: &[u8], filtered: &[u8]) -> NeverWorseOutcome {
 }
 
 // ---------------------------------------------------------------------------------------------
-// Trust store (interfaces.md §7.3)
+// Trust store
 // ---------------------------------------------------------------------------------------------
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -541,7 +541,7 @@ pub struct FilterLookup<'a> {
     pub project_filters_path: Option<&'a Path>,
     pub user_filters_path: Option<&'a Path>,
     pub trust_store_path: &'a Path,
-    /// `TOKENFOLD_TRUST_PROJECT_FILTERS=1` (interfaces.md §7.3): bypasses the trust-store check
+    /// `TOKENFOLD_TRUST_PROJECT_FILTERS=1`: bypasses the trust-store check
     /// for the *project* tier only (its literal name is the documented CI override, not a
     /// generic "trust everything" switch) — user-tier filters always require an explicit
     /// `filters trust` regardless of this flag.
@@ -642,10 +642,10 @@ pub fn parse_pack_file(path: &Path) -> Result<FilterPack, TokenFoldError> {
 }
 
 // ---------------------------------------------------------------------------------------------
-// Built-ins (D-002 dogfooding scope: git diff, git status, cargo build/test log framing)
+// Built-ins (solo-dogfooding scope: git diff, git status, cargo build/test log framing)
 // ---------------------------------------------------------------------------------------------
 
-/// D-002's resolved first consumer is solo dogfooding (git diffs, build/test logs, agent
+/// This project's first consumer is solo dogfooding (git diffs, build/test logs, agent
 /// tool-call JSON) — these three built-ins deliberately cover only that, not a broad command
 /// library. Encoded as Rust struct literals rather than parsed TOML strings: for packs this
 /// small, embedding multi-line fixture text in Rust raw strings and building the structs
@@ -785,7 +785,7 @@ fn build_test_log_pack() -> FilterPack {
             version: "1.0.0".to_string(),
         },
         filters: vec![Filter {
-            // Generic cargo build/test log filter (D-002 dogfooding scope: this repo's own
+            // Generic cargo build/test log filter (dogfooding scope: this repo's own
             // `cargo test`/`cargo build`); `match_command = ["cargo"]` fires on any cargo
             // subcommand rather than one filter per subcommand, keeping this a single built-in.
             id: "cargo-build-test-log".to_string(),

{tokenfold-0.4.0 → tokenfold-0.4.1}/crates/tokenfold-core/src/modes.rs

@@ -4,7 +4,7 @@
 //! cannot be disabled (see `budget::CompressionPolicyBuilder::build`).
 //!
 //! `tests/fixtures/mode_matrix.toml` mirrors this table for cross-surface testing
-//! (INTERFACES.md Part 2 is the authoritative reference for both).
+//! (this table is authoritative; the fixture must be kept in sync with it).
 
 use crate::budget::{CompressionMode, TaskScope};
 use crate::input::InputFormat;
@@ -71,12 +71,12 @@ impl ModeEntry {
     }
 }
 
-// Canonical ordered table — order here IS the pipeline execution order (INTERFACES.md Part 2:
-// lossless before lossy, higher-savings before lower-savings, within each mode).
+// Canonical ordered table — order here IS the pipeline execution order (lossless before lossy,
+// higher-savings before lower-savings, within each mode).
 //
-// ponytail: `table_compaction` is intentionally omitted. The First Consumer worksheet
-// (plan.md) doesn't name tables among the dominant payload types, so F-019 stays out of
-// scope until a consumer worksheet asks for it (roadmap.md F-019 dependency).
+// ponytail: `table_compaction` is intentionally omitted. Tabular payloads aren't among this
+// project's dominant input types, so a table transform stays out of scope until a real consumer
+// asks for it.
 pub static ALL_ENTRIES: &[ModeEntry] = &[
     ModeEntry {
         transform_id: TransformId::JsonMinify,
@@ -164,11 +164,11 @@ pub static ALL_ENTRIES: &[ModeEntry] = &[
     ModeEntry {
         transform_id: TransformId::LogCompaction,
         version: "1.0.0",
-        // Promoted out of --experimental (roadmap.md Phase 5 Task 9, 2026-07-12): the
-        // full-lossy-promotion fidelity gate clears every D-005 draft threshold cleanly
+        // Promoted out of --experimental (Phase 5 fidelity gate, 2026-07-12): the
+        // full-lossy-promotion gate profile clears every draft fidelity threshold cleanly
         // (quality_retention=1.0, contrastive_failure_rate=0.0, critical_token_survival_rate=1.0).
-        // conservative_enabled stays false — per plan.md's mode table, Conservative never runs
-        // lossy-with-evidence transforms at all, same convention table_compaction documents.
+        // conservative_enabled stays false — Conservative never runs lossy-with-evidence
+        // transforms at all, same convention table_compaction documents.
         conservative_enabled: false,
         balanced_enabled: true,
         aggressive_enabled: true,
@@ -182,12 +182,14 @@ pub static ALL_ENTRIES: &[ModeEntry] = &[
     ModeEntry {
         transform_id: TransformId::DiffCompaction,
         version: "1.0.0",
-        // Stays --experimental (roadmap.md Phase 5 Task 9, 2026-07-12 re-investigation): the
+        // Stays --experimental (Phase 5 fidelity gate, 2026-07-12 re-investigation): the
         // full-lossy-promotion gate's per_variant breakdown checked the default (body-preserving,
         // task_scope != ChangeSummary) and header-only (TaskScope::ChangeSummary) forms
-        // separately, as F-013 requires, and BOTH miss the D-005 draft thresholds on their own
-        // (default: quality_retention=0.36, contrastive_failure_rate=0.5, critical_token_survival=
-        // 0.5). Root cause: compact_diff has no fallback for non-diff-shaped input — it drops
+        // separately, as diff_compaction's two documented forms require, and BOTH miss the draft
+        // fidelity thresholds on their own — the bar is quality_retention >= 0.95,
+        // contrastive_failure_rate <= 0.005, critical_token_survival >= 0.99, and the default form
+        // measured quality_retention=0.36, contrastive_failure_rate=0.5, critical_token_survival=
+        // 0.5. Root cause: compact_diff has no fallback for non-diff-shaped input — it drops
         // everything, critical tokens included, when no line matches a unified-diff prefix. See
         // eval/tasks/FIXTURES.md's "Scorer status" section for the full measured breakdown.
         conservative_enabled: false,
@@ -205,7 +207,7 @@ pub static ALL_ENTRIES: &[ModeEntry] = &[
         ],
     },
     // v0.2+ entries (table_compaction, prose_extraction, code_digest, conversation) added
-    // here after their fidelity approval / D-002 scope decisions land.
+    // here after their fidelity approval / scope decisions land.
 ];
 
 /// Returns the ordered, applicable transform list for a given (mode, task_scope, format).