tokenfold 0.1.0__tar.gz → 0.2.0__tar.gz

{tokenfold-0.1.0 → tokenfold-0.2.0}/Cargo.lock

@@ -1013,7 +1013,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-adapters"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "serde",
  "serde_json",
@@ -1022,7 +1022,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-admin"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "ed25519-dalek",
  "serde",
@@ -1032,7 +1032,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-cli"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "clap",
  "serde",
@@ -1045,7 +1045,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-core"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "proptest",
  "regex",
@@ -1059,25 +1059,25 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-image"
-version = "0.1.0"
+version = "0.2.0"
 
 [[package]]
 name = "tokenfold-learn"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-output"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-proxy"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "clap",
  "serde",
@@ -1089,7 +1089,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-py"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "pyo3",
  "serde_json",
@@ -1098,7 +1098,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-rag"
-version = "0.1.0"
+version = "0.2.0"
 
 [[package]]
 name = "toml"

{tokenfold-0.1.0 → tokenfold-0.2.0}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/tokenfold-core", "crates/tokenfold-py"]
 
 [workspace.package]
-version = "0.1.0"
+version = "0.2.0"
 edition = "2024"
 rust-version = "1.97"
 license = "Apache-2.0"

{tokenfold-0.1.0 → tokenfold-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tokenfold
-Version: 0.1.0
+Version: 0.2.0
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: 3 :: Only
@@ -48,7 +48,8 @@ exactly what changed. Preview every transformation before applying it.
 
 ## Quick start
 
-tokenfold is currently source-only. Build the CLI with the pinned Rust toolchain:
+The library and Python package are published — `cargo add tokenfold-core` and
+`pip install tokenfold`. To use the CLI, build it from source with the pinned Rust toolchain:
 
 ```bash
 git clone https://github.com/snchimata/tokenfold.git
@@ -81,6 +82,15 @@ target/release/tokenfold compress examples/openai_payload.json \
   --target-tokens 250 > compressed.json
 ```
 
+Compress generic JSON **data** (API responses, records, logs — not just message payloads).
+tokenfold folds arrays of same-shape objects into columnar form and dictionaries repeated
+values, all losslessly:
+
+```bash
+target/release/tokenfold inspect examples/api_response.json --format json
+# larger, more repetitive data compresses further — a 50-record blob folds 0% → ~68%
+```
+
 Or trim noisy output from any command:
 
 ```bash
@@ -114,8 +124,14 @@ target/release/tokenfold wrap -- git diff
 
 ### Python
 
-The Python package is a native `pyo3` binding for CPython 3.9+. Until wheels are published,
-install it locally with [maturin](https://www.maturin.rs/):
+The Python package is a native `pyo3` binding for CPython 3.9+:
+
+```bash
+pip install tokenfold
+```
+
+A Windows wheel is published; on Linux/macOS pip builds the sdist from source (needs a Rust
+toolchain). To build from a checkout instead, use [maturin](https://www.maturin.rs/):
 
 ```bash
 python -m venv .venv
@@ -177,18 +193,25 @@ input  ──▶  redact secrets  ──▶  compact  ──▶  recount  ──
 | `secret_redaction` | Always on | Redacts secret-shaped values before reporting, logging, or persistence |
 | `json_minify` | Default | Removes insignificant JSON whitespace |
 | `schema_compaction` | Default | Shortens examples while preserving descriptions and required fields |
+| `json_field_fold` | Balanced / aggressive | Folds arrays of same-shape objects into columnar `{cols, rows}` — each key once, not once per row (generic JSON data) |
+| `json_value_dict` | Balanced / aggressive | Replaces repeated large values with dictionary references (generic JSON data) |
 | `log_compaction` | Balanced / aggressive | Collapses repeated, low-signal log lines |
 | `diff_compaction` | Experimental | Reduces low-signal diff context |
 
-Lossy transforms must clear a downstream fidelity gate before promotion. Originals can be
-stored by SHA-256 hash when reversibility is requested; secret-shaped content is never
-persisted.
+`json_field_fold` and `json_value_dict` are **lossless and reversible**: each is applied only
+if it passes an exact round-trip check (`unfold(fold(x)) == x`) *and* lowers the exact token
+count, so a payload can never come out larger or altered. Lossy transforms must clear a
+downstream fidelity gate before promotion. Originals can also be stored by SHA-256 hash when
+reversibility is requested; secret-shaped content is never persisted.
 
 ## Measured, not marketed
 
 The structured-JSON benchmark measures **45.63% exact token savings** in balanced mode on a
-deterministic, pretty-printed OpenAI tool-schema fixture. It is a reproducible benchmark,
-not a promise for every workload.
+deterministic, pretty-printed OpenAI tool-schema fixture. On generic JSON **data**, the v0.2
+columnar fold + value dictionary reach **~61–68% exact savings** on repetitive records
+(30–50 rows), losslessly. All numbers are quoted against exact `o200k_base` counts and are
+reproducible benchmarks, not a promise for every workload — ragged or already-compact JSON
+correctly reports single-digit or no savings rather than pretending.
 
 ```bash
 cargo bench -p tokenfold-core

{tokenfold-0.1.0 → tokenfold-0.2.0}/README.md

@@ -36,7 +36,8 @@ exactly what changed. Preview every transformation before applying it.
 
 ## Quick start
 
-tokenfold is currently source-only. Build the CLI with the pinned Rust toolchain:
+The library and Python package are published — `cargo add tokenfold-core` and
+`pip install tokenfold`. To use the CLI, build it from source with the pinned Rust toolchain:
 
 ```bash
 git clone https://github.com/snchimata/tokenfold.git
@@ -69,6 +70,15 @@ target/release/tokenfold compress examples/openai_payload.json \
   --target-tokens 250 > compressed.json
 ```
 
+Compress generic JSON **data** (API responses, records, logs — not just message payloads).
+tokenfold folds arrays of same-shape objects into columnar form and dictionaries repeated
+values, all losslessly:
+
+```bash
+target/release/tokenfold inspect examples/api_response.json --format json
+# larger, more repetitive data compresses further — a 50-record blob folds 0% → ~68%
+```
+
 Or trim noisy output from any command:
 
 ```bash
@@ -102,8 +112,14 @@ target/release/tokenfold wrap -- git diff
 
 ### Python
 
-The Python package is a native `pyo3` binding for CPython 3.9+. Until wheels are published,
-install it locally with [maturin](https://www.maturin.rs/):
+The Python package is a native `pyo3` binding for CPython 3.9+:
+
+```bash
+pip install tokenfold
+```
+
+A Windows wheel is published; on Linux/macOS pip builds the sdist from source (needs a Rust
+toolchain). To build from a checkout instead, use [maturin](https://www.maturin.rs/):
 
 ```bash
 python -m venv .venv
@@ -165,18 +181,25 @@ input  ──▶  redact secrets  ──▶  compact  ──▶  recount  ──
 | `secret_redaction` | Always on | Redacts secret-shaped values before reporting, logging, or persistence |
 | `json_minify` | Default | Removes insignificant JSON whitespace |
 | `schema_compaction` | Default | Shortens examples while preserving descriptions and required fields |
+| `json_field_fold` | Balanced / aggressive | Folds arrays of same-shape objects into columnar `{cols, rows}` — each key once, not once per row (generic JSON data) |
+| `json_value_dict` | Balanced / aggressive | Replaces repeated large values with dictionary references (generic JSON data) |
 | `log_compaction` | Balanced / aggressive | Collapses repeated, low-signal log lines |
 | `diff_compaction` | Experimental | Reduces low-signal diff context |
 
-Lossy transforms must clear a downstream fidelity gate before promotion. Originals can be
-stored by SHA-256 hash when reversibility is requested; secret-shaped content is never
-persisted.
+`json_field_fold` and `json_value_dict` are **lossless and reversible**: each is applied only
+if it passes an exact round-trip check (`unfold(fold(x)) == x`) *and* lowers the exact token
+count, so a payload can never come out larger or altered. Lossy transforms must clear a
+downstream fidelity gate before promotion. Originals can also be stored by SHA-256 hash when
+reversibility is requested; secret-shaped content is never persisted.
 
 ## Measured, not marketed
 
 The structured-JSON benchmark measures **45.63% exact token savings** in balanced mode on a
-deterministic, pretty-printed OpenAI tool-schema fixture. It is a reproducible benchmark,
-not a promise for every workload.
+deterministic, pretty-printed OpenAI tool-schema fixture. On generic JSON **data**, the v0.2
+columnar fold + value dictionary reach **~61–68% exact savings** on repetitive records
+(30–50 rows), losslessly. All numbers are quoted against exact `o200k_base` counts and are
+reproducible benchmarks, not a promise for every workload — ragged or already-compact JSON
+correctly reports single-digit or no savings rather than pretending.
 
 ```bash
 cargo bench -p tokenfold-core

{tokenfold-0.1.0 → tokenfold-0.2.0}/crates/tokenfold-core/examples/quickstart.rs

@@ -12,12 +12,21 @@
 
 use tokenfold_core::{CompressionInput, CompressionMode, CompressionPolicy, compress};
 
-// The payload ships with the repo (examples/openai_payload.json). Embedding it with
-// include_str! keeps this example runnable from any working directory.
+// The payloads ship with the repo. Embedding them with include_str! keeps this example
+// runnable from any working directory.
 const PAYLOAD: &str = include_str!("../../../examples/openai_payload.json");
+const API_RESPONSE: &str = include_str!("../../../examples/api_response.json");
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // 1. Wrap the raw request bytes, telling tokenfold it's an OpenAI-style JSON body.
+    compress_openai_body()?;
+    println!("\n{}\n", "-".repeat(72));
+    compress_json_data()?;
+    Ok(())
+}
+
+// --- 1. An LLM request body (chat messages + a verbose tool schema) -----------------------
+fn compress_openai_body() -> Result<(), Box<dyn std::error::Error>> {
+    // Wrap the raw request bytes, telling tokenfold it's an OpenAI-style JSON body.
     let input = CompressionInput::openai_json(PAYLOAD.as_bytes().to_vec());
 
     // 2. Pick a policy. Balanced is the safe default; JSON minification and schema
@@ -30,6 +39,30 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let out = compress(input, &policy)?;
     let report = &out.report;
 
+    println!("== OpenAI request body ==");
+    print_report(report);
+    Ok(())
+}
+
+// --- 2. Generic JSON data (API response, records, logs) -----------------------------------
+// This is the v0.2 path: data-JSON that isn't an LLM message payload. tokenfold minifies it,
+// folds arrays of same-shape objects into columnar form (each key once, not once per row),
+// and dictionaries repeated values — all losslessly (every stage is round-trip gated).
+fn compress_json_data() -> Result<(), Box<dyn std::error::Error>> {
+    let input = CompressionInput::json(API_RESPONSE.as_bytes().to_vec());
+    let policy = CompressionPolicy::builder()
+        .mode(CompressionMode::Balanced)
+        .build()?;
+    let out = compress(input, &policy)?;
+
+    println!("== Generic JSON data ==");
+    print_report(&out.report);
+    println!("\ncompressed payload ({} bytes):", out.bytes.len());
+    println!("{}", String::from_utf8_lossy(&out.bytes));
+    Ok(())
+}
+
+fn print_report(report: &tokenfold_core::report::CompressionReport) {
     println!("status:    {:?}", report.status);
     println!(
         "tokens:    {} -> {}  ({} saved, {:.1}%)",
@@ -39,23 +72,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         "estimator: {} (exact: {})",
         report.estimator.backend, report.estimator.is_exact,
     );
-
-    // 4. The report itemizes what each transform did — receipts, not guesses.
-    println!("\ntransforms:");
+    // The report itemizes what each transform did — receipts, not guesses.
+    println!("transforms:");
     for t in &report.transforms {
         println!(
             "  - {:<24} {:<10?} (saved {} tokens)",
             t.id, t.status, t.saved_tokens,
         );
     }
-
-    for w in &report.warnings {
-        println!("  warning: {}", w.message);
-    }
-
-    // 5. out.bytes is the compressed payload, ready to send to any provider.
-    println!("\ncompressed payload ({} bytes):", out.bytes.len());
-    println!("{}", String::from_utf8_lossy(&out.bytes));
-
-    Ok(())
 }

{tokenfold-0.1.0 → tokenfold-0.2.0}/crates/tokenfold-core/src/budget.rs

@@ -220,7 +220,12 @@ pub fn protected_segments(input: &CompressionInput, policy: &CompressionPolicy)
         InputFormat::GitDiff => extract_diff_protected(&input.bytes),
         // ponytail: no transform touches plain text/command output structure yet beyond
         // log/diff compaction (task-scope gated), so nothing is unconditionally protected.
-        InputFormat::PlainText | InputFormat::CommandOutput | InputFormat::Auto => Vec::new(),
+        // Generic Json has no "protected" sub-segment either — json_field_fold's own
+        // round-trip safety gate is what guarantees its data is preserved.
+        InputFormat::PlainText
+        | InputFormat::CommandOutput
+        | InputFormat::Json
+        | InputFormat::Auto => Vec::new(),
     }
 }
 

{tokenfold-0.1.0 → tokenfold-0.2.0}/crates/tokenfold-core/src/input.rs

@@ -8,6 +8,9 @@ pub enum InputFormat {
     Auto,
     OpenAiJson,
     AnthropicJson,
+    /// Generic JSON data (API responses, records, logs) that isn't an LLM message payload.
+    /// Unlocks the JSON-data transforms (`json_minify`, `json_field_fold`).
+    Json,
     PlainText,
     CommandOutput,
     GitDiff,
@@ -34,6 +37,15 @@ impl CompressionInput {
         }
     }
 
+    /// Generic JSON data (not an LLM message payload). Enables `json_minify`,
+    /// `json_field_fold`, and `json_value_dict`.
+    pub fn json(bytes: impl Into<Vec<u8>>) -> Self {
+        Self {
+            format: InputFormat::Json,
+            bytes: bytes.into(),
+        }
+    }
+
     pub fn plain_text(bytes: impl Into<Vec<u8>>) -> Self {
         Self {
             format: InputFormat::PlainText,

{tokenfold-0.1.0 → tokenfold-0.2.0}/crates/tokenfold-core/src/modes.rs

@@ -12,6 +12,8 @@ use crate::input::InputFormat;
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum TransformId {
     JsonMinify,
+    JsonFieldFold,
+    JsonValueDict,
     SchemaCompaction,
     LogCompaction,
     DiffCompaction,
@@ -21,6 +23,8 @@ impl TransformId {
     pub fn as_str(&self) -> &'static str {
         match self {
             TransformId::JsonMinify => "json_minify",
+            TransformId::JsonFieldFold => "json_field_fold",
+            TransformId::JsonValueDict => "json_value_dict",
             TransformId::SchemaCompaction => "schema_compaction",
             TransformId::LogCompaction => "log_compaction",
             TransformId::DiffCompaction => "diff_compaction",
@@ -83,7 +87,45 @@ pub static ALL_ENTRIES: &[ModeEntry] = &[
         max_ratio_balanced: 1.0,
         max_ratio_aggressive: 1.0,
         task_scopes: &[TaskScope::All],
-        applicable_formats: &[InputFormat::OpenAiJson, InputFormat::AnthropicJson],
+        applicable_formats: &[
+            InputFormat::OpenAiJson,
+            InputFormat::AnthropicJson,
+            InputFormat::Json,
+        ],
+    },
+    // json_field_fold (v0.2): reversible columnar fold of arrays of homogeneous objects.
+    // Lossless (round-trip gated in the pipeline), so max_ratio is unrestricted (1.0), but
+    // it restructures what the model sees, so it stays out of Conservative (same convention
+    // as log_compaction) and only runs on generic Json data, never on OpenAI/Anthropic
+    // message bodies (whose API shape must not change).
+    ModeEntry {
+        transform_id: TransformId::JsonFieldFold,
+        version: "1.0.0",
+        conservative_enabled: false,
+        balanced_enabled: true,
+        aggressive_enabled: true,
+        experimental: false,
+        max_ratio_conservative: 0.0,
+        max_ratio_balanced: 1.0,
+        max_ratio_aggressive: 1.0,
+        task_scopes: &[TaskScope::All],
+        applicable_formats: &[InputFormat::Json],
+    },
+    // json_value_dict (v0.2): reversible value deduplication. Runs AFTER json_field_fold so it
+    // also collapses the repeated nested values folding surfaces across rows. Lossless
+    // (round-trip gated), unrestricted ratio, out of Conservative, generic Json only.
+    ModeEntry {
+        transform_id: TransformId::JsonValueDict,
+        version: "1.0.0",
+        conservative_enabled: false,
+        balanced_enabled: true,
+        aggressive_enabled: true,
+        experimental: false,
+        max_ratio_conservative: 0.0,
+        max_ratio_balanced: 1.0,
+        max_ratio_aggressive: 1.0,
+        task_scopes: &[TaskScope::All],
+        applicable_formats: &[InputFormat::Json],
     },
     ModeEntry {
         transform_id: TransformId::SchemaCompaction,

{tokenfold-0.1.0 → tokenfold-0.2.0}/crates/tokenfold-core/src/pipeline.rs

@@ -354,6 +354,12 @@ fn apply_single_transform(
 ) -> Result<Vec<u8>, String> {
     match transform_id {
         TransformId::JsonMinify => transforms::json::minify_json(bytes).map_err(|e| e.to_string()),
+        TransformId::JsonFieldFold => {
+            transforms::json_fold::fold_json(bytes).map_err(|e| e.to_string())
+        }
+        TransformId::JsonValueDict => {
+            transforms::json_dict::dict_json(bytes).map_err(|e| e.to_string())
+        }
         TransformId::SchemaCompaction => {
             // ponytail: a fixed example cap for now; per-mode example counts are a future
             // config knob (F-011 acceptance criteria only requires the count be configurable,
@@ -379,18 +385,45 @@ fn validate_safety(
     after: &[u8],
     protected: &[Vec<u8>],
 ) -> bool {
-    let is_json_format = matches!(format, InputFormat::OpenAiJson | InputFormat::AnthropicJson);
-    let is_json_transform = matches!(
-        transform_id,
-        TransformId::JsonMinify | TransformId::SchemaCompaction
-    );
-    if is_json_format && is_json_transform {
-        if !safety::json_still_valid(after) {
-            return false;
+    match transform_id {
+        // json_field_fold intentionally restructures JSON (arrays of objects -> columnar
+        // form), so key-order preservation does NOT apply. Its safety invariant is instead
+        // exact reversibility: unfolding the output must reproduce the input's data.
+        TransformId::JsonFieldFold => {
+            if !safety::json_still_valid(after) {
+                return false;
+            }
+            if !transforms::json_fold::round_trips(before, after) {
+                return false;
+            }
+        }
+        // json_value_dict replaces repeated values with dictionary references — also a
+        // reversible restructure, gated on exact round-trip reconstruction.
+        TransformId::JsonValueDict => {
+            if !safety::json_still_valid(after) {
+                return false;
+            }
+            if !transforms::json_dict::round_trips(before, after) {
+                return false;
+            }
         }
-        if !safety::json_key_order_preserved(before, after) {
-            return false;
+        // json_minify / schema_compaction on any JSON-family format: output must stay valid
+        // JSON with byte-for-byte key order preserved.
+        TransformId::JsonMinify | TransformId::SchemaCompaction => {
+            let is_json_format = matches!(
+                format,
+                InputFormat::OpenAiJson | InputFormat::AnthropicJson | InputFormat::Json
+            );
+            if is_json_format {
+                if !safety::json_still_valid(after) {
+                    return false;
+                }
+                if !safety::json_key_order_preserved(before, after) {
+                    return false;
+                }
+            }
         }
+        TransformId::LogCompaction | TransformId::DiffCompaction => {}
     }
     safety::protected_segments_present(protected, after)
 }
@@ -470,6 +503,7 @@ fn format_label(format: InputFormat) -> &'static str {
         InputFormat::Auto => "auto",
         InputFormat::OpenAiJson => "openai_json",
         InputFormat::AnthropicJson => "anthropic_json",
+        InputFormat::Json => "json",
         InputFormat::PlainText => "plain_text",
         InputFormat::CommandOutput => "command_output",
         InputFormat::GitDiff => "git_diff",
@@ -588,6 +622,35 @@ mod tests {
         assert!(serde_json::from_slice::<serde_json::Value>(&output.bytes).is_ok());
     }
 
+    #[test]
+    fn json_data_transforms_never_regress_token_count() {
+        // The "exact-token chooser" property: each JSON-data stage (minify -> fold -> dict) is
+        // adopted only if it lowers the exact token count, so no input can come out larger —
+        // including the shapes that sink naive TOON/CSV-ization: ragged, scalar, already-compact.
+        let cases: &[&[u8]] = &[
+            br#"[{"a":1,"b":2},{"a":3,"b":4},{"a":5,"b":6}]"#, // foldable
+            br#"[{"a":1},{"b":2},{"c":3}]"#,                    // ragged, heterogeneous
+            br#"{"x":[1,2,3],"y":"already compact scalar"}"#,   // no repeated structure
+            br#"[1,2,3,4,5,6,7,8,9,10]"#,                       // scalars only
+            br#"{}"#,                                            // trivial
+        ];
+        for case in cases {
+            let input = CompressionInput::json(case.to_vec());
+            let policy = CompressionPolicy::builder().build().unwrap();
+            let output = compress_with_estimator(input, &policy, &ByteHeuristicEstimator).unwrap();
+            assert!(
+                output.report.compressed_tokens <= output.report.original_tokens,
+                "regressed on {}: {} -> {}",
+                String::from_utf8_lossy(case),
+                output.report.original_tokens,
+                output.report.compressed_tokens,
+            );
+            // and every adopted transform round-trips is guaranteed by the pipeline safety gate;
+            // here we just assert the output is still valid JSON.
+            assert!(serde_json::from_slice::<serde_json::Value>(&output.bytes).is_ok());
+        }
+    }
+
     #[test]
     fn secret_redaction_warning_always_present_when_redaction_runs() {
         let input = CompressionInput::plain_text(b"nothing secret here".to_vec());

tokenfold-0.2.0/crates/tokenfold-core/src/transforms/json_dict.rs

@@ -0,0 +1,289 @@
+//! `json_value_dict` transform (canonical id `"json_value_dict"`, v1.0.0).
+//!
+//! Content-aware, **losslessly reversible** value deduplication for generic JSON data.
+//! Where `json_field_fold` factors out repeated *keys*, this factors out repeated *values*:
+//! the same large object/array/string appearing many times (a constant nested record, a
+//! repeated timestamp, an identical config blob) is stored once in a dictionary and every
+//! occurrence is replaced by a compact reference.
+//!
+//! ```text
+//! {"a":{"x":1,"y":2,"z":3},"b":{"x":1,"y":2,"z":3}}
+//!   -> {"__tf_dict__":[{"x":1,"y":2,"z":3}],
+//!       "__tf_data__":{"a":{"__tf_ref__":0},"b":{"__tf_ref__":0}}}
+//! ```
+//!
+//! It runs *after* `json_field_fold` in the pipeline, so it also collapses the repeated
+//! nested values that folding surfaces across rows. Only values whose serialized form is
+//! large enough that a reference is cheaper are dictionaried (see `MIN_VALUE_BYTES`), and the
+//! pipeline's exact-token gate drops the whole transform if it fails to net a saving — so it
+//! can never make a payload worse. Reversibility is guaranteed by `round_trips` (the pipeline
+//! safety gate): a fold that wouldn't restore exactly is rolled back.
+
+use std::collections::HashMap;
+
+use serde_json::{Map, Value};
+
+/// Canonical transform id, as registered with the pipeline.
+pub const TRANSFORM_ID: &str = "json_value_dict";
+
+/// Semantic version of this transform's output behavior.
+pub const TRANSFORM_VERSION: &str = "1.0.0";
+
+const DICT: &str = "__tf_dict__";
+const DATA: &str = "__tf_data__";
+const REF: &str = "__tf_ref__";
+
+/// Minimum serialized byte length for a value to be worth dictionarying. A reference is
+/// `{"__tf_ref__":N}` (~16 bytes), so shorter values would only grow; the exact-token gate is
+/// the final arbiter, this is just a cheap pre-filter that keeps the transform net-positive.
+const MIN_VALUE_BYTES: usize = 24;
+
+/// A value must repeat at least this many times to be dictionaried.
+const MIN_COUNT: usize = 2;
+
+#[derive(Debug, thiserror::Error)]
+pub enum JsonDictError {
+    #[error("invalid json: {0}")]
+    Invalid(#[from] serde_json::Error),
+}
+
+/// Replaces repeated large values in `input` with dictionary references. No-op-safe: empty
+/// input returns empty; input with nothing worth dictionarying returns unchanged.
+pub fn dict_json(input: &[u8]) -> Result<Vec<u8>, JsonDictError> {
+    if input.is_empty() {
+        return Ok(Vec::new());
+    }
+    let value: Value = serde_json::from_slice(input)?;
+
+    let mut counts: HashMap<String, usize> = HashMap::new();
+    count_dictable(&value, &mut counts);
+
+    let mut index: Vec<String> = Vec::new();
+    let mut index_of: HashMap<String, usize> = HashMap::new();
+    let data = replace(&value, &counts, &mut index, &mut index_of);
+
+    if index.is_empty() {
+        return Ok(input.to_vec());
+    }
+    let dict: Vec<Value> = index
+        .iter()
+        .map(|s| serde_json::from_str(s).expect("canonical string from our own to_string"))
+        .collect();
+    let mut wrapper = Map::new();
+    wrapper.insert(DICT.to_string(), Value::Array(dict));
+    wrapper.insert(DATA.to_string(), data);
+    Ok(serde_json::to_vec(&Value::Object(wrapper))?)
+}
+
+/// Inverse of [`dict_json`]: expands every `{__tf_ref__: i}` back to `dict[i]`.
+pub fn undict_json(input: &[u8]) -> Result<Vec<u8>, JsonDictError> {
+    if input.is_empty() {
+        return Ok(Vec::new());
+    }
+    let value: Value = serde_json::from_slice(input)?;
+    let restored = restore(&value);
+    Ok(serde_json::to_vec(&restored)?)
+}
+
+/// True iff expanding `after` reproduces `before` exactly (as JSON values) — the pipeline's
+/// safety gate for this transform.
+pub fn round_trips(before: &[u8], after: &[u8]) -> bool {
+    let (Ok(before_v), Ok(after_v)) = (
+        serde_json::from_slice::<Value>(before),
+        serde_json::from_slice::<Value>(after),
+    ) else {
+        return false;
+    };
+    restore(&after_v) == before_v
+}
+
+/// Canonical dictionary key for a value, or `None` if it isn't a dictionary candidate
+/// (too small, or a scalar that a reference couldn't beat).
+fn dict_key(v: &Value) -> Option<String> {
+    if !matches!(v, Value::Object(_) | Value::Array(_) | Value::String(_)) {
+        return None;
+    }
+    let key = serde_json::to_string(v).ok()?;
+    (key.len() >= MIN_VALUE_BYTES).then_some(key)
+}
+
+fn count_dictable(v: &Value, counts: &mut HashMap<String, usize>) {
+    if let Some(k) = dict_key(v) {
+        *counts.entry(k).or_insert(0) += 1;
+    }
+    match v {
+        Value::Object(m) => m.values().for_each(|val| count_dictable(val, counts)),
+        Value::Array(a) => a.iter().for_each(|item| count_dictable(item, counts)),
+        _ => {}
+    }
+}
+
+/// Top-down replacement: the outermost repeated value at any position becomes a reference and
+/// is not descended into (so a repeated parent object swallows its children rather than
+/// leaving dead dictionary entries). Indices are assigned lazily, so only referenced values
+/// end up in the dictionary.
+fn replace(
+    v: &Value,
+    counts: &HashMap<String, usize>,
+    index: &mut Vec<String>,
+    index_of: &mut HashMap<String, usize>,
+) -> Value {
+    if let Some(k) = dict_key(v)
+        && counts.get(&k).copied().unwrap_or(0) >= MIN_COUNT
+    {
+        let idx = *index_of.entry(k.clone()).or_insert_with(|| {
+            index.push(k);
+            index.len() - 1
+        });
+        let mut r = Map::new();
+        r.insert(REF.to_string(), Value::from(idx as u64));
+        return Value::Object(r);
+    }
+    match v {
+        Value::Object(m) => {
+            let mut o = Map::new();
+            for (kk, val) in m {
+                o.insert(kk.clone(), replace(val, counts, index, index_of));
+            }
+            Value::Object(o)
+        }
+        Value::Array(a) => Value::Array(a.iter().map(|it| replace(it, counts, index, index_of)).collect()),
+        _ => v.clone(),
+    }
+}
+
+/// Expands a whole document: if it's our `{__tf_dict__, __tf_data__}` wrapper, resolve every
+/// reference in the data against the dictionary; otherwise return it unchanged.
+fn restore(v: &Value) -> Value {
+    let Some((dict, data)) = as_wrapper(v) else {
+        return v.clone();
+    };
+    expand(data, dict)
+}
+
+fn as_wrapper(v: &Value) -> Option<(&Vec<Value>, &Value)> {
+    let m = v.as_object()?;
+    if m.len() != 2 {
+        return None;
+    }
+    let dict = m.get(DICT)?.as_array()?;
+    let data = m.get(DATA)?;
+    Some((dict, data))
+}
+
+fn expand(v: &Value, dict: &[Value]) -> Value {
+    if let Some(idx) = as_ref(v) {
+        // Dictionary entries are stored ref-free, so this resolves in one step; out-of-range
+        // indices are left as-is (the round-trip gate then rejects the fold).
+        return dict.get(idx).cloned().unwrap_or_else(|| v.clone());
+    }
+    match v {
+        Value::Object(m) => {
+            let mut o = Map::new();
+            for (k, val) in m {
+                o.insert(k.clone(), expand(val, dict));
+            }
+            Value::Object(o)
+        }
+        Value::Array(a) => Value::Array(a.iter().map(|it| expand(it, dict)).collect()),
+        _ => v.clone(),
+    }
+}
+
+fn as_ref(v: &Value) -> Option<usize> {
+    let m = v.as_object()?;
+    if m.len() != 1 {
+        return None;
+    }
+    Some(m.get(REF)?.as_u64()? as usize)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn parse(b: &[u8]) -> Value {
+        serde_json::from_slice(b).unwrap()
+    }
+
+    #[test]
+    fn dedups_a_repeated_large_object() {
+        let obj = r#"{"kind":"widget","color":"blue","size":"large"}"#;
+        let input = format!(r#"{{"a":{obj},"b":{obj},"c":{obj}}}"#);
+        let out = dict_json(input.as_bytes()).unwrap();
+        let s = String::from_utf8(out.clone()).unwrap();
+        assert!(s.contains("__tf_dict__"), "expected dict form, got {s}");
+        // the object literal appears once (in the dict), not three times
+        assert_eq!(s.matches("widget").count(), 1);
+        assert!(round_trips(input.as_bytes(), &out));
+    }
+
+    #[test]
+    fn dict_then_undict_reproduces_original() {
+        let obj = r#"{"kind":"widget","color":"blue","size":"large"}"#;
+        let input = format!(r#"[{obj},{obj},{obj},{obj}]"#);
+        let folded = dict_json(input.as_bytes()).unwrap();
+        let back = undict_json(&folded).unwrap();
+        assert_eq!(parse(&back), parse(input.as_bytes()));
+    }
+
+    #[test]
+    fn short_repeated_values_are_left_alone() {
+        // "member" is short; a reference would be longer, so it must NOT be dictionaried.
+        let input = br#"{"a":"member","b":"member","c":"member"}"#;
+        let out = dict_json(input).unwrap();
+        assert_eq!(parse(&out), parse(input));
+        assert!(!String::from_utf8(out).unwrap().contains("__tf_dict__"));
+    }
+
+    #[test]
+    fn single_occurrence_is_not_dictionaried() {
+        let input = br#"{"only":{"a":"quite a long value here indeed yes"}}"#;
+        let out = dict_json(input).unwrap();
+        assert!(!String::from_utf8(out).unwrap().contains("__tf_dict__"));
+    }
+
+    #[test]
+    fn nested_repeats_do_not_create_dead_entries() {
+        let inner = r#"{"deeply":"nested repeated value that is long"}"#;
+        let parent = format!(r#"{{"p":{inner},"q":"x"}}"#);
+        let input = format!(r#"[{parent},{parent}]"#);
+        let out = dict_json(input.as_bytes()).unwrap();
+        assert!(round_trips(input.as_bytes(), &out));
+    }
+
+    #[test]
+    fn empty_and_invalid_inputs() {
+        assert!(dict_json(b"").unwrap().is_empty());
+        assert!(dict_json(b"{bad").is_err());
+    }
+
+    use proptest::prelude::*;
+
+    fn arb_json() -> impl Strategy<Value = Value> {
+        let leaf = prop_oneof![
+            Just(Value::Null),
+            any::<bool>().prop_map(Value::Bool),
+            any::<i64>().prop_map(|n| Value::Number(n.into())),
+            "[^\"\\\\]{0,20}".prop_map(Value::String),
+        ];
+        leaf.prop_recursive(4, 48, 6, |inner| {
+            prop_oneof![
+                prop::collection::vec(inner.clone(), 0..6).prop_map(Value::Array),
+                prop::collection::hash_map("[a-z]{1,6}", inner, 0..6)
+                    .prop_map(|m| Value::Object(m.into_iter().collect())),
+            ]
+        })
+    }
+
+    proptest! {
+        #[test]
+        fn dict_then_undict_is_the_identity_on_arbitrary_json(v in arb_json()) {
+            let bytes = serde_json::to_vec(&v).unwrap();
+            let dicted = dict_json(&bytes).unwrap();
+            prop_assert!(round_trips(&bytes, &dicted));
+            let back: Value = serde_json::from_slice(&undict_json(&dicted).unwrap()).unwrap();
+            prop_assert_eq!(back, v);
+        }
+    }
+}

tokenfold-0.2.0/crates/tokenfold-core/src/transforms/json_fold.rs

@@ -0,0 +1,308 @@
+//! `json_field_fold` transform (canonical id `"json_field_fold"`, v1.0.0).
+//!
+//! Content-aware, **losslessly reversible** structural compression for generic JSON data
+//! (`InputFormat::Json`). Where `json_minify` only removes whitespace, this transform removes
+//! the dominant source of token waste in data-JSON: **repeated object keys**. An array of N
+//! objects that all share the same keys emits each key N times; folding rewrites it to a
+//! columnar form that emits each key exactly once.
+//!
+//! ```text
+//! [ {"id":1,"role":"member"}, {"id":2,"role":"member"} ]
+//!   -> {"__tf_cols__":["id","role"],"__tf_rows__":[[1,"member"],[2,"member"]]}
+//! ```
+//!
+//! The fold is applied recursively (values are folded before their enclosing array is), so
+//! nested arrays-of-objects fold too. It is a pure structural rewrite — every value is
+//! preserved — so `unfold_json` reconstructs the original data exactly (as a `serde_json`
+//! value; number *spelling* like `1e10` may be renormalized by re-serialization, but the
+//! numeric value is unchanged). The pipeline gates adoption on that round-trip
+//! (`round_trips`), so a fold that would ever lose data is rolled back rather than emitted.
+
+use serde_json::{Map, Value};
+
+/// Canonical transform id, as registered with the pipeline.
+pub const TRANSFORM_ID: &str = "json_field_fold";
+
+/// Semantic version of this transform's output behavior.
+pub const TRANSFORM_VERSION: &str = "1.0.0";
+
+/// Reserved keys marking a folded array. Chosen to be collision-unlikely in real data; any
+/// actual collision is caught by the pipeline's round-trip safety gate, never corrupts data.
+const COLS: &str = "__tf_cols__";
+const ROWS: &str = "__tf_rows__";
+
+/// Minimum array length worth folding. Below this the `{cols,rows}` framing overhead can
+/// exceed the saved keys; the pipeline also rolls back any net regression, so this is only a
+/// cheap early-out, not the correctness boundary.
+const MIN_ROWS: usize = 2;
+
+#[derive(Debug, thiserror::Error)]
+pub enum JsonFoldError {
+    #[error("invalid json: {0}")]
+    Invalid(#[from] serde_json::Error),
+}
+
+/// Folds arrays of homogeneous objects in `input` into columnar `{__tf_cols__, __tf_rows__}`
+/// form. No-op-safe: empty input returns empty; input with no foldable arrays returns a
+/// (compact) re-serialization with the same data.
+pub fn fold_json(input: &[u8]) -> Result<Vec<u8>, JsonFoldError> {
+    if input.is_empty() {
+        return Ok(Vec::new());
+    }
+    let value: Value = serde_json::from_slice(input)?;
+    let folded = fold_value(&value);
+    Ok(serde_json::to_vec(&folded)?)
+}
+
+/// Inverse of [`fold_json`]: expands every `{__tf_cols__, __tf_rows__}` node back into an
+/// array of objects. This is the reversible half that makes the transform safe.
+pub fn unfold_json(input: &[u8]) -> Result<Vec<u8>, JsonFoldError> {
+    if input.is_empty() {
+        return Ok(Vec::new());
+    }
+    let value: Value = serde_json::from_slice(input)?;
+    let unfolded = unfold_value(&value);
+    Ok(serde_json::to_vec(&unfolded)?)
+}
+
+/// True iff unfolding `after` reproduces `before` exactly (as JSON values). The pipeline's
+/// safety gate for this transform — folding is only adopted when this holds.
+pub fn round_trips(before: &[u8], after: &[u8]) -> bool {
+    let (Ok(before_v), Ok(after_v)) = (
+        serde_json::from_slice::<Value>(before),
+        serde_json::from_slice::<Value>(after),
+    ) else {
+        return false;
+    };
+    unfold_value(&after_v) == before_v
+}
+
+fn fold_value(v: &Value) -> Value {
+    match v {
+        Value::Array(items) => {
+            let folded: Vec<Value> = items.iter().map(fold_value).collect();
+            try_fold_array(&folded).unwrap_or(Value::Array(folded))
+        }
+        Value::Object(map) => {
+            let mut out = Map::new();
+            for (k, val) in map {
+                out.insert(k.clone(), fold_value(val));
+            }
+            Value::Object(out)
+        }
+        _ => v.clone(),
+    }
+}
+
+/// Returns the columnar node if `items` is a foldable array (>= MIN_ROWS objects sharing the
+/// same key set, none empty, none already using the reserved markers), else `None`.
+fn try_fold_array(items: &[Value]) -> Option<Value> {
+    if items.len() < MIN_ROWS {
+        return None;
+    }
+    let first = items[0].as_object()?;
+    if first.is_empty() || first.contains_key(COLS) || first.contains_key(ROWS) {
+        return None;
+    }
+    // Column order is the first object's key order; membership is a set check so objects that
+    // carry the same keys in a different order still fold (values are placed by key, so this
+    // stays lossless).
+    let cols: Vec<String> = first.keys().cloned().collect();
+    for item in items {
+        let obj = item.as_object()?;
+        if obj.len() != cols.len() || !cols.iter().all(|k| obj.contains_key(k)) {
+            return None;
+        }
+    }
+    let rows: Vec<Value> = items
+        .iter()
+        .map(|item| {
+            let obj = item.as_object().expect("checked above");
+            Value::Array(cols.iter().map(|k| obj[k].clone()).collect())
+        })
+        .collect();
+    let mut folded = Map::new();
+    folded.insert(
+        COLS.to_string(),
+        Value::Array(cols.into_iter().map(Value::String).collect()),
+    );
+    folded.insert(ROWS.to_string(), Value::Array(rows));
+    Some(Value::Object(folded))
+}
+
+fn unfold_value(v: &Value) -> Value {
+    match v {
+        Value::Object(map) => {
+            if let Some(arr) = try_unfold_node(map) {
+                arr
+            } else {
+                let mut out = Map::new();
+                for (k, val) in map {
+                    out.insert(k.clone(), unfold_value(val));
+                }
+                Value::Object(out)
+            }
+        }
+        Value::Array(items) => Value::Array(items.iter().map(unfold_value).collect()),
+        _ => v.clone(),
+    }
+}
+
+/// Expands one `{__tf_cols__, __tf_rows__}` node back to an array of objects, or `None` if
+/// `map` isn't a well-formed folded node (wrong keys, wrong types, or a row whose width
+/// doesn't match the column count — all of which mean it's real data, not our framing).
+fn try_unfold_node(map: &Map<String, Value>) -> Option<Value> {
+    if map.len() != 2 {
+        return None;
+    }
+    let cols = map.get(COLS)?.as_array()?;
+    let rows = map.get(ROWS)?.as_array()?;
+    let col_names: Vec<&str> = cols.iter().map(|c| c.as_str()).collect::<Option<_>>()?;
+    let mut out = Vec::with_capacity(rows.len());
+    for row in rows {
+        let vals = row.as_array()?;
+        if vals.len() != col_names.len() {
+            return None;
+        }
+        let mut obj = Map::new();
+        for (name, val) in col_names.iter().zip(vals) {
+            obj.insert((*name).to_string(), unfold_value(val));
+        }
+        out.push(Value::Object(obj));
+    }
+    Some(Value::Array(out))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn parse(b: &[u8]) -> Value {
+        serde_json::from_slice(b).unwrap()
+    }
+
+    #[test]
+    fn folds_homogeneous_array_and_removes_repeated_keys() {
+        let input = br#"[{"id":1,"role":"member"},{"id":2,"role":"member"}]"#;
+        let out = fold_json(input).unwrap();
+        let s = String::from_utf8(out.clone()).unwrap();
+        assert!(s.contains("__tf_cols__"), "expected folded form, got {s}");
+        // "role" appears once as a column, not once per row.
+        assert_eq!(s.matches("role").count(), 1);
+        // and it round-trips
+        assert!(round_trips(input, &out));
+    }
+
+    #[test]
+    fn fold_then_unfold_reproduces_original_value() {
+        let input = br#"{"count":2,"results":[{"a":1,"b":2},{"a":3,"b":4}]}"#;
+        let folded = fold_json(input).unwrap();
+        let unfolded = unfold_json(&folded).unwrap();
+        assert_eq!(parse(&unfolded), parse(input));
+    }
+
+    #[test]
+    fn nested_arrays_of_objects_fold_too() {
+        let input = br#"{"groups":[{"users":[{"id":1},{"id":2}]},{"users":[{"id":3},{"id":4}]}]}"#;
+        let out = fold_json(input).unwrap();
+        assert!(round_trips(input, &out));
+        // both the outer "groups" array and each inner "users" array are homogeneous → folded.
+        let s = String::from_utf8(out).unwrap();
+        assert!(s.contains("__tf_cols__"));
+    }
+
+    #[test]
+    fn heterogeneous_array_is_left_alone() {
+        // different key sets → not foldable, must stay a plain array (still round-trips).
+        let input = br#"[{"a":1},{"b":2}]"#;
+        let out = fold_json(input).unwrap();
+        assert_eq!(parse(&out), parse(input));
+        assert!(!String::from_utf8(out).unwrap().contains("__tf_cols__"));
+    }
+
+    #[test]
+    fn array_of_non_objects_is_left_alone() {
+        let input = br#"[1,2,3,4]"#;
+        let out = fold_json(input).unwrap();
+        assert_eq!(parse(&out), parse(input));
+    }
+
+    #[test]
+    fn single_object_array_is_not_folded() {
+        let input = br#"[{"a":1,"b":2}]"#;
+        let out = fold_json(input).unwrap();
+        assert!(!String::from_utf8(out).unwrap().contains("__tf_cols__"));
+    }
+
+    #[test]
+    fn same_keys_different_order_still_folds_losslessly() {
+        let input = br#"[{"a":1,"b":2},{"b":4,"a":3}]"#;
+        let out = fold_json(input).unwrap();
+        assert!(round_trips(input, &out));
+    }
+
+    #[test]
+    fn source_data_using_reserved_markers_does_not_round_trip_falsely() {
+        // If real data already looks like our framing, folding must not silently corrupt it.
+        // Here the array isn't foldable anyway; the point is unfold doesn't misread real data
+        // that happens to be a genuine 2-key object with other names.
+        let input = br#"{"__tf_cols__":["x"],"__tf_rows__":[[1]]}"#;
+        // This *is* a valid folded node shape; unfold would expand it. round_trips guards the
+        // pipeline: folding this input produces identical bytes (nothing foldable), and
+        // unfold(after) != before, so the pipeline would not adopt a fold here.
+        let folded = fold_json(input).unwrap();
+        assert!(!round_trips(input, &folded));
+    }
+
+    #[test]
+    fn empty_input_is_a_noop() {
+        assert!(fold_json(b"").unwrap().is_empty());
+        assert!(unfold_json(b"").unwrap().is_empty());
+    }
+
+    #[test]
+    fn invalid_json_is_rejected() {
+        assert!(fold_json(b"{not json").is_err());
+    }
+
+    #[test]
+    fn numbers_and_nulls_survive_the_round_trip() {
+        let input = br#"[{"n":1.5,"z":null,"b":true},{"n":2.5,"z":null,"b":false}]"#;
+        let out = fold_json(input).unwrap();
+        assert!(round_trips(input, &out));
+    }
+
+    use proptest::prelude::*;
+
+    // Arbitrary JSON (integer numbers only, to keep value-equality exact — float spelling is
+    // the one thing re-serialization may renormalize, and it's covered by the unit test above).
+    fn arb_json() -> impl Strategy<Value = Value> {
+        let leaf = prop_oneof![
+            Just(Value::Null),
+            any::<bool>().prop_map(Value::Bool),
+            any::<i64>().prop_map(|n| Value::Number(n.into())),
+            "[^\"\\\\]{0,12}".prop_map(Value::String),
+        ];
+        leaf.prop_recursive(4, 48, 6, |inner| {
+            prop_oneof![
+                prop::collection::vec(inner.clone(), 0..6).prop_map(Value::Array),
+                prop::collection::hash_map("[a-z]{1,6}", inner, 0..6)
+                    .prop_map(|m| Value::Object(m.into_iter().collect())),
+            ]
+        })
+    }
+
+    proptest! {
+        // The core safety guarantee: folding never loses data. For ANY JSON value, unfolding
+        // the folded form reproduces it exactly, and round_trips() (the pipeline's gate) agrees.
+        #[test]
+        fn fold_then_unfold_is_the_identity_on_arbitrary_json(v in arb_json()) {
+            let bytes = serde_json::to_vec(&v).unwrap();
+            let folded = fold_json(&bytes).unwrap();
+            prop_assert!(round_trips(&bytes, &folded), "round_trips() rejected a valid fold");
+            let unfolded = unfold_json(&folded).unwrap();
+            let back: Value = serde_json::from_slice(&unfolded).unwrap();
+            prop_assert_eq!(back, v);
+        }
+    }
+}

{tokenfold-0.1.0 → tokenfold-0.2.0}/crates/tokenfold-core/src/transforms/mod.rs

@@ -4,6 +4,8 @@
 
 pub mod diff;
 pub mod json;
+pub mod json_dict;
+pub mod json_fold;
 pub mod logs;
 pub mod redaction;
 pub mod schema;

{tokenfold-0.1.0 → tokenfold-0.2.0}/crates/tokenfold-core/tests/integration.rs

@@ -258,6 +258,7 @@ fn mode_matrix_fixture_mirrors_the_rust_source_of_truth() {
         let format_from_str = |s: &str| match s {
             "openai_json" => InputFormat::OpenAiJson,
             "anthropic_json" => InputFormat::AnthropicJson,
+            "json" => InputFormat::Json,
             "plain_text" => InputFormat::PlainText,
             "command_output" => InputFormat::CommandOutput,
             "git_diff" => InputFormat::GitDiff,

{tokenfold-0.1.0 → tokenfold-0.2.0}/crates/tokenfold-py/src/lib.rs

@@ -111,6 +111,8 @@ pub enum PyInputFormat {
     OpenaiJson,
     #[pyo3(name = "ANTHROPIC_JSON")]
     AnthropicJson,
+    #[pyo3(name = "JSON")]
+    Json,
     #[pyo3(name = "PLAIN_TEXT")]
     PlainText,
     #[pyo3(name = "COMMAND_OUTPUT")]
@@ -125,6 +127,7 @@ impl From<PyInputFormat> for CoreFormat {
             PyInputFormat::Auto => CoreFormat::Auto,
             PyInputFormat::OpenaiJson => CoreFormat::OpenAiJson,
             PyInputFormat::AnthropicJson => CoreFormat::AnthropicJson,
+            PyInputFormat::Json => CoreFormat::Json,
             PyInputFormat::PlainText => CoreFormat::PlainText,
             PyInputFormat::CommandOutput => CoreFormat::CommandOutput,
             PyInputFormat::GitDiff => CoreFormat::GitDiff,
@@ -137,6 +140,7 @@ fn parse_format_str(s: &str) -> PyResult<CoreFormat> {
         "AUTO" => Ok(CoreFormat::Auto),
         "OPENAI_JSON" => Ok(CoreFormat::OpenAiJson),
         "ANTHROPIC_JSON" => Ok(CoreFormat::AnthropicJson),
+        "JSON" => Ok(CoreFormat::Json),
         "PLAIN_TEXT" => Ok(CoreFormat::PlainText),
         "COMMAND_OUTPUT" => Ok(CoreFormat::CommandOutput),
         "GIT_DIFF" => Ok(CoreFormat::GitDiff),