tokenfold 0.3.2__tar.gz → 0.3.3__tar.gz

{tokenfold-0.3.2 → tokenfold-0.3.3}/Cargo.lock

@@ -1013,7 +1013,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-adapters"
-version = "0.3.2"
+version = "0.3.3"
 dependencies = [
  "serde",
  "serde_json",
@@ -1022,7 +1022,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-admin"
-version = "0.3.2"
+version = "0.3.3"
 dependencies = [
  "ed25519-dalek",
  "serde",
@@ -1032,7 +1032,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-cli"
-version = "0.3.2"
+version = "0.3.3"
 dependencies = [
  "clap",
  "serde",
@@ -1045,7 +1045,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-core"
-version = "0.3.2"
+version = "0.3.3"
 dependencies = [
  "proptest",
  "regex",
@@ -1059,25 +1059,25 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-image"
-version = "0.3.2"
+version = "0.3.3"
 
 [[package]]
 name = "tokenfold-learn"
-version = "0.3.2"
+version = "0.3.3"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-output"
-version = "0.3.2"
+version = "0.3.3"
 dependencies = [
  "tokenfold-core",
 ]
 
 [[package]]
 name = "tokenfold-proxy"
-version = "0.3.2"
+version = "0.3.3"
 dependencies = [
  "clap",
  "serde",
@@ -1089,7 +1089,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-py"
-version = "0.3.2"
+version = "0.3.3"
 dependencies = [
  "pyo3",
  "serde_json",
@@ -1098,7 +1098,7 @@ dependencies = [
 
 [[package]]
 name = "tokenfold-rag"
-version = "0.3.2"
+version = "0.3.3"
 
 [[package]]
 name = "toml"

{tokenfold-0.3.2 → tokenfold-0.3.3}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/tokenfold-core", "crates/tokenfold-py"]
 
 [workspace.package]
-version = "0.3.2"
+version = "0.3.3"
 edition = "2024"
 rust-version = "1.97"
 license = "Apache-2.0"

{tokenfold-0.3.2 → tokenfold-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tokenfold
-Version: 0.3.2
+Version: 0.3.3
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: 3 :: Only
@@ -49,8 +49,8 @@ cargo add tokenfold-core    # Rust library
 cargo install tokenfold-cli # Rust CLI
 ```
 
-Or download the `v0.3.1` CLI for Linux, macOS, or Windows from
-[GitHub Releases](https://github.com/snchimata/tokenfold/releases/tag/v0.3.1),
+Or download the CLI for Linux, macOS, or Windows from
+[GitHub Releases](https://github.com/snchimata/tokenfold/releases/latest),
 then verify it with the adjacent `.sha256` file.
 
 Or install the Python package:
@@ -244,20 +244,6 @@ cargo run --release --locked -p tokenfold-cli -- \
 The sample reports 382 → 206 estimated tokens, a **46.1% reduction**. Ragged
 or compact inputs may save little; Tokenfold reports that result honestly.
 
-## Status
-
-Version **0.3.1** is live on every supported distribution channel:
-
-| Ecosystem | Published packages | Artifacts |
-| --- | --- | --- |
-| Python | [`tokenfold==0.3.1`](https://pypi.org/project/tokenfold/0.3.1/) | Source distribution and ABI3 wheels for Linux x64/ARM64, macOS x64/ARM64, and Windows x64 |
-| TypeScript | [`tokenfold@0.3.1`](https://www.npmjs.com/package/tokenfold/v/0.3.1) | Typed Node.js API plus the matching `@tokenfold/cli-*` native package, with no post-install download or local Rust build |
-| Rust | [`tokenfold-core`](https://crates.io/crates/tokenfold-core/0.3.1), [`tokenfold-output`](https://crates.io/crates/tokenfold-output/0.3.1), [`tokenfold-learn`](https://crates.io/crates/tokenfold-learn/0.3.1), and [`tokenfold-cli`](https://crates.io/crates/tokenfold-cli/0.3.1) | Source packages published in dependency order |
-
-[GitHub release `v0.3.1`](https://github.com/snchimata/tokenfold/releases/tag/v0.3.1)
-provides SHA-256-checksummed (not signed) CLI binaries for Linux, macOS, and
-Windows, plus CycloneDX SBOMs.
-
 ## Contributing
 
 Issues and pull requests are welcome. Run the core checks before opening a PR:

{tokenfold-0.3.2 → tokenfold-0.3.3}/README.md

@@ -37,8 +37,8 @@ cargo add tokenfold-core    # Rust library
 cargo install tokenfold-cli # Rust CLI
 ```
 
-Or download the `v0.3.1` CLI for Linux, macOS, or Windows from
-[GitHub Releases](https://github.com/snchimata/tokenfold/releases/tag/v0.3.1),
+Or download the CLI for Linux, macOS, or Windows from
+[GitHub Releases](https://github.com/snchimata/tokenfold/releases/latest),
 then verify it with the adjacent `.sha256` file.
 
 Or install the Python package:
@@ -232,20 +232,6 @@ cargo run --release --locked -p tokenfold-cli -- \
 The sample reports 382 → 206 estimated tokens, a **46.1% reduction**. Ragged
 or compact inputs may save little; Tokenfold reports that result honestly.
 
-## Status
-
-Version **0.3.1** is live on every supported distribution channel:
-
-| Ecosystem | Published packages | Artifacts |
-| --- | --- | --- |
-| Python | [`tokenfold==0.3.1`](https://pypi.org/project/tokenfold/0.3.1/) | Source distribution and ABI3 wheels for Linux x64/ARM64, macOS x64/ARM64, and Windows x64 |
-| TypeScript | [`tokenfold@0.3.1`](https://www.npmjs.com/package/tokenfold/v/0.3.1) | Typed Node.js API plus the matching `@tokenfold/cli-*` native package, with no post-install download or local Rust build |
-| Rust | [`tokenfold-core`](https://crates.io/crates/tokenfold-core/0.3.1), [`tokenfold-output`](https://crates.io/crates/tokenfold-output/0.3.1), [`tokenfold-learn`](https://crates.io/crates/tokenfold-learn/0.3.1), and [`tokenfold-cli`](https://crates.io/crates/tokenfold-cli/0.3.1) | Source packages published in dependency order |
-
-[GitHub release `v0.3.1`](https://github.com/snchimata/tokenfold/releases/tag/v0.3.1)
-provides SHA-256-checksummed (not signed) CLI binaries for Linux, macOS, and
-Windows, plus CycloneDX SBOMs.
-
 ## Contributing
 
 Issues and pull requests are welcome. Run the core checks before opening a PR:

{tokenfold-0.3.2 → tokenfold-0.3.3}/crates/tokenfold-core/src/modes.rs

@@ -15,6 +15,7 @@ pub enum TransformId {
     JsonFieldFold,
     JsonValueDict,
     SchemaCompaction,
+    LogFieldFold,
     LogCompaction,
     DiffCompaction,
 }
@@ -26,6 +27,7 @@ impl TransformId {
             TransformId::JsonFieldFold => "json_field_fold",
             TransformId::JsonValueDict => "json_value_dict",
             TransformId::SchemaCompaction => "schema_compaction",
+            TransformId::LogFieldFold => "log_field_fold",
             TransformId::LogCompaction => "log_compaction",
             TransformId::DiffCompaction => "diff_compaction",
         }
@@ -140,6 +142,25 @@ pub static ALL_ENTRIES: &[ModeEntry] = &[
         task_scopes: &[TaskScope::All],
         applicable_formats: &[InputFormat::OpenAiJson, InputFormat::AnthropicJson],
     },
+    // log_field_fold (v0.4): reversible columnar fold of TEMPLATED log lines — the log-line
+    // analogue of json_field_fold (emit each shared line skeleton once + per-line captured fields).
+    // Lossless (round-trip gated in the pipeline), so max_ratio is unrestricted (1.0), but like
+    // log_compaction it restructures what the model sees, so it stays out of Conservative and ships
+    // behind --experimental until its fidelity gate is green (the same path json_field_fold and
+    // log_compaction took). Runs before the lossy log_compaction (lossless-before-lossy ordering).
+    ModeEntry {
+        transform_id: TransformId::LogFieldFold,
+        version: "1.0.0",
+        conservative_enabled: false,
+        balanced_enabled: false,
+        aggressive_enabled: false,
+        experimental: true,
+        max_ratio_conservative: 0.0,
+        max_ratio_balanced: 1.0,
+        max_ratio_aggressive: 1.0,
+        task_scopes: &[TaskScope::All],
+        applicable_formats: &[InputFormat::PlainText, InputFormat::CommandOutput],
+    },
     ModeEntry {
         transform_id: TransformId::LogCompaction,
         version: "1.0.0",

{tokenfold-0.3.2 → tokenfold-0.3.3}/crates/tokenfold-core/src/pipeline.rs

@@ -366,6 +366,10 @@ fn apply_single_transform(
             // not that Phase 2 ship distinct values per mode).
             transforms::schema::compact_schema(bytes, 1).map_err(|e| e.to_string())
         }
+        TransformId::LogFieldFold => {
+            let text = std::str::from_utf8(bytes).map_err(|e| e.to_string())?;
+            Ok(transforms::log_fold::fold_log(text).into_bytes())
+        }
         TransformId::LogCompaction => {
             let text = std::str::from_utf8(bytes).map_err(|e| e.to_string())?;
             Ok(transforms::logs::compact(text, false).into_bytes())
@@ -423,6 +427,13 @@ fn validate_safety(
                 }
             }
         }
+        // log_field_fold restructures templated log lines into a columnar form, so its safety
+        // invariant is exact reversibility: unfolding the output must reproduce the input bytes.
+        TransformId::LogFieldFold => {
+            if !transforms::log_fold::round_trips(before, after) {
+                return false;
+            }
+        }
         TransformId::LogCompaction | TransformId::DiffCompaction => {}
     }
     safety::protected_segments_present(protected, after)

tokenfold-0.3.3/crates/tokenfold-core/src/transforms/log_fold.rs

@@ -0,0 +1,296 @@
+//! `log_field_fold` transform (canonical id `"log_field_fold"`, v1.0.0).
+//!
+//! Content-aware, **losslessly reversible** structural compression for line-oriented text
+//! (`InputFormat::PlainText` / `CommandOutput`). It is the log-line analogue of
+//! [`json_field_fold`](super::json_fold): where that folds arrays of homogeneous *objects* by
+//! emitting each repeated key once, this folds runs of *templated* log lines by emitting each
+//! repeated line **template** once.
+//!
+//! Most log lines share a fixed skeleton and vary only in a few fields (timestamps, ids, counts):
+//!
+//! ```text
+//! 2026-07-01T10:05:11Z req=req-0311 status=200 ms=41
+//! 2026-07-01T10:05:12Z req=req-0312 status=200 ms=44
+//! ```
+//!
+//! Replacing each variable token (a run of digits, or a `0x…` hex literal) with a placeholder
+//! yields one shared template `2026-\x00-\x00T\x00:\x00:\x00Z req=req-\x00 status=\x00 ms=\x00`
+//! plus a per-line tuple of the captured values. The fold emits every distinct template once and
+//! a compact row per original line (`template_id` + its captured values), so the skeleton text is
+//! paid for once instead of per line — a large win on repetitive logs, unlike
+//! [`log_compaction`](super::logs) which only collapses *identical adjacent* lines.
+//!
+//! It is a pure structural rewrite: `unfold_log` reconstructs the original bytes exactly, and the
+//! pipeline gates adoption on that round-trip ([`round_trips`]) — a fold that would ever lose data
+//! (or a genuine input that happens to look like our framing) is rolled back rather than emitted.
+
+/// Canonical transform id, as registered with the pipeline.
+pub const TRANSFORM_ID: &str = "log_field_fold";
+
+/// Semantic version of this transform's output behavior.
+pub const TRANSFORM_VERSION: &str = "1.0.0";
+
+/// First line of the folded blob. Collision-unlikely in real logs; any actual collision is caught
+/// by the pipeline's round-trip safety gate, so it never corrupts data.
+const HEADER: &str = "__tf_logfold1__";
+
+/// Placeholder standing in for a captured variable token inside a template.
+const PH: char = '\u{0}';
+
+/// Minimum number of lines worth folding, and the maximum distinct-template fraction below which a
+/// fold can pay off. Both are cheap early-outs; the pipeline also rolls back any net token
+/// regression, so these are heuristics, not the correctness boundary.
+const MIN_LINES: usize = 3;
+
+use regex::Regex;
+use std::sync::OnceLock;
+
+/// Matches a maximal variable token: a `0x…` hex literal or a run of decimal digits. Linear-time
+/// (no backtracking); see `deny.toml`.
+fn var_pattern() -> &'static Regex {
+    static RE: OnceLock<Regex> = OnceLock::new();
+    RE.get_or_init(|| Regex::new(r"0x[0-9a-fA-F]+|[0-9]+").expect("var_pattern is a valid literal"))
+}
+
+/// Replaces each variable token in `segment` with [`PH`], returning the template and the captured
+/// tokens in left-to-right order. `caps` never contain [`PH`], `\n`, or spaces (the pattern only
+/// matches `[0-9a-fA-F]`/`x`), which keeps the row serialization below unambiguous.
+fn templatize(segment: &str) -> (String, Vec<&str>) {
+    let re = var_pattern();
+    let mut template = String::with_capacity(segment.len());
+    let mut caps = Vec::new();
+    let mut last = 0;
+    for m in re.find_iter(segment) {
+        template.push_str(&segment[last..m.start()]);
+        template.push(PH);
+        caps.push(m.as_str());
+        last = m.end();
+    }
+    template.push_str(&segment[last..]);
+    (template, caps)
+}
+
+/// Folds runs of templated lines in `input` into a header + a JSON array of distinct templates +
+/// one compact row (`template_id` then captured values) per original line. Returns `input`
+/// unchanged when folding cannot help (too few lines, no shared templates, or a line contains the
+/// placeholder char). Never panics.
+pub fn fold_log(input: &str) -> String {
+    if input.is_empty() || input.contains(PH) {
+        return input.to_string();
+    }
+    // split_inclusive keeps each line's trailing '\n' as part of the segment, so reassembly is
+    // byte-exact (CRLF, blank lines, and a missing final newline are all preserved).
+    let segments: Vec<&str> = input.split_inclusive('\n').collect();
+    if segments.len() < MIN_LINES {
+        return input.to_string();
+    }
+
+    let mut templates: Vec<String> = Vec::new();
+    let mut ids: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
+    let mut rows: Vec<(usize, Vec<&str>)> = Vec::with_capacity(segments.len());
+    for seg in &segments {
+        let (template, caps) = templatize(seg);
+        let id = *ids.entry(template.clone()).or_insert_with(|| {
+            templates.push(template);
+            templates.len() - 1
+        });
+        rows.push((id, caps));
+    }
+
+    // Only worth it if templates are actually shared (skeleton text amortizes). If every line is
+    // its own template there is nothing to save; let the pipeline keep the original.
+    if templates.len() >= segments.len() {
+        return input.to_string();
+    }
+
+    let mut out = String::with_capacity(input.len() / 2);
+    out.push_str(HEADER);
+    out.push('\n');
+    out.push_str(&serde_json::to_string(&templates).expect("Vec<String> always serializes"));
+    for (id, caps) in &rows {
+        out.push('\n');
+        out.push_str(&id.to_string());
+        for cap in caps {
+            out.push(' ');
+            out.push_str(cap);
+        }
+    }
+    out
+}
+
+/// Inverse of [`fold_log`]: expands a folded blob back to the original text. Returns `input`
+/// unchanged if it is not a well-formed folded blob (so a genuine log that merely starts with the
+/// header is not mangled — the pipeline's [`round_trips`] gate makes that safe either way).
+pub fn unfold_log(input: &str) -> String {
+    match try_unfold(input) {
+        Some(s) => s,
+        None => input.to_string(),
+    }
+}
+
+fn try_unfold(input: &str) -> Option<String> {
+    let mut lines = input.split('\n');
+    if lines.next()? != HEADER {
+        return None;
+    }
+    let templates: Vec<String> = serde_json::from_str(lines.next()?).ok()?;
+    let mut out = String::with_capacity(input.len() * 2);
+    for row in lines {
+        let mut fields = row.split(' ');
+        let id: usize = fields.next()?.parse().ok()?;
+        let template = templates.get(id)?;
+        let mut caps = fields;
+        // Interleave the template's constant segments with the captured values. The number of
+        // placeholders must equal the number of captures, else this isn't our framing.
+        let mut parts = template.split(PH);
+        out.push_str(parts.next()?);
+        for part in parts {
+            out.push_str(caps.next()?);
+            out.push_str(part);
+        }
+        if caps.next().is_some() {
+            return None; // more captures than placeholders → malformed
+        }
+    }
+    Some(out)
+}
+
+/// True iff unfolding `after` reproduces `before` exactly. The pipeline's safety gate for this
+/// transform — folding is only adopted when this holds.
+pub fn round_trips(before: &[u8], after: &[u8]) -> bool {
+    let (Ok(before_s), Ok(after_s)) = (std::str::from_utf8(before), std::str::from_utf8(after))
+    else {
+        return false;
+    };
+    unfold_log(after_s) == before_s
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn assert_lossless(input: &str) {
+        let folded = fold_log(input);
+        assert!(
+            round_trips(input.as_bytes(), folded.as_bytes()),
+            "round_trips() rejected the fold of {input:?}"
+        );
+        assert_eq!(
+            unfold_log(&folded),
+            input,
+            "unfold != original for {input:?}"
+        );
+    }
+
+    #[test]
+    fn folds_templated_log_and_emits_skeleton_once() {
+        // A realistically-sized run of one skeleton, so the shared template amortizes: the header +
+        // template JSON is paid once, the per-line skeleton text is not.
+        let input: String = (0..40)
+            .map(|i| format!("req=req-{i:04} status=200 ms={}\n", 30 + i % 20))
+            .collect();
+        let folded = fold_log(&input);
+        assert!(
+            folded.starts_with(HEADER),
+            "expected folded form, got {folded:?}"
+        );
+        // the shared skeleton word "status" is emitted once (in the single template), not per line.
+        assert_eq!(folded.matches("status").count(), 1);
+        assert!(
+            folded.len() < input.len(),
+            "fold ({}) not smaller than input ({})",
+            folded.len(),
+            input.len()
+        );
+        assert_lossless(&input);
+    }
+
+    #[test]
+    fn preserves_crlf_blank_lines_and_missing_final_newline() {
+        assert_lossless("a=1\r\na=2\r\na=3\r\n");
+        assert_lossless("x=1\n\nx=2\n\nx=3\n");
+        assert_lossless("x=1\nx=2\nx=3"); // no trailing newline
+    }
+
+    #[test]
+    fn no_shared_templates_is_left_unchanged() {
+        // every line a distinct skeleton → nothing to fold.
+        let input = "alpha\nbeta gamma\ndelta epsilon zeta\n";
+        assert_eq!(fold_log(input), input);
+    }
+
+    #[test]
+    fn fewer_than_min_lines_is_left_unchanged() {
+        let input = "req=1 ok\nreq=2 ok\n";
+        assert_eq!(fold_log(input), input);
+    }
+
+    #[test]
+    fn empty_input_is_a_noop() {
+        assert_eq!(fold_log(""), "");
+        assert_eq!(unfold_log(""), "");
+    }
+
+    #[test]
+    fn lines_with_no_variable_tokens_still_fold_when_identical() {
+        // three identical constant lines share one (capture-less) template.
+        let input = "heartbeat ok\nheartbeat ok\nheartbeat ok\n";
+        assert_lossless(input);
+    }
+
+    #[test]
+    fn hex_and_decimal_tokens_both_captured() {
+        let input = "addr=0x1f val=10\naddr=0x2a val=20\naddr=0x3b val=30\n";
+        assert_lossless(input);
+        // the "addr=" / "val=" skeleton is shared → one template.
+        assert!(fold_log(input).starts_with(HEADER));
+    }
+
+    #[test]
+    fn input_containing_the_placeholder_char_is_not_folded() {
+        let input = "a\u{0}b\na\u{0}c\na\u{0}d\n";
+        assert_eq!(fold_log(input), input);
+    }
+
+    #[test]
+    fn genuine_input_shaped_like_the_header_round_trips_safely() {
+        // If real text starts with our header, unfold must not silently corrupt it: fold_log of a
+        // 2-line input is a no-op (< MIN_LINES), and round_trips gates the pipeline regardless.
+        let input = "__tf_logfold1__\n[\"x\"]\n0 boom";
+        let folded = fold_log(input);
+        // Either it wasn't folded (identity) or, if it were, the pipeline would only adopt it when
+        // round_trips holds — so the original is always recoverable either way.
+        assert_eq!(unfold_log(&folded), input);
+    }
+
+    use proptest::prelude::*;
+
+    // Lines built from a small alphabet of skeletons + digit/hex fields, so templates recur (the
+    // regime the fold targets) while still exercising CRLF, blanks, and missing final newlines.
+    fn arb_log() -> impl Strategy<Value = String> {
+        let line = prop_oneof![
+            (0u32..999u32).prop_map(|n| format!("req={n} status=200")),
+            (0u32..999u32).prop_map(|n| format!("addr=0x{n:x} ok")),
+            Just("heartbeat".to_string()),
+            "[a-z ]{0,8}".prop_map(|s| s),
+        ];
+        (prop::collection::vec(line, 0..40), any::<bool>()).prop_map(|(lines, trailing)| {
+            let mut s = lines.join("\n");
+            if trailing && !s.is_empty() {
+                s.push('\n');
+            }
+            s
+        })
+    }
+
+    proptest! {
+        // Core safety guarantee: folding never loses data. For ANY log-ish input, unfolding the
+        // folded form reproduces it exactly, and round_trips() (the pipeline's gate) agrees.
+        #[test]
+        fn fold_then_unfold_is_the_identity(input in arb_log()) {
+            let folded = fold_log(&input);
+            prop_assert!(round_trips(input.as_bytes(), folded.as_bytes()));
+            prop_assert_eq!(unfold_log(&folded), input);
+        }
+    }
+}

{tokenfold-0.3.2 → tokenfold-0.3.3}/crates/tokenfold-core/src/transforms/mod.rs

@@ -6,6 +6,7 @@ pub mod diff;
 pub mod json;
 pub mod json_dict;
 pub mod json_fold;
+pub mod log_fold;
 pub mod logs;
 pub mod redaction;
 pub mod schema;

{tokenfold-0.3.2 → tokenfold-0.3.3}/crates/tokenfold-py/Cargo.toml

@@ -16,7 +16,7 @@ name = "tokenfold_py"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-tokenfold-core = { version = "0.3.2", path = "../tokenfold-core" }
+tokenfold-core = { version = "0.3.3", path = "../tokenfold-core" }
 serde_json = { workspace = true }
 pyo3 = { version = "0.29.0", features = ["abi3-py39"] }