dumpling-cli 0.4.1__tar.gz → 0.4.3__tar.gz

{dumpling_cli-0.4.1 → dumpling_cli-0.4.3}/CHANGELOG.md

@@ -7,6 +7,19 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht
 
 ## [Unreleased]
 
+## [0.4.3] - 2026-05-03
+
+### Fixed
+
+- **COPY row integrity after anonymization**: Control characters in anonymized COPY text fields are escaped so tab/newline/etc. cannot break column alignment or row boundaries ([#53](https://github.com/ababic/dumpling/pull/53)).
+
+## [0.4.2] - 2026-05-03
+
+### Fixed
+
+- **JSON path rules on non-JSON cells**: Path-based `[rules]` anonymization is skipped when the cell is not strict JSON, leaving the original value unchanged (consistent with row-filter JSON path behavior).
+- **JSON scalar types in path-based anonymization**: Replacements at JSON paths preserve number and boolean leaf types where possible (numeric and boolean coercion from generated text).
+
 ## [0.4.1] - 2026-05-03
 
 ### Fixed
@@ -61,6 +74,8 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht
 - Configurable output scan severities and per-category thresholds via `[output_scan]`.
 - JSON report section for output scan findings including category, count, threshold, severity, and sample locations.
 
+[0.4.3]: https://github.com/ababic/dumpling/compare/v0.4.2...v0.4.3
+[0.4.2]: https://github.com/ababic/dumpling/compare/v0.4.1...v0.4.2
 [0.4.1]: https://github.com/ababic/dumpling/compare/v0.4.0...v0.4.1
 [0.4.0]: https://github.com/ababic/dumpling/compare/v0.3.0...v0.4.0
 [0.3.0]: https://github.com/ababic/dumpling/compare/v0.2.0...v0.3.0

{dumpling_cli-0.4.1 → dumpling_cli-0.4.3}/Cargo.lock

@@ -262,7 +262,7 @@ dependencies = [
 
 [[package]]
 name = "dumpling"
-version = "0.4.1"
+version = "0.4.3"
 dependencies = [
  "anyhow",
  "chrono",

{dumpling_cli-0.4.1 → dumpling_cli-0.4.3}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "dumpling"
-version = "0.4.1"
+version = "0.4.3"
 edition = "2021"
 readme = "README.md"
 

{dumpling_cli-0.4.1 → dumpling_cli-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dumpling-cli
-Version: 0.4.1
+Version: 0.4.3
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers

{dumpling_cli-0.4.1 → dumpling_cli-0.4.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "dumpling-cli"
-version = "0.4.1"
+version = "0.4.3"
 description = "Static anonymizer for plain SQL dumps (PostgreSQL, SQLite, SQL Server)."
 readme = "README.md"
 requires-python = ">=3.8"

{dumpling_cli-0.4.1 → dumpling_cli-0.4.3}/src/filter.rs

@@ -224,25 +224,118 @@ fn replacement_to_json_value(repl: &Replacement) -> serde_json::Value {
         .unwrap_or_else(|_| serde_json::Value::String(repl.value.clone()))
 }
 
+/// When rewriting JSON at a path, map `Replacement` back into [`serde_json::Value`] while keeping
+/// the leaf's JSON type when the strategy still returns text (e.g. `Replacement::quoted` for
+/// `string`, `hash`, etc.): numeric and boolean leaves stay JSON numbers/bools if the replacement
+/// text parses as such.
+fn coerce_json_path_replacement(
+    original: &serde_json::Value,
+    repl: &Replacement,
+) -> serde_json::Value {
+    if repl.is_null {
+        return serde_json::Value::Null;
+    }
+    match original {
+        serde_json::Value::Bool(_) => {
+            if let Some(b) = parse_loose_json_bool(&repl.value) {
+                return serde_json::Value::Bool(b);
+            }
+            if !repl.force_quoted {
+                if let Ok(v) = serde_json::from_str::<serde_json::Value>(&repl.value) {
+                    match v {
+                        serde_json::Value::Bool(b) => return serde_json::Value::Bool(b),
+                        serde_json::Value::Number(n) => {
+                            if n.as_u64() == Some(0) || n.as_i64() == Some(0) {
+                                return serde_json::Value::Bool(false);
+                            }
+                            if n.as_u64() == Some(1) || n.as_i64() == Some(1) {
+                                return serde_json::Value::Bool(true);
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            serde_json::Value::String(repl.value.clone())
+        }
+        serde_json::Value::Number(_) => {
+            if let Some(n) = parse_loose_json_number(&repl.value) {
+                return serde_json::Value::Number(n);
+            }
+            if !repl.force_quoted {
+                if let Ok(serde_json::Value::Number(n)) =
+                    serde_json::from_str::<serde_json::Value>(&repl.value)
+                {
+                    return serde_json::Value::Number(n);
+                }
+            }
+            serde_json::Value::String(repl.value.clone())
+        }
+        serde_json::Value::String(_) => {
+            if repl.force_quoted {
+                serde_json::Value::String(repl.value.clone())
+            } else {
+                serde_json::from_str(&repl.value)
+                    .unwrap_or_else(|_| serde_json::Value::String(repl.value.clone()))
+            }
+        }
+        serde_json::Value::Null => replacement_to_json_value(repl),
+        serde_json::Value::Array(_) | serde_json::Value::Object(_) => {
+            replacement_to_json_value(repl)
+        }
+    }
+}
+
+fn parse_loose_json_bool(s: &str) -> Option<bool> {
+    match s.trim().to_ascii_lowercase().as_str() {
+        "true" => Some(true),
+        "false" => Some(false),
+        _ => None,
+    }
+}
+
+fn parse_loose_json_number(s: &str) -> Option<serde_json::Number> {
+    let t = s.trim();
+    if t.is_empty() {
+        return None;
+    }
+    if let Ok(i) = t.parse::<i64>() {
+        return Some(i.into());
+    }
+    if let Ok(u) = t.parse::<u64>() {
+        return Some(u.into());
+    }
+    let f = t.parse::<f64>().ok()?;
+    serde_json::Number::from_f64(f)
+}
+
 fn apply_leaf_replacement(target: &mut serde_json::Value, repl: &Replacement) {
-    *target = replacement_to_json_value(repl);
+    let original = target.clone();
+    *target = coerce_json_path_replacement(&original, repl);
 }
 
 /// Mutate JSON document strings at configured paths using the same path semantics as predicates.
+///
+/// Returns [`None`] when `raw_json` is not valid strict JSON (same tolerance as row-filter JSON
+/// path extraction): path rules are skipped for that cell and callers should passthrough the
+/// original value unchanged.
 pub fn rewrite_json_paths_with_rules(
     registry: &AnonymizerRegistry,
     column_max_len: Option<usize>,
     json_rules: &[(Vec<String>, AnonymizerSpec)],
     raw_json: &str,
-) -> anyhow::Result<String> {
-    let mut root = serde_json::from_str::<serde_json::Value>(raw_json)?;
+) -> anyhow::Result<Option<String>> {
+    let mut root = match serde_json::from_str::<serde_json::Value>(raw_json) {
+        Ok(v) => v,
+        Err(_) => return Ok(None),
+    };
     for (path, spec) in json_rules {
         let mut apply = |original_cell: Option<String>| {
             apply_anonymizer(registry, spec, original_cell.as_deref(), column_max_len)
         };
         mutate_json_at_path(&mut root, path, &mut apply)?;
     }
-    Ok(root.to_string())
+    Ok(Some(root.to_string()))
 }
 
 fn mutate_json_at_path<F>(
@@ -457,7 +550,8 @@ fn get_cached_regex(pat: &str, case_insensitive: bool) -> regex::Regex {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::settings::{ResolvedConfig, RowFilterSet};
+    use crate::settings::{AnonymizerSpec, ResolvedConfig, RowFilterSet};
+    use crate::transform::AnonymizerRegistry;
     use std::collections::HashMap;
 
     #[test]
@@ -630,4 +724,139 @@ mod tests {
             &[Some(r#"{"items":[{"kind":"secondary"}]}"#.to_string())]
         ));
     }
+
+    #[test]
+    fn rewrite_json_paths_skips_non_json_cells_like_row_filters() {
+        let mut rules: HashMap<String, HashMap<String, AnonymizerSpec>> = HashMap::new();
+        let spec = AnonymizerSpec {
+            strategy: "string".to_string(),
+            salt: None,
+            min: None,
+            max: None,
+            length: Some(4),
+            min_days: None,
+            max_days: None,
+            min_seconds: None,
+            max_seconds: None,
+            domain: None,
+            unique_within_domain: None,
+            as_string: Some(true),
+            locale: None,
+            faker: None,
+            format: None,
+        };
+        rules.insert("public.t".to_string(), HashMap::new());
+        let cfg = ResolvedConfig {
+            salt: None,
+            rules,
+            row_filters: HashMap::new(),
+            column_cases: HashMap::new(),
+            sensitive_columns: HashMap::new(),
+            output_scan: crate::settings::OutputScanConfig::default(),
+            source_path: None,
+        };
+        let registry = AnonymizerRegistry::from_config(&cfg);
+        let json_rules: Vec<(Vec<String>, AnonymizerSpec)> = vec![(
+            vec!["profile".to_string(), "secret".to_string()],
+            spec.clone(),
+        )];
+        assert!(
+            rewrite_json_paths_with_rules(&registry, None, &json_rules, "{not json")
+                .unwrap()
+                .is_none()
+        );
+        let out = rewrite_json_paths_with_rules(
+            &registry,
+            None,
+            &json_rules,
+            r#"{"profile":{"secret":"x"}}"#,
+        )
+        .unwrap()
+        .expect("valid JSON should rewrite");
+        let v: serde_json::Value = serde_json::from_str(&out).unwrap();
+        assert_ne!(v["profile"]["secret"], "x");
+    }
+
+    #[test]
+    fn rewrite_json_paths_preserves_number_and_bool_leaf_types_for_quoted_replacements() {
+        let mut rules: HashMap<String, HashMap<String, AnonymizerSpec>> = HashMap::new();
+        rules.insert("public.t".to_string(), HashMap::new());
+        let cfg = ResolvedConfig {
+            salt: None,
+            rules,
+            row_filters: HashMap::new(),
+            column_cases: HashMap::new(),
+            sensitive_columns: HashMap::new(),
+            output_scan: crate::settings::OutputScanConfig::default(),
+            source_path: None,
+        };
+        let registry = AnonymizerRegistry::from_config(&cfg);
+
+        let int_spec = AnonymizerSpec {
+            strategy: "int_range".to_string(),
+            salt: None,
+            min: Some(0),
+            max: Some(9),
+            length: None,
+            min_days: None,
+            max_days: None,
+            min_seconds: None,
+            max_seconds: None,
+            domain: Some("coerce_int_leaf".to_string()),
+            unique_within_domain: None,
+            as_string: None,
+            locale: None,
+            faker: None,
+            format: None,
+        };
+        let out = rewrite_json_paths_with_rules(
+            &registry,
+            None,
+            &[(vec!["n".to_string()], int_spec)],
+            r#"{"n":1,"b":true,"s":"x"}"#,
+        )
+        .unwrap()
+        .unwrap();
+        let v: serde_json::Value = serde_json::from_str(&out).unwrap();
+        assert!(
+            v["n"].is_number(),
+            "int_range replacement should stay JSON number, got {:?}",
+            v["n"]
+        );
+        assert_eq!(v["b"], true);
+        assert_eq!(v["s"], "x");
+
+        let string_spec = AnonymizerSpec {
+            strategy: "int_range".to_string(),
+            salt: None,
+            min: Some(0),
+            max: Some(0),
+            length: None,
+            min_days: None,
+            max_days: None,
+            min_seconds: None,
+            max_seconds: None,
+            domain: Some("coerce_bool_leaf".to_string()),
+            unique_within_domain: None,
+            as_string: None,
+            locale: None,
+            faker: None,
+            format: None,
+        };
+        let out2 = rewrite_json_paths_with_rules(
+            &registry,
+            None,
+            &[(vec!["b".to_string()], string_spec)],
+            r#"{"b":false}"#,
+        )
+        .unwrap()
+        .unwrap();
+        let v2: serde_json::Value = serde_json::from_str(&out2).unwrap();
+        assert!(
+            v2["b"].is_boolean(),
+            "unquoted 0 from int_range should coerce to bool at bool leaf, got {:?}",
+            v2["b"]
+        );
+        assert_eq!(v2["b"], false);
+    }
 }

{dumpling_cli-0.4.1 → dumpling_cli-0.4.3}/src/sql.rs

@@ -299,7 +299,8 @@ impl SqlStreamProcessor {
                                     if repl.is_null {
                                         new_fields.push(r"\N".to_string());
                                     } else {
-                                        new_fields.push(repl.value);
+                                        new_fields
+                                            .push(escape_postgres_copy_text_field(&repl.value));
                                     }
                                 }
                                 Err(e) => return Err(e),
@@ -562,7 +563,11 @@ impl SqlStreamProcessor {
             None => return Ok(None),
         };
         let specs: Vec<AnonymizerSpec> = json_owned.iter().map(|(_, s)| s.clone()).collect();
-        let out = rewrite_json_paths_with_rules(&self.anonymizers, col_len, &json_owned, raw)?;
+        let Some(out) =
+            rewrite_json_paths_with_rules(&self.anonymizers, col_len, &json_owned, raw)?
+        else {
+            return Ok(None);
+        };
         let repl = Replacement::quoted(out);
         Ok(Some((repl, specs)))
     }
@@ -1177,6 +1182,32 @@ impl Cell {
     }
 }
 
+/// Escapes a field value for PostgreSQL `COPY ... FROM stdin` **text** format so the output
+/// line still has one physical TAB-separated field per logical column. Without this, a
+/// replacement containing a literal TAB or newline would split the row on restore and surface
+/// as PostgreSQL errors like `missing data for column "..."`.
+fn escape_postgres_copy_text_field(s: &str) -> String {
+    let mut out = String::with_capacity(s.len());
+    for c in s.chars() {
+        match c {
+            '\\' => out.push_str("\\\\"),
+            '\t' => out.push_str("\\t"),
+            '\n' => out.push_str("\\n"),
+            '\r' => out.push_str("\\r"),
+            '\u{0008}' => out.push_str("\\b"),
+            '\u{000c}' => out.push_str("\\f"),
+            '\u{000b}' => out.push_str("\\v"),
+            '\0' => out.push_str("\\0"),
+            c if (c as u32) < 0x20 => {
+                use std::fmt::Write;
+                let _ = write!(out, "\\x{:02x}", c as u32);
+            }
+            c => out.push(c),
+        }
+    }
+    out
+}
+
 fn render_cell(repl: &Replacement, original: &Cell) -> String {
     let trailing = original.trailing_expr.as_deref().unwrap_or("");
     if repl.is_null {
@@ -2115,6 +2146,15 @@ INSERT INTO public.users (id, email) VALUES
         );
     }
 
+    #[test]
+    fn escape_postgres_copy_text_field_escapes_control_chars() {
+        assert_eq!(
+            escape_postgres_copy_text_field("a\tb\nc\\"),
+            "a\\tb\\nc\\\\"
+        );
+        assert_eq!(escape_postgres_copy_text_field("\0\u{01}"), "\\0\\x01");
+    }
+
     #[test]
     fn domain_mapping_null_and_non_null_cross_table_consistency() {
         // When the same domain spans two tables, NULL stays NULL in both, and
@@ -2398,6 +2438,156 @@ COPY public.events (id, payload) FROM stdin;
         );
     }
 
+    #[test]
+    fn pipeline_json_path_rules_passthrough_non_json_cells() {
+        let mut rules: HashMap<String, HashMap<String, AnonymizerSpec>> = HashMap::new();
+        let mut cols: HashMap<String, AnonymizerSpec> = HashMap::new();
+        cols.insert(
+            "payload.profile.secret".to_string(),
+            AnonymizerSpec {
+                strategy: "string".to_string(),
+                salt: None,
+                min: None,
+                max: None,
+                length: Some(8),
+                min_days: None,
+                max_days: None,
+                min_seconds: None,
+                max_seconds: None,
+                domain: Some("secrets".to_string()),
+                unique_within_domain: None,
+                as_string: Some(true),
+                locale: None,
+                faker: None,
+                format: None,
+            },
+        );
+        rules.insert("public.events".to_string(), cols);
+        let cfg = ResolvedConfig {
+            salt: None,
+            rules,
+            row_filters: HashMap::new(),
+            column_cases: HashMap::new(),
+            sensitive_columns: HashMap::new(),
+            output_scan: crate::settings::OutputScanConfig::default(),
+            source_path: None,
+        };
+        let reg = AnonymizerRegistry::from_config(&cfg);
+        let mut proc =
+            SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres);
+        let input = r#"
+CREATE TABLE public.events (id int, payload jsonb);
+INSERT INTO public.events (id, payload) VALUES
+  (1, '{not strict json}'),
+  (2, '{"profile":{"tier":"gold","secret":"alpha"}}');
+
+COPY public.events (id, payload) FROM stdin;
+3	{not strict json}
+4	{"profile":{"tier":"gold","secret":"alpha"}}
+\.
+"#;
+        let mut reader = std::io::BufReader::new(input.as_bytes());
+        let mut out = Vec::new();
+        proc.process(&mut reader, &mut out).unwrap();
+        let s = String::from_utf8(out).unwrap();
+        assert!(
+            s.contains("(1, '{not strict json}')"),
+            "non-JSON INSERT cell should passthrough unchanged, got:\n{s}"
+        );
+        assert!(
+            !s.contains("alpha"),
+            "valid JSON INSERT row should still anonymize nested paths, got:\n{s}"
+        );
+        assert!(
+            s.contains("\n3\t{not strict json}\n"),
+            "non-JSON COPY cell should passthrough unchanged, got:\n{s}"
+        );
+        assert!(
+            !s.contains("\n4\t{\"profile\":{\"tier\":\"gold\",\"secret\":\"alpha\"}}\n"),
+            "valid JSON COPY row should anonymize nested secret, got:\n{s}"
+        );
+    }
+
+    #[test]
+    fn pipeline_json_path_int_range_preserves_json_number_type() {
+        let mut rules: HashMap<String, HashMap<String, AnonymizerSpec>> = HashMap::new();
+        let mut cols: HashMap<String, AnonymizerSpec> = HashMap::new();
+        cols.insert(
+            "payload.score".to_string(),
+            AnonymizerSpec {
+                strategy: "int_range".to_string(),
+                salt: None,
+                min: Some(0),
+                max: Some(100),
+                length: None,
+                min_days: None,
+                max_days: None,
+                min_seconds: None,
+                max_seconds: None,
+                domain: Some("pipeline_json_num".to_string()),
+                unique_within_domain: None,
+                as_string: None,
+                locale: None,
+                faker: None,
+                format: None,
+            },
+        );
+        rules.insert("public.events".to_string(), cols);
+        let cfg = ResolvedConfig {
+            salt: None,
+            rules,
+            row_filters: HashMap::new(),
+            column_cases: HashMap::new(),
+            sensitive_columns: HashMap::new(),
+            output_scan: crate::settings::OutputScanConfig::default(),
+            source_path: None,
+        };
+        let reg = AnonymizerRegistry::from_config(&cfg);
+        let mut proc =
+            SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres);
+        let input = r#"
+CREATE TABLE public.events (id int, payload jsonb);
+INSERT INTO public.events (id, payload) VALUES
+  (1, '{"score":42,"label":"x"}');
+
+COPY public.events (id, payload) FROM stdin;
+2	{"score":42,"label":"x"}
+\.
+"#;
+        let mut reader = std::io::BufReader::new(input.as_bytes());
+        let mut out = Vec::new();
+        proc.process(&mut reader, &mut out).unwrap();
+        let s = String::from_utf8(out).unwrap();
+        let insert_pos = s.find("INSERT INTO public.events").unwrap();
+        let insert_tail = &s[insert_pos..];
+        let insert_end = insert_tail.find(";\n").unwrap() + insert_pos;
+        let ins_stmt = &s[insert_pos..=insert_end];
+        let vals_idx = ins_stmt.to_uppercase().find("VALUES").unwrap();
+        let ins_block = strip_trailing_semicolon(ins_stmt[vals_idx + "VALUES".len()..].trim());
+        let ins_rows = parse_values_rows(ins_block).unwrap();
+        let copy_line = s
+            .lines()
+            .find(|l| l.starts_with("2\t{"))
+            .expect("expected COPY data row");
+        let copy_json = copy_line.split_once('\t').unwrap().1;
+        let v_ins =
+            serde_json::from_str::<serde_json::Value>(ins_rows[0][1].original.as_ref().unwrap())
+                .unwrap();
+        let v_copy = serde_json::from_str::<serde_json::Value>(copy_json).unwrap();
+        assert!(
+            v_ins["score"].is_number(),
+            "INSERT payload.score should remain JSON number, got {:?}",
+            v_ins["score"]
+        );
+        assert!(
+            v_copy["score"].is_number(),
+            "COPY payload.score should remain JSON number, got {:?}",
+            v_copy["score"]
+        );
+        assert_eq!(v_ins["score"], v_copy["score"]);
+        assert_eq!(v_ins["label"], "x");
+    }
+
     #[test]
     fn parse_values_rows_tracks_trailing_cast_for_quoted_literals() {
         let rows =