datafog-core 0.4.1__tar.gz → 0.4.2__tar.gz

{datafog_core-0.4.1 → datafog_core-0.4.2}/Cargo.lock

@@ -80,7 +80,7 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"
 
 [[package]]
 name = "datafog-core"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "base64",
  "futures",
@@ -94,7 +94,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-core-python"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "datafog-core",
  "pyo3",
@@ -104,7 +104,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-node"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "datafog-core",
  "napi",
@@ -115,7 +115,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-wasm"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "datafog-core",
  "serde",

{datafog_core-0.4.1 → datafog_core-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datafog-core
-Version: 0.4.1
+Version: 0.4.2
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
@@ -79,8 +79,12 @@ full-match regex values:
 }
 ```
 
-`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean }, transform: ... }` so
+`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean, format?: "text" | "env" | "sql" }, transform: ... }` so
 detection settings remain separate from transformation policy.
+For `.env` assignments or standard SQL quoted strings, select `format: "env"`
+or `format: "sql"` to preserve email boundaries. See the
+[email source-context policy](docs/guides/email-boundaries.mdx) for supported
+syntax, raw escaped spans, and limitations. Default text scanning is unchanged.
 
 ## Packages
 
@@ -321,8 +325,8 @@ boundaries, never newlines. Returned text and offsets preserve the source.
 Passport and residence-permit patterns are legacy heuristics with limited
 coverage. See the [German entity reference](docs/reference/german-entities.mdx)
 and [migration differences](docs/guides/migrating-from-datafog-python.mdx).
-Core 0.4.0 is published. The higher-level Python adapter and its dependency
-update remain separate and unreleased in [draft PR #179](https://github.com/DataFog/datafog-python/pull/179).
+The higher-level Python adapter and its dependency update were merged in
+[PR #179](https://github.com/DataFog/datafog-python/pull/179). Its package release is separate from Core.
 
 ## UUID identifiers
 
@@ -332,7 +336,7 @@ structured scans, independently of locale. It emits `UUID` findings for versions
 sensitivity. See the [UUID reference](docs/reference/uuid.mdx) for boundaries,
 excluded sentinel forms and transformation examples.
 
-**0.4.0 is published** for Rust, Python, Node.js, and WASM. The higher-level Python adapter remains a separate unreleased integration in [draft PR #179](https://github.com/DataFog/datafog-python/pull/179). See the [release notes](docs/releases/0-4-0.mdx), [runtime capabilities](docs/reference/capabilities.mdx), and [0.4.x compatibility policy](docs/reference/compatibility.mdx).
+**0.4.1 is published** for Rust, Python, Node.js, and WASM. The higher-level Python adapter is integrated in [merged PR #179](https://github.com/DataFog/datafog-python/pull/179); its package release is separate. See the [release notes](docs/releases/0-4-1.mdx), [runtime capabilities](docs/reference/capabilities.mdx), and [0.4.x compatibility policy](docs/reference/compatibility.mdx).
 
-The unreleased **0.4.1 candidate** adds three default detectors: provider-prefixed [`API_KEY`](docs/reference/api-keys.mdx), context-required [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), and whole PostgreSQL [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx). The registry reports 23 supported entities and 14 defaults with capability contract 1. See the [candidate release notes](docs/releases/0-4-1.mdx) for exact scopes, preserved legacy Python overlap behavior, and the required exact-wheel Python 4.9 integration gate. Python's default backend is unchanged.
+**Core 0.4.1** adds three default detectors: provider-prefixed [`API_KEY`](docs/reference/api-keys.mdx), context-required [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), and whole PostgreSQL [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx). The registry reports 23 supported entities and 14 defaults with capability contract 1. See the [release notes](docs/releases/0-4-1.mdx) for exact scopes, preserved legacy Python overlap behavior, and the passed local exact-wheel Python 4.9 integration gate. Python's default backend is unchanged.
 

{datafog_core-0.4.1/crates/core → datafog_core-0.4.2}/README.md

@@ -56,8 +56,12 @@ full-match regex values:
 }
 ```
 
-`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean }, transform: ... }` so
+`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean, format?: "text" | "env" | "sql" }, transform: ... }` so
 detection settings remain separate from transformation policy.
+For `.env` assignments or standard SQL quoted strings, select `format: "env"`
+or `format: "sql"` to preserve email boundaries. See the
+[email source-context policy](docs/guides/email-boundaries.mdx) for supported
+syntax, raw escaped spans, and limitations. Default text scanning is unchanged.
 
 ## Packages
 
@@ -298,8 +302,8 @@ boundaries, never newlines. Returned text and offsets preserve the source.
 Passport and residence-permit patterns are legacy heuristics with limited
 coverage. See the [German entity reference](docs/reference/german-entities.mdx)
 and [migration differences](docs/guides/migrating-from-datafog-python.mdx).
-Core 0.4.0 is published. The higher-level Python adapter and its dependency
-update remain separate and unreleased in [draft PR #179](https://github.com/DataFog/datafog-python/pull/179).
+The higher-level Python adapter and its dependency update were merged in
+[PR #179](https://github.com/DataFog/datafog-python/pull/179). Its package release is separate from Core.
 
 ## UUID identifiers
 
@@ -309,6 +313,6 @@ structured scans, independently of locale. It emits `UUID` findings for versions
 sensitivity. See the [UUID reference](docs/reference/uuid.mdx) for boundaries,
 excluded sentinel forms and transformation examples.
 
-**0.4.0 is published** for Rust, Python, Node.js, and WASM. The higher-level Python adapter remains a separate unreleased integration in [draft PR #179](https://github.com/DataFog/datafog-python/pull/179). See the [release notes](docs/releases/0-4-0.mdx), [runtime capabilities](docs/reference/capabilities.mdx), and [0.4.x compatibility policy](docs/reference/compatibility.mdx).
+**0.4.1 is published** for Rust, Python, Node.js, and WASM. The higher-level Python adapter is integrated in [merged PR #179](https://github.com/DataFog/datafog-python/pull/179); its package release is separate. See the [release notes](docs/releases/0-4-1.mdx), [runtime capabilities](docs/reference/capabilities.mdx), and [0.4.x compatibility policy](docs/reference/compatibility.mdx).
 
-The unreleased **0.4.1 candidate** adds three default detectors: provider-prefixed [`API_KEY`](docs/reference/api-keys.mdx), context-required [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), and whole PostgreSQL [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx). The registry reports 23 supported entities and 14 defaults with capability contract 1. See the [candidate release notes](docs/releases/0-4-1.mdx) for exact scopes, preserved legacy Python overlap behavior, and the required exact-wheel Python 4.9 integration gate. Python's default backend is unchanged.
+**Core 0.4.1** adds three default detectors: provider-prefixed [`API_KEY`](docs/reference/api-keys.mdx), context-required [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), and whole PostgreSQL [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx). The registry reports 23 supported entities and 14 defaults with capability contract 1. See the [release notes](docs/releases/0-4-1.mdx) for exact scopes, preserved legacy Python overlap behavior, and the passed local exact-wheel Python 4.9 integration gate. Python's default backend is unchanged.

{datafog_core-0.4.1 → datafog_core-0.4.2}/bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core-python"
-version = "0.4.1"
+version = "0.4.2"
 edition = "2024"
 rust-version = "1.88"
 description = "Python bindings for datafog-core"

datafog_core-0.4.2/bindings/python/tests/email_context_conformance.py

@@ -0,0 +1,49 @@
+"""Email source-context fixtures against the installed extension."""
+
+import json
+from pathlib import Path
+
+import datafog_core as api
+
+
+def verify():
+    fixture = Path(__file__).resolve().parents[3] / "fixtures/email-context.jsonl"
+    for line in fixture.read_text().splitlines():
+        row = json.loads(line)
+        text = row["text"]
+        findings = api.scan(text, row["config"])
+        emails = [f for f in findings if f.entity_type == "EMAIL"]
+        assert [
+            dict(text=f.matched_text, start=f.codepoint_range.start, end=f.codepoint_range.end)
+            for f in emails
+        ] == row["matches"], row["id"]
+        for finding in emails:
+            assert text.encode()[finding.byte_range.start:finding.byte_range.end].decode() == finding.matched_text
+            assert text[finding.codepoint_range.start:finding.codepoint_range.end] == finding.matched_text
+        data = {"source": text}
+        located = api.scan_structured(data, row["config"]).findings
+        assert [f.finding for f in located] == findings
+        for strategy in ["redact", "mask", "remove"]:
+            policy = {"default": {"strategy": strategy}, "entities": ["EMAIL"]}
+            envelope = {"scan": row["config"], "transform": policy}
+            result = api.scan_and_transform(text, envelope)
+            assert result.text == row["outputs"][strategy], (row["id"], strategy)
+            assert api.transform(text, findings, policy) == result
+            for record, finding in zip(result.transformations, emails):
+                assert record.source_byte_range == finding.byte_range
+                assert record.source_codepoint_range == finding.codepoint_range
+                assert result.text.encode()[record.output_byte_range.start:record.output_byte_range.end].decode() == record.replacement
+            assert api.scan_and_transform_structured(data, envelope).data == {"source": result.text}
+        if row["config"]["format"] == "text":
+            assert api.scan(text) == findings
+    for invalid in [None, 1, True, {}, "yaml", "ENV"]:
+        for call in [
+            lambda: api.scan("", {"format": invalid}),
+            lambda: api.scan_structured({"source": ""}, {"format": invalid}),
+        ]:
+            try:
+                call()
+            except api.DataFogConfigurationError as error:
+                assert error.path == "/format"
+            else:
+                raise AssertionError("Invalid source format accepted")

{datafog_core-0.4.1 → datafog_core-0.4.2}/bindings/python/tests/test_installed.py

@@ -12,6 +12,7 @@ import jwt_conformance
 import bearer_token_conformance
 import private_key_conformance
 import api_key_conformance
+import email_context_conformance
 import uuid_conformance
 import credential_uri_conformance
 import us_routing_number_conformance
@@ -133,6 +134,7 @@ def main() -> None:
     bearer_token_conformance.verify()
     private_key_conformance.verify()
     api_key_conformance.verify()
+    email_context_conformance.verify()
     uuid_conformance.verify()
     credential_uri_conformance.verify()
     us_routing_number_conformance.verify()

{datafog_core-0.4.1 → datafog_core-0.4.2}/bindings/python/tests/test_typing.py

@@ -42,7 +42,7 @@ config: _TransformationConfig = {
     },
     "allow": {"exact": {"EMAIL": ["safe@example.com"]}, "regex": {"EMAIL": [{"pattern": "safe", "case_sensitive": False}]}},
 }
-findings = scan("jane@example.com", {"locale": "en-US", "detect_uuid": True})
+findings = scan("jane@example.com", {"locale": "en-US", "detect_uuid": True, "format": "env"})
 assert_type(findings, list[Finding])
 finding = findings[0]
 assert_type(finding.entity_type, str)
@@ -127,6 +127,7 @@ INVALID = """
 from datafog_core import capabilities, scan, transform, scan_structured, PrivacyManager, TransformResult
 capabilities(1)  # error
 scan(123)  # error
+scan("text", {"format": "yaml"})  # error
 scan("text", {"locale": 123})  # error
 scan("text", {"detect_uuid": "true"})  # error
 scan("text", {"max_bytes": 100})  # error

{datafog_core-0.4.1 → datafog_core-0.4.2}/crates/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core"
-version = "0.4.1"
+version = "0.4.2"
 edition = "2024"
 rust-version = "1.88"
 description = "Fast structured PII detection library"

{datafog_core-0.4.1 → datafog_core-0.4.2/crates/core}/README.md

@@ -56,8 +56,12 @@ full-match regex values:
 }
 ```
 
-`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean }, transform: ... }` so
+`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean, format?: "text" | "env" | "sql" }, transform: ... }` so
 detection settings remain separate from transformation policy.
+For `.env` assignments or standard SQL quoted strings, select `format: "env"`
+or `format: "sql"` to preserve email boundaries. See the
+[email source-context policy](docs/guides/email-boundaries.mdx) for supported
+syntax, raw escaped spans, and limitations. Default text scanning is unchanged.
 
 ## Packages
 
@@ -298,8 +302,8 @@ boundaries, never newlines. Returned text and offsets preserve the source.
 Passport and residence-permit patterns are legacy heuristics with limited
 coverage. See the [German entity reference](docs/reference/german-entities.mdx)
 and [migration differences](docs/guides/migrating-from-datafog-python.mdx).
-Core 0.4.0 is published. The higher-level Python adapter and its dependency
-update remain separate and unreleased in [draft PR #179](https://github.com/DataFog/datafog-python/pull/179).
+The higher-level Python adapter and its dependency update were merged in
+[PR #179](https://github.com/DataFog/datafog-python/pull/179). Its package release is separate from Core.
 
 ## UUID identifiers
 
@@ -309,6 +313,6 @@ structured scans, independently of locale. It emits `UUID` findings for versions
 sensitivity. See the [UUID reference](docs/reference/uuid.mdx) for boundaries,
 excluded sentinel forms and transformation examples.
 
-**0.4.0 is published** for Rust, Python, Node.js, and WASM. The higher-level Python adapter remains a separate unreleased integration in [draft PR #179](https://github.com/DataFog/datafog-python/pull/179). See the [release notes](docs/releases/0-4-0.mdx), [runtime capabilities](docs/reference/capabilities.mdx), and [0.4.x compatibility policy](docs/reference/compatibility.mdx).
+**0.4.1 is published** for Rust, Python, Node.js, and WASM. The higher-level Python adapter is integrated in [merged PR #179](https://github.com/DataFog/datafog-python/pull/179); its package release is separate. See the [release notes](docs/releases/0-4-1.mdx), [runtime capabilities](docs/reference/capabilities.mdx), and [0.4.x compatibility policy](docs/reference/compatibility.mdx).
 
-The unreleased **0.4.1 candidate** adds three default detectors: provider-prefixed [`API_KEY`](docs/reference/api-keys.mdx), context-required [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), and whole PostgreSQL [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx). The registry reports 23 supported entities and 14 defaults with capability contract 1. See the [candidate release notes](docs/releases/0-4-1.mdx) for exact scopes, preserved legacy Python overlap behavior, and the required exact-wheel Python 4.9 integration gate. Python's default backend is unchanged.
+**Core 0.4.1** adds three default detectors: provider-prefixed [`API_KEY`](docs/reference/api-keys.mdx), context-required [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), and whole PostgreSQL [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx). The registry reports 23 supported entities and 14 defaults with capability contract 1. See the [release notes](docs/releases/0-4-1.mdx) for exact scopes, preserved legacy Python overlap behavior, and the passed local exact-wheel Python 4.9 integration gate. Python's default backend is unchanged.

{datafog_core-0.4.1 → datafog_core-0.4.2}/crates/core/src/capabilities.rs

@@ -152,7 +152,11 @@ pub(super) fn detect(text: &str, config: &ScanConfig, candidates: &mut Vec<Candi
         .iter()
         .filter(|detector| detector.activation.enabled(config))
     {
-        (detector.detect)(text, candidates);
+        if detector.labels == [Label::Email] {
+            crate::email_context::detect(text, config.format(), candidates);
+        } else {
+            (detector.detect)(text, candidates);
+        }
     }
 }
 

datafog_core-0.4.2/crates/core/src/email_context.rs

@@ -0,0 +1,172 @@
+//! Delimit original source slices before applying the shared email matcher.
+use crate::{Candidate, ScanFormat, detect_email};
+
+pub(super) fn detect(text: &str, format: ScanFormat, candidates: &mut Vec<Candidate>) {
+    match format {
+        ScanFormat::Text => detect_email(text, candidates),
+        ScanFormat::Env => detect_env(text, candidates),
+        ScanFormat::Sql => detect_sql(text, candidates),
+    }
+}
+
+fn detect_slice(text: &str, start: usize, end: usize, candidates: &mut Vec<Candidate>) {
+    let first = candidates.len();
+    detect_email(&text[start..end], candidates);
+    for candidate in &mut candidates[first..] {
+        candidate.start_byte += start;
+        candidate.end_byte += start;
+    }
+}
+
+fn line_end(text: &str, start: usize) -> usize {
+    text[start..]
+        .find('\n')
+        .map_or(text.len(), |end| start + end)
+}
+
+fn skip_horizontal_space(bytes: &[u8], mut cursor: usize) -> usize {
+    while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\r')) {
+        cursor += 1;
+    }
+    cursor
+}
+
+fn env_value_start(line: &str) -> Option<usize> {
+    let bytes = line.as_bytes();
+    let mut cursor = skip_horizontal_space(bytes, 0);
+    if line[cursor..].starts_with("export") && matches!(bytes.get(cursor + 6), Some(b' ' | b'\t')) {
+        cursor = skip_horizontal_space(bytes, cursor + 6);
+    }
+    if !bytes
+        .get(cursor)
+        .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_')
+    {
+        return None;
+    }
+    cursor += 1;
+    while bytes
+        .get(cursor)
+        .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_')
+    {
+        cursor += 1;
+    }
+    cursor = skip_horizontal_space(bytes, cursor);
+    if bytes.get(cursor) != Some(&b'=') {
+        return None;
+    }
+    Some(skip_horizontal_space(bytes, cursor + 1))
+}
+
+// Return the closing delimiter or EOF. No values are decoded, so offsets and
+// escaped local parts remain contiguous spans of the caller's source text.
+fn quote_end(bytes: &[u8], opening: usize, doubled: bool, backslash: bool) -> usize {
+    let quote = bytes[opening];
+    let mut cursor = opening + 1;
+    while cursor < bytes.len() {
+        if backslash && bytes[cursor] == b'\\' {
+            cursor += if cursor + 1 < bytes.len() { 2 } else { 1 };
+        } else if bytes[cursor] == quote {
+            if doubled && bytes.get(cursor + 1) == Some(&quote) {
+                cursor += 2;
+            } else {
+                return cursor;
+            }
+        } else {
+            cursor += 1;
+        }
+    }
+    bytes.len()
+}
+
+fn detect_env(text: &str, candidates: &mut Vec<Candidate>) {
+    let bytes = text.as_bytes();
+    let mut cursor = 0;
+    while cursor < text.len() {
+        let end = line_end(text, cursor);
+        if let Some(value) = env_value_start(&text[cursor..end]) {
+            let start = cursor + value;
+            if matches!(bytes.get(start), Some(b'\'' | b'"')) {
+                let close = quote_end(bytes, start, false, bytes[start] == b'"');
+                detect_slice(text, start + 1, close, candidates);
+                if close == text.len() {
+                    break;
+                }
+                let tail_end = line_end(text, close + 1);
+                detect_env_unquoted(text, close + 1, tail_end, candidates);
+                cursor = tail_end;
+            } else {
+                detect_env_unquoted(text, start, end, candidates);
+                cursor = end;
+            }
+        } else {
+            detect_env_unquoted(text, cursor, end, candidates);
+            cursor = end;
+        }
+        if cursor < text.len() {
+            cursor += 1;
+        }
+    }
+}
+
+fn detect_env_unquoted(text: &str, start: usize, end: usize, candidates: &mut Vec<Candidate>) {
+    let bytes = text.as_bytes();
+    for cursor in start..end {
+        if bytes[cursor] == b'#' && (cursor == start || bytes[cursor - 1].is_ascii_whitespace()) {
+            detect_slice(text, start, cursor, candidates);
+            detect_slice(text, cursor + 1, end, candidates);
+            return;
+        }
+    }
+    detect_slice(text, start, end, candidates);
+}
+
+fn detect_sql(text: &str, candidates: &mut Vec<Candidate>) {
+    let bytes = text.as_bytes();
+    let mut cursor = 0;
+    let mut plain_start = 0;
+    while cursor < bytes.len() {
+        let end = if matches!(bytes[cursor], b'\'' | b'"') {
+            detect_slice(text, plain_start, cursor, candidates);
+            let close = quote_end(bytes, cursor, true, false);
+            detect_slice(text, cursor + 1, close, candidates);
+            if close < bytes.len() {
+                close + 1
+            } else {
+                close
+            }
+        } else if bytes[cursor..].starts_with(b"--") {
+            detect_slice(text, plain_start, cursor, candidates);
+            let end = line_end(text, cursor);
+            detect_slice(text, cursor + 2, end, candidates);
+            end
+        } else if bytes[cursor..].starts_with(b"/*") {
+            detect_slice(text, plain_start, cursor, candidates);
+            let mut end = cursor + 2;
+            let mut comment_start = end;
+            let mut depth = 1;
+            while end < bytes.len() && depth > 0 {
+                if bytes[end..].starts_with(b"/*") {
+                    detect_slice(text, comment_start, end, candidates);
+                    depth += 1;
+                    end += 2;
+                    comment_start = end;
+                } else if bytes[end..].starts_with(b"*/") {
+                    detect_slice(text, comment_start, end, candidates);
+                    depth -= 1;
+                    end += 2;
+                    comment_start = end;
+                } else {
+                    end += 1;
+                }
+            }
+            detect_slice(text, comment_start, end, candidates);
+            end
+        } else {
+            cursor += 1;
+            continue;
+        };
+        cursor = end;
+        plain_start = end;
+    }
+    detect_slice(text, plain_start, text.len(), candidates);
+}

{datafog_core-0.4.1 → datafog_core-0.4.2}/crates/core/src/lib.rs

@@ -3,6 +3,7 @@ mod api_key;
 mod bearer_token;
 mod capabilities;
 mod credential_uri;
+mod email_context;
 pub use capabilities::{
     ActivationScanConfig, Capabilities, EntityActivation, EntityCapabilities, LocaleCapabilities,
     capabilities,
@@ -535,15 +536,39 @@ fn json_pointer_segment(segment: &str) -> String {
     segment.replace('~', "~0").replace('/', "~1")
 }
 
-/// Scanner configuration. Current built-in detectors share the same execution
-/// path; German locale aliases activate German detectors, and UUIDs are opt-in.
+/// Source syntax used to delimit email candidates. Other detectors are unchanged.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub enum ScanFormat {
+    /// Unstructured text; retain the full supported email local-part alphabet.
+    #[default]
+    Text,
+    /// Environment assignments with optional `export` and quoted values.
+    Env,
+    /// SQL strings using standard doubled-quote escaping.
+    Sql,
+}
+
+/// Scanner configuration. German locale aliases activate German detectors,
+/// UUIDs are opt-in, and source format controls email boundaries.
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
 pub struct ScanConfig {
     locale: Option<String>,
     detect_uuid: bool,
+    format: ScanFormat,
 }
 
 impl ScanConfig {
+    /// Select source syntax for email boundaries; defaults to unstructured text.
+    pub fn with_format(mut self, format: ScanFormat) -> Self {
+        self.format = format;
+        self
+    }
+
+    /// Source syntax used for email boundaries.
+    pub fn format(&self) -> ScanFormat {
+        self.format
+    }
+
     /// Create scanner configuration using detector defaults.
     pub fn new() -> Self {
         Self::default()
@@ -755,8 +780,11 @@ pub fn parse_scan_and_transform_config(
 /// Parse canonical scanner configuration.
 pub fn parse_scan_config(value: &serde_json::Value) -> Result<ScanConfig, PrivacyError> {
     let object = require_object(value, "", "scan configuration must be an object")?;
-    reject_unknown_fields(object, &["locale", "detect_uuid"], "")?;
+    reject_unknown_fields(object, &["locale", "detect_uuid", "format"], "")?;
     let mut config = ScanConfig::new();
+    if let Some(format) = object.get("format") {
+        config = config.with_format(parse_scan_format(format)?);
+    }
     if let Some(enabled) = object.get("detect_uuid") {
         config = config.with_uuid_detection(parse_uuid_detection(enabled)?);
     }
@@ -767,6 +795,20 @@ pub fn parse_scan_config(value: &serde_json::Value) -> Result<ScanConfig, Privac
     Ok(config)
 }
 
+fn parse_scan_format(value: &serde_json::Value) -> Result<ScanFormat, PrivacyError> {
+    let format = require_string(value, "/format", "scan format must be a string")?;
+    match format.as_str() {
+        "text" => Ok(ScanFormat::Text),
+        "env" => Ok(ScanFormat::Env),
+        "sql" => Ok(ScanFormat::Sql),
+        _ => Err(PrivacyError::invalid_configuration(
+            PrivacyErrorReason::InvalidValue,
+            "/format",
+            "scan format must be text, env, or sql",
+        )),
+    }
+}
+
 fn parse_uuid_detection(value: &serde_json::Value) -> Result<bool, PrivacyError> {
     value.as_bool().ok_or_else(|| {
         PrivacyError::invalid_configuration(

{datafog_core-0.4.1 → datafog_core-0.4.2}/crates/core/src/structured.rs

@@ -79,7 +79,7 @@ fn validate_pointer(pointer: &str, path: &str) -> Result<(), PrivacyError> {
     Ok(())
 }
 
-/// Parse `{ locale?, detect_uuid?, discover_person?, mappings?: {pointer: "PERSON"}, exclude?: [pointer] }`.
+/// Parse `{ locale?, detect_uuid?, format?, discover_person?, mappings?: {pointer: "PERSON"}, exclude?: [pointer] }`.
 pub fn parse_scan_config(value: &Value) -> Result<StructuredScanConfig, PrivacyError> {
     let object = require_object(value, "", "structured scan configuration must be an object")?;
     reject_unknown_fields(
@@ -87,6 +87,7 @@ pub fn parse_scan_config(value: &Value) -> Result<StructuredScanConfig, PrivacyE
         &[
             "locale",
             "detect_uuid",
+            "format",
             "discover_person",
             "mappings",
             "exclude",
@@ -94,6 +95,9 @@ pub fn parse_scan_config(value: &Value) -> Result<StructuredScanConfig, PrivacyE
         "",
     )?;
     let mut config = StructuredScanConfig::default();
+    if let Some(format) = object.get("format") {
+        config.scan = config.scan.with_format(parse_scan_format(format)?);
+    }
     if let Some(enabled) = object.get("detect_uuid") {
         config.scan = config
             .scan

datafog_core-0.4.2/crates/core/tests/email_context.rs

@@ -0,0 +1,107 @@
+use datafog_core::{
+    parse_scan_and_transform_config, parse_scan_config, parse_transformation_config, scan,
+    scan_and_transform, scan_with_config, structured, transform, utf16_range,
+};
+use serde_json::{Value, json};
+
+#[test]
+fn email_context_preserves_source_spans_and_syntax() {
+    for line in include_str!("../../../fixtures/email-context.jsonl").lines() {
+        let row: Value = serde_json::from_str(line).unwrap();
+        let text = row["text"].as_str().unwrap();
+        let findings = scan_with_config(text, &parse_scan_config(&row["config"]).unwrap());
+        let emails: Vec<_> = findings
+            .iter()
+            .filter(|f| f.entity_type == "EMAIL")
+            .collect();
+        let matches: Vec<_> = emails.iter().map(|f| {
+            assert_eq!(&text[f.byte_range.start..f.byte_range.end], f.matched_text);
+            let range = utf16_range(text, f.byte_range).unwrap();
+            let units: Vec<_> = text.encode_utf16().collect();
+            assert_eq!(String::from_utf16(&units[range.start..range.end]).unwrap(), f.matched_text);
+            json!({"text":f.matched_text,"start":f.codepoint_range.start,"end":f.codepoint_range.end})
+        }).collect();
+        assert_eq!(json!(matches), row["matches"], "{}", row["id"]);
+        let data = json!({"source":text});
+        let located = structured::scan(
+            &data,
+            &structured::parse_scan_config(&row["config"]).unwrap(),
+        )
+        .unwrap();
+        assert_eq!(
+            located
+                .findings
+                .iter()
+                .map(|f| f.finding.clone())
+                .collect::<Vec<_>>(),
+            findings
+        );
+        for strategy in ["redact", "mask", "remove"] {
+            let policy = json!({"default":{"strategy":strategy},"entities":["EMAIL"]});
+            let envelope = json!({"scan":row["config"],"transform":policy});
+            let result =
+                scan_and_transform(text, &parse_scan_and_transform_config(&envelope).unwrap())
+                    .unwrap();
+            assert_eq!(
+                result.text, row["outputs"][strategy],
+                "{} {strategy}",
+                row["id"]
+            );
+            assert_eq!(
+                transform(
+                    text,
+                    &findings,
+                    &parse_transformation_config(&policy).unwrap()
+                )
+                .unwrap(),
+                result
+            );
+            for (record, finding) in result.transformations.iter().zip(&emails) {
+                assert_eq!(record.source_byte_range, finding.byte_range);
+                assert_eq!(record.source_codepoint_range, finding.codepoint_range);
+                assert_eq!(
+                    &result.text[record.output_byte_range.start..record.output_byte_range.end],
+                    record.replacement
+                );
+            }
+            let result = structured::scan_and_transform(
+                &data,
+                &structured::parse_scan_and_transform_config(&envelope).unwrap(),
+            )
+            .unwrap();
+            assert_eq!(result.data, json!({"source":row["outputs"][strategy]}));
+        }
+        if row["config"]["format"] == "text" {
+            assert_eq!(scan(text), findings);
+        }
+    }
+}
+
+#[test]
+fn invalid_format_is_reported_at_the_configuration_path() {
+    for format in [
+        json!(null),
+        json!(1),
+        json!(true),
+        json!({}),
+        json!("yaml"),
+        json!("ENV"),
+    ] {
+        let config = json!({"format":format});
+        assert_eq!(
+            parse_scan_config(&config).unwrap_err().path(),
+            Some("/format")
+        );
+        assert_eq!(
+            structured::parse_scan_config(&config).unwrap_err().path(),
+            Some("/format")
+        );
+        let envelope = json!({"scan":config,"transform":{"default":{"strategy":"redact"}}});
+        assert_eq!(
+            parse_scan_and_transform_config(&envelope)
+                .unwrap_err()
+                .path(),
+            Some("/scan/format")
+        );
+    }
+}

{datafog_core-0.4.1 → datafog_core-0.4.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "datafog-core"
-version = "0.4.1"
+version = "0.4.2"
 description = "Fast structured PII detection implemented in Rust"
 readme = "README.md"
 license = "MIT"

{datafog_core-0.4.1 → datafog_core-0.4.2}/python/datafog_core/__init__.pyi

@@ -9,6 +9,7 @@ _JsonValue: TypeAlias = (
 _JsonDocument: TypeAlias = list[_JsonValue] | dict[str, _JsonValue]
 
 class _ScanConfig(TypedDict, total=False):
+    format: Literal["text", "env", "sql"]
     locale: str
     detect_uuid: bool