datafog-core 0.4.2__tar.gz → 0.4.4__tar.gz

{datafog_core-0.4.2 → datafog_core-0.4.4}/Cargo.lock

@@ -80,7 +80,7 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"
 
 [[package]]
 name = "datafog-core"
-version = "0.4.2"
+version = "0.4.4"
 dependencies = [
  "base64",
  "futures",
@@ -94,7 +94,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-core-python"
-version = "0.4.2"
+version = "0.4.4"
 dependencies = [
  "datafog-core",
  "pyo3",
@@ -104,7 +104,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-node"
-version = "0.4.2"
+version = "0.4.4"
 dependencies = [
  "datafog-core",
  "napi",
@@ -115,7 +115,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-wasm"
-version = "0.4.2"
+version = "0.4.4"
 dependencies = [
  "datafog-core",
  "serde",

{datafog_core-0.4.2 → datafog_core-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datafog-core
-Version: 0.4.2
+Version: 0.4.4
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
@@ -38,6 +38,13 @@ existing detectors and returns field paths plus string-local findings. No model
 or dictionary download is needed. See [person-field discovery](docs/guides/person-discovery.mdx)
 for `scan_structured` / `scanStructured` and structured transformation APIs.
 
+Phone candidates without `+` are interpreted as North American numbers: ten
+digits, optionally preceded by `1`, with area and exchange codes starting with
+`2` through `9`. Candidates starting with `+` retain the existing permissive
+7–15 digit check, including shortened synthetic examples such as `+1-555-1002`.
+These checks do not validate country calling codes or assigned phone numbers,
+and numeric identifiers that fit the accepted patterns can still match `PHONE`.
+
 Both ranges use zero-based, end-exclusive offsets. The byte range addresses the
 UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based
 detectors currently report no confidence score. Node.js and browser WASM also
@@ -79,13 +86,26 @@ full-match regex values:
 }
 ```
 
-`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean, format?: "text" | "env" | "sql" }, transform: ... }` so
+`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean, format?: "text" | "env" | "sql", entities?: string[] }, transform: ... }` so
 detection settings remain separate from transformation policy.
 For `.env` assignments or standard SQL quoted strings, select `format: "env"`
 or `format: "sql"` to preserve email boundaries. See the
 [email source-context policy](docs/guides/email-boundaries.mdx) for supported
 syntax, raw escaped spans, and limitations. Default text scanning is unchanged.
 
+## Credit-card detection and decimal numbers
+
+To reduce decimal false positives, `CREDIT_CARD` detection suppresses contiguous
+digit candidates immediately after a digit followed by `.` or `,`, such as
+`3.4111111111111111` and `3,4111111111111111`. Cards containing spaces or hyphens,
+such as `3.4111-1111-1111-1111`, still undergo the normal card checks.
+
+In plain text, `3,4111111111111111` could also be two CSV fields: `3` and a card
+number. The detector cannot distinguish that structure from a comma-decimal
+number, so it suppresses the card candidate. For CSV input, parse the CSV and
+scan each field separately so a card value is checked without the preceding
+field and delimiter.
+
 ## Packages
 
 | Runtime | Distribution | Import | Status |
@@ -328,6 +348,26 @@ and [migration differences](docs/guides/migrating-from-datafog-python.mdx).
 The higher-level Python adapter and its dependency update were merged in
 [PR #179](https://github.com/DataFog/datafog-python/pull/179). Its package release is separate from Core.
 
+## Text detector selection
+
+Pass `{"entities":["EMAIL","SSN"]}` to a text scan to run only those
+selected detectors, rather than detecting every type and filtering findings
+later. In Rust, use `ScanConfig::new().with_entities(vec!["EMAIL".into(),
+"SSN".into()])?`. The same selection works inside `scan_and_transform`'s
+`scan` configuration; transformation selection remains independent.
+
+Omitting `entities` preserves existing detection. An explicit selection must be
+a nonempty array of unique, exact canonical text entity names. Unknown names,
+lowercase names, duplicates, and structured-only `PERSON` are rejected using
+configuration errors with `/entities` or `/entities/<index>` paths. Consult
+`capabilities().entities` for types with the `text` scope.
+
+Selection narrows enabled detectors: selecting `UUID` still requires
+`detect_uuid=true`, and German types still require a German locale. Text format
+handling, finding order, and byte/codepoint ranges are unchanged. Structured
+scans do not accept this option. It avoids unselected detector work and findings,
+but does not cap the selected findings or bound the size of a scan response.
+
 ## UUID identifiers
 
 Canonical UUID detection is opt-in: pass `{"detect_uuid":true}` to text or

{datafog_core-0.4.2/crates/core → datafog_core-0.4.4}/README.md

@@ -15,6 +15,13 @@ existing detectors and returns field paths plus string-local findings. No model
 or dictionary download is needed. See [person-field discovery](docs/guides/person-discovery.mdx)
 for `scan_structured` / `scanStructured` and structured transformation APIs.
 
+Phone candidates without `+` are interpreted as North American numbers: ten
+digits, optionally preceded by `1`, with area and exchange codes starting with
+`2` through `9`. Candidates starting with `+` retain the existing permissive
+7–15 digit check, including shortened synthetic examples such as `+1-555-1002`.
+These checks do not validate country calling codes or assigned phone numbers,
+and numeric identifiers that fit the accepted patterns can still match `PHONE`.
+
 Both ranges use zero-based, end-exclusive offsets. The byte range addresses the
 UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based
 detectors currently report no confidence score. Node.js and browser WASM also
@@ -56,13 +63,26 @@ full-match regex values:
 }
 ```
 
-`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean, format?: "text" | "env" | "sql" }, transform: ... }` so
+`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean, format?: "text" | "env" | "sql", entities?: string[] }, transform: ... }` so
 detection settings remain separate from transformation policy.
 For `.env` assignments or standard SQL quoted strings, select `format: "env"`
 or `format: "sql"` to preserve email boundaries. See the
 [email source-context policy](docs/guides/email-boundaries.mdx) for supported
 syntax, raw escaped spans, and limitations. Default text scanning is unchanged.
 
+## Credit-card detection and decimal numbers
+
+To reduce decimal false positives, `CREDIT_CARD` detection suppresses contiguous
+digit candidates immediately after a digit followed by `.` or `,`, such as
+`3.4111111111111111` and `3,4111111111111111`. Cards containing spaces or hyphens,
+such as `3.4111-1111-1111-1111`, still undergo the normal card checks.
+
+In plain text, `3,4111111111111111` could also be two CSV fields: `3` and a card
+number. The detector cannot distinguish that structure from a comma-decimal
+number, so it suppresses the card candidate. For CSV input, parse the CSV and
+scan each field separately so a card value is checked without the preceding
+field and delimiter.
+
 ## Packages
 
 | Runtime | Distribution | Import | Status |
@@ -305,6 +325,26 @@ and [migration differences](docs/guides/migrating-from-datafog-python.mdx).
 The higher-level Python adapter and its dependency update were merged in
 [PR #179](https://github.com/DataFog/datafog-python/pull/179). Its package release is separate from Core.
 
+## Text detector selection
+
+Pass `{"entities":["EMAIL","SSN"]}` to a text scan to run only those
+selected detectors, rather than detecting every type and filtering findings
+later. In Rust, use `ScanConfig::new().with_entities(vec!["EMAIL".into(),
+"SSN".into()])?`. The same selection works inside `scan_and_transform`'s
+`scan` configuration; transformation selection remains independent.
+
+Omitting `entities` preserves existing detection. An explicit selection must be
+a nonempty array of unique, exact canonical text entity names. Unknown names,
+lowercase names, duplicates, and structured-only `PERSON` are rejected using
+configuration errors with `/entities` or `/entities/<index>` paths. Consult
+`capabilities().entities` for types with the `text` scope.
+
+Selection narrows enabled detectors: selecting `UUID` still requires
+`detect_uuid=true`, and German types still require a German locale. Text format
+handling, finding order, and byte/codepoint ranges are unchanged. Structured
+scans do not accept this option. It avoids unselected detector work and findings,
+but does not cap the selected findings or bound the size of a scan response.
+
 ## UUID identifiers
 
 Canonical UUID detection is opt-in: pass `{"detect_uuid":true}` to text or

{datafog_core-0.4.2 → datafog_core-0.4.4}/bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core-python"
-version = "0.4.2"
+version = "0.4.4"
 edition = "2024"
 rust-version = "1.88"
 description = "Python bindings for datafog-core"

{datafog_core-0.4.2 → datafog_core-0.4.4}/bindings/python/tests/test_installed.py

@@ -126,7 +126,44 @@ def verify_structured() -> None:
             raise AssertionError("invalid structured input accepted")
 
 
+def verify_text_selection() -> None:
+    text = "👋 jane@example.com SSN 123-45-6789"
+    complete = scan(text)
+    selected = scan(text, {"entities": ["EMAIL"]})
+    expected = [finding for finding in complete if finding.entity_type == "EMAIL"]
+    assert len(selected) == len(expected) == 1
+    assert selected[0].codepoint_range.start == expected[0].codepoint_range.start
+    assert selected[0].byte_range.start == expected[0].byte_range.start
+    assert selected[0].matched_text == "jane@example.com"
+    transformed = scan_and_transform(text, {
+        "scan": {"entities": ["EMAIL"]},
+        "transform": {"default": {"strategy": "redact"}},
+    })
+    assert "123-45-6789" in transformed.text
+    assert "jane@example.com" not in transformed.text
+    for entities in ([], ["EMAIL", "EMAIL"], ["PERSON"], ["email"], [1], "EMAIL"):
+        try:
+            scan(text, {"entities": entities})
+        except DataFogConfigurationError as error:
+            assert error.code == "invalid_configuration"
+            assert error.path.startswith("/entities")
+        else:
+            raise AssertionError(f"invalid selection accepted: {entities!r}")
+    uuid = "550e8400-e29b-41d4-a716-446655440000"
+    assert not scan(uuid, {"entities": ["UUID"]})
+    assert [item.entity_type for item in scan(uuid, {"entities": ["UUID"], "detect_uuid": True})] == ["UUID"]
+    assert not scan(uuid, {"entities": ["EMAIL"], "detect_uuid": True})
+    assert scan("contact=jane@example.com", {"format": "env", "entities": ["EMAIL"]})[0].matched_text == "jane@example.com"
+    try:
+        scan_structured({"email": "jane@example.com"}, {"entities": ["EMAIL"]})
+    except DataFogConfigurationError as error:
+        assert error.reason == "unknown_field"
+    else:
+        raise AssertionError("text selection was accepted in structured scan")
+
+
 def main() -> None:
+    verify_text_selection()
     capabilities_conformance.verify()
     german_conformance.verify()
     credential_overlap_conformance.verify()

{datafog_core-0.4.2 → datafog_core-0.4.4}/bindings/python/tests/test_typing.py

@@ -42,7 +42,7 @@ config: _TransformationConfig = {
     },
     "allow": {"exact": {"EMAIL": ["safe@example.com"]}, "regex": {"EMAIL": [{"pattern": "safe", "case_sensitive": False}]}},
 }
-findings = scan("jane@example.com", {"locale": "en-US", "detect_uuid": True, "format": "env"})
+findings = scan("jane@example.com", {"locale": "en-US", "detect_uuid": True, "format": "env", "entities": ["EMAIL"]})
 assert_type(findings, list[Finding])
 finding = findings[0]
 assert_type(finding.entity_type, str)
@@ -131,6 +131,8 @@ scan("text", {"format": "yaml"})  # error
 scan("text", {"locale": 123})  # error
 scan("text", {"detect_uuid": "true"})  # error
 scan("text", {"max_bytes": 100})  # error
+scan("text", {"entities": "EMAIL"})  # error
+scan_structured({"email": "jane@example.com"}, {"entities": ["EMAIL"]})  # error
 scan("text")[0].entity_type = "OTHER"  # error
 scan("text")[0].byte_range.start = 1  # error
 transform("text", ["not a finding"], {"default": {"strategy": "redact"}})  # error

{datafog_core-0.4.2 → datafog_core-0.4.4}/crates/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core"
-version = "0.4.2"
+version = "0.4.4"
 edition = "2024"
 rust-version = "1.88"
 description = "Fast structured PII detection library"

{datafog_core-0.4.2 → datafog_core-0.4.4/crates/core}/README.md

@@ -15,6 +15,13 @@ existing detectors and returns field paths plus string-local findings. No model
 or dictionary download is needed. See [person-field discovery](docs/guides/person-discovery.mdx)
 for `scan_structured` / `scanStructured` and structured transformation APIs.
 
+Phone candidates without `+` are interpreted as North American numbers: ten
+digits, optionally preceded by `1`, with area and exchange codes starting with
+`2` through `9`. Candidates starting with `+` retain the existing permissive
+7–15 digit check, including shortened synthetic examples such as `+1-555-1002`.
+These checks do not validate country calling codes or assigned phone numbers,
+and numeric identifiers that fit the accepted patterns can still match `PHONE`.
+
 Both ranges use zero-based, end-exclusive offsets. The byte range addresses the
 UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based
 detectors currently report no confidence score. Node.js and browser WASM also
@@ -56,13 +63,26 @@ full-match regex values:
 }
 ```
 
-`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean, format?: "text" | "env" | "sql" }, transform: ... }` so
+`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean, format?: "text" | "env" | "sql", entities?: string[] }, transform: ... }` so
 detection settings remain separate from transformation policy.
 For `.env` assignments or standard SQL quoted strings, select `format: "env"`
 or `format: "sql"` to preserve email boundaries. See the
 [email source-context policy](docs/guides/email-boundaries.mdx) for supported
 syntax, raw escaped spans, and limitations. Default text scanning is unchanged.
 
+## Credit-card detection and decimal numbers
+
+To reduce decimal false positives, `CREDIT_CARD` detection suppresses contiguous
+digit candidates immediately after a digit followed by `.` or `,`, such as
+`3.4111111111111111` and `3,4111111111111111`. Cards containing spaces or hyphens,
+such as `3.4111-1111-1111-1111`, still undergo the normal card checks.
+
+In plain text, `3,4111111111111111` could also be two CSV fields: `3` and a card
+number. The detector cannot distinguish that structure from a comma-decimal
+number, so it suppresses the card candidate. For CSV input, parse the CSV and
+scan each field separately so a card value is checked without the preceding
+field and delimiter.
+
 ## Packages
 
 | Runtime | Distribution | Import | Status |
@@ -305,6 +325,26 @@ and [migration differences](docs/guides/migrating-from-datafog-python.mdx).
 The higher-level Python adapter and its dependency update were merged in
 [PR #179](https://github.com/DataFog/datafog-python/pull/179). Its package release is separate from Core.
 
+## Text detector selection
+
+Pass `{"entities":["EMAIL","SSN"]}` to a text scan to run only those
+selected detectors, rather than detecting every type and filtering findings
+later. In Rust, use `ScanConfig::new().with_entities(vec!["EMAIL".into(),
+"SSN".into()])?`. The same selection works inside `scan_and_transform`'s
+`scan` configuration; transformation selection remains independent.
+
+Omitting `entities` preserves existing detection. An explicit selection must be
+a nonempty array of unique, exact canonical text entity names. Unknown names,
+lowercase names, duplicates, and structured-only `PERSON` are rejected using
+configuration errors with `/entities` or `/entities/<index>` paths. Consult
+`capabilities().entities` for types with the `text` scope.
+
+Selection narrows enabled detectors: selecting `UUID` still requires
+`detect_uuid=true`, and German types still require a German locale. Text format
+handling, finding order, and byte/codepoint ranges are unchanged. Structured
+scans do not accept this option. It avoids unselected detector work and findings,
+but does not cap the selected findings or bound the size of a scan response.
+
 ## UUID identifiers
 
 Canonical UUID detection is opt-in: pass `{"detect_uuid":true}` to text or

{datafog_core-0.4.2 → datafog_core-0.4.4}/crates/core/src/capabilities.rs

@@ -88,6 +88,14 @@ struct Detector {
 }
 
 impl Detector {
+    fn enabled(&self, config: &ScanConfig) -> bool {
+        self.activation.enabled(config)
+            && self
+                .labels
+                .iter()
+                .any(|label| config.includes_entity(label.as_str()))
+    }
+
     fn single(label: Label, detect: fn(&str, &mut Vec<Candidate>), activation: Activation) -> Self {
         Self {
             labels: vec![label],
@@ -147,13 +155,18 @@ pub(super) fn valid_locale(locale: &str) -> bool {
     locale_activation(locale).is_some()
 }
 
-pub(super) fn detect(text: &str, config: &ScanConfig, candidates: &mut Vec<Candidate>) {
-    for detector in REGISTRY
+pub(super) fn text_entity_supported(entity: &str) -> bool {
+    REGISTRY
         .iter()
-        .filter(|detector| detector.activation.enabled(config))
-    {
+        .any(|detector| detector.labels.iter().any(|label| label.as_str() == entity))
+}
+
+pub(super) fn detect(text: &str, config: &ScanConfig, candidates: &mut Vec<Candidate>) {
+    for detector in REGISTRY.iter().filter(|detector| detector.enabled(config)) {
         if detector.labels == [Label::Email] {
             crate::email_context::detect(text, config.format(), candidates);
+        } else if detector.activation == Activation::German {
+            german::detect_selected(text, config, candidates);
         } else {
             (detector.detect)(text, candidates);
         }
@@ -218,3 +231,38 @@ pub fn capabilities() -> Capabilities {
         entities,
     }
 }
+
+#[cfg(test)]
+mod selection_dispatch_tests {
+    use super::*;
+
+    #[test]
+    fn dispatch_skips_every_registry_detector_without_a_selected_label() {
+        let activated = ScanConfig::new()
+            .with_locale("de")
+            .unwrap()
+            .with_uuid_detection(true);
+        for entity in capabilities()
+            .supported_entities
+            .iter()
+            .filter(|name| name.as_str() != "PERSON")
+        {
+            let selected = activated
+                .clone()
+                .with_entities(vec![entity.clone()])
+                .unwrap();
+            let enabled: Vec<_> = REGISTRY
+                .iter()
+                .filter(|detector| detector.enabled(&selected))
+                .collect();
+            assert_eq!(enabled.len(), 1, "{entity}");
+            assert!(
+                enabled[0]
+                    .labels
+                    .iter()
+                    .any(|label| label.as_str() == entity)
+            );
+        }
+        assert!(REGISTRY.iter().all(|detector| detector.enabled(&activated)));
+    }
+}

{datafog_core-0.4.2 → datafog_core-0.4.4}/crates/core/src/german.rs

@@ -1,5 +1,5 @@
 //! Locale-gated lexical detection, without official identifier validation.
-use crate::{Candidate, Label};
+use crate::{Candidate, Label, ScanConfig};
 use regex::Regex;
 use std::sync::LazyLock;
 
@@ -71,7 +71,14 @@ fn ascii_boundary(text: &str, start: usize, end: usize) -> bool {
 }
 
 pub(super) fn detect(text: &str, candidates: &mut Vec<Candidate>) {
-    for (label, regex) in DETECTORS.iter() {
+    detect_selected(text, &ScanConfig::default(), candidates);
+}
+
+pub(super) fn detect_selected(text: &str, config: &ScanConfig, candidates: &mut Vec<Candidate>) {
+    for (label, regex) in DETECTORS
+        .iter()
+        .filter(|(label, _)| config.includes_entity(label.as_str()))
+    {
         for captures in regex.captures_iter(text) {
             // Both captures are unconditional in the statically compiled expressions.
             let whole = captures.get(0).expect("whole regex match");

{datafog_core-0.4.2 → datafog_core-0.4.4}/crates/core/src/lib.rs

@@ -555,9 +555,54 @@ pub struct ScanConfig {
     locale: Option<String>,
     detect_uuid: bool,
     format: ScanFormat,
+    entities: Option<BTreeSet<String>>,
 }
 
 impl ScanConfig {
+    /// Restrict text scanning to exact canonical entity types. Selection only
+    /// narrows enabled detectors; locale and UUID activation remain independent.
+    pub fn with_entities(mut self, entities: Vec<String>) -> Result<Self, PrivacyError> {
+        if entities.is_empty() {
+            return Err(PrivacyError::invalid_configuration(
+                PrivacyErrorReason::EmptyValue,
+                "/entities",
+                "entities must contain at least one entity type",
+            ));
+        }
+        let mut selected = BTreeSet::new();
+        for (index, entity) in entities.into_iter().enumerate() {
+            let path = format!("/entities/{index}");
+            validate_entity_name(&entity, &path)?;
+            if !capabilities::text_entity_supported(&entity) {
+                return Err(PrivacyError::invalid_configuration(
+                    PrivacyErrorReason::InvalidValue,
+                    &path,
+                    "entity type is not supported by text scanning",
+                ));
+            }
+            if !selected.insert(entity) {
+                return Err(PrivacyError::invalid_configuration(
+                    PrivacyErrorReason::DuplicateValue,
+                    path,
+                    "entity selection contains a duplicate entity type",
+                ));
+            }
+        }
+        self.entities = Some(selected);
+        Ok(self)
+    }
+
+    /// Explicit text entity selection, or None for the existing detector defaults.
+    pub fn entities(&self) -> Option<&BTreeSet<String>> {
+        self.entities.as_ref()
+    }
+
+    pub(crate) fn includes_entity(&self, entity: &str) -> bool {
+        self.entities
+            .as_ref()
+            .is_none_or(|entities| entities.contains(entity))
+    }
+
     /// Select source syntax for email boundaries; defaults to unstructured text.
     pub fn with_format(mut self, format: ScanFormat) -> Self {
         self.format = format;
@@ -780,8 +825,20 @@ pub fn parse_scan_and_transform_config(
 /// Parse canonical scanner configuration.
 pub fn parse_scan_config(value: &serde_json::Value) -> Result<ScanConfig, PrivacyError> {
     let object = require_object(value, "", "scan configuration must be an object")?;
-    reject_unknown_fields(object, &["locale", "detect_uuid", "format"], "")?;
+    reject_unknown_fields(object, &["locale", "detect_uuid", "format", "entities"], "")?;
     let mut config = ScanConfig::new();
+    if let Some(entities) = object.get("entities") {
+        let entities = require_array(entities, "/entities", "entities must be an array")?;
+        let mut parsed = Vec::with_capacity(entities.len());
+        for (index, entity) in entities.iter().enumerate() {
+            parsed.push(require_string(
+                entity,
+                &format!("/entities/{index}"),
+                "entity type must be a string",
+            )?);
+        }
+        config = config.with_entities(parsed)?;
+    }
     if let Some(format) = object.get("format") {
         config = config.with_format(parse_scan_format(format)?);
     }
@@ -3100,10 +3157,30 @@ fn is_valid_phone(candidate: &str) -> bool {
         .collect();
 
     if candidate.starts_with('+') {
-        (7..=15).contains(&digits.len())
-    } else {
-        digits.len() == 10 || (digits.len() == 11 && digits.starts_with('1'))
+        return (7..=15).contains(&digits.len());
     }
+
+    let national_number = match digits.len() {
+        10 => digits.as_str(),
+        11 if digits.starts_with('1') => &digits[1..],
+        _ => return false,
+    };
+
+    follows_nanp_rule(national_number.as_bytes())
+}
+
+/// Checks that the 1st and 4th digits of a 10-digit North American number are
+/// between 2 and 9. These are the first digits of the area code and the
+/// exchange code, and the North American Numbering Plan does not allow either
+/// to start with 0 or 1.
+///
+/// This is a special case to stop Unix timestamps, such as `1800005580`, being
+/// detected as phone numbers. It only works while timestamps start with 1,
+/// which is true until 2033. It is not full phone number validation. If better
+/// vetting is needed, Google's libphonenumber is a good alternative.
+fn follows_nanp_rule(national_number: &[u8]) -> bool {
+    let is_valid_code_start = |digit: u8| (b'2'..=b'9').contains(&digit);
+    is_valid_code_start(national_number[0]) && is_valid_code_start(national_number[3])
 }
 
 fn detect_phone(text: &str, candidates: &mut Vec<Candidate>) {
@@ -3291,6 +3368,12 @@ fn card_parts_at(bytes: &[u8], start: usize) -> (usize, Vec<u8>) {
     (last_digit_end, digits)
 }
 
+/// A contiguous digit run after `3.` or `3,` may be a decimal fraction.
+/// Comma-separated fields are ambiguous without structured input context.
+fn follows_decimal_separator(bytes: &[u8], start: usize) -> bool {
+    start >= 2 && matches!(bytes[start - 1], b'.' | b',') && bytes[start - 2].is_ascii_digit()
+}
+
 fn detect_credit_card(text: &str, candidates: &mut Vec<Candidate>) {
     let bytes = text.as_bytes();
     let mut start = 0;
@@ -3304,7 +3387,10 @@ fn detect_credit_card(text: &str, candidates: &mut Vec<Candidate>) {
 
         let (end, digits) = card_parts_at(bytes, start);
 
-        if end == start || (end < bytes.len() && bytes[end].is_ascii_alphanumeric()) {
+        if end == start
+            || (end < bytes.len() && bytes[end].is_ascii_alphanumeric())
+            || (follows_decimal_separator(bytes, start) && end - start == digits.len())
+        {
             start += 1;
             continue;
         }
@@ -3661,6 +3747,9 @@ fn detect_ip_address(text: &str, candidates: &mut Vec<Candidate>) {
 #[cfg(test)]
 mod selection_tests;
 
+#[cfg(test)]
+mod scan_selection_tests;
+
 #[cfg(test)]
 mod tests {
     mod synthetic_fixtures {
@@ -4305,6 +4394,122 @@ mod tests {
         assert!(scan("order212-555-0100x").is_empty());
     }
 
+    #[test]
+    fn detects_phone_numbers_following_nanp_rule() {
+        for phone in [
+            "2125550100",
+            "212-555-0100",
+            "212.555.0100",
+            "1-212-555-0100",
+            "12125550100",
+            "9995550100",
+            "2122220100",
+        ] {
+            let text = format!("Call {phone}.");
+            assert_eq!(
+                scan(&text),
+                vec![expected_ascii_finding(
+                    "PHONE",
+                    phone,
+                    5,
+                    5 + phone.len(),
+                    "datafog-core/phone",
+                )],
+                "{text}"
+            );
+        }
+    }
+
+    #[test]
+    fn rejects_phone_candidates_breaking_nanp_rule() {
+        for text in [
+            "Call 0125550100.",       // area code starts with 0
+            "Call (112) 555-0100.",   // area code starts with 1
+            "Call 212-055-0100.",     // exchange code starts with 0
+            "Call 1-212-155-0100.",   // exchange code starts with 1
+            "\"start\": 1800005580,", // Unix timestamp
+            "\"end\": 1800021600",    // Unix timestamp
+        ] {
+            assert!(scan(text).is_empty(), "{text}");
+        }
+    }
+
+    #[test]
+    fn preserves_permissive_plus_phone_detection() {
+        for phone in [
+            "+1234567",
+            "+1-555-1002",
+            "+1-112-555-0100",
+            "+1-212-155-0100",
+            "+44 20 7946 0958",
+            "+999222333444555",
+        ] {
+            let text = format!("Call {phone}.");
+            assert_eq!(
+                scan(&text),
+                vec![expected_ascii_finding(
+                    "PHONE",
+                    phone,
+                    5,
+                    5 + phone.len(),
+                    "datafog-core/phone",
+                )],
+                "{text}"
+            );
+        }
+    }
+
+    #[test]
+    fn rejects_plus_phone_candidates_outside_length_or_boundary_rules() {
+        for text in [
+            "Call +123456.",
+            "Call +9992223334445556.",
+            "order+44 20 7946 0958",
+            "Call +44 20 7946 0958x",
+        ] {
+            assert!(scan(text).is_empty(), "{text}");
+        }
+    }
+
+    #[test]
+    fn reports_phone_ranges_in_unicode_text() {
+        for phone in ["212-555-0100", "1-212-555-0100", "+1-555-1002"] {
+            let text = format!("☎👋 {phone}。");
+            let findings = scan(&text);
+            assert_eq!(
+                findings,
+                vec![expected_finding(
+                    "PHONE",
+                    phone,
+                    (8, 8 + phone.len()),
+                    (3, 3 + phone.len()),
+                    "datafog-core/phone",
+                )],
+                "{text}"
+            );
+            let finding = &findings[0];
+            assert_eq!(
+                &text[finding.byte_range.start..finding.byte_range.end],
+                phone
+            );
+            assert_eq!(
+                text.chars()
+                    .skip(finding.codepoint_range.start)
+                    .take(finding.codepoint_range.end - finding.codepoint_range.start)
+                    .collect::<String>(),
+                phone
+            );
+            assert_eq!(
+                utf16_range(&text, finding.byte_range).unwrap(),
+                TextRange {
+                    start: 4,
+                    end: 4 + phone.len(),
+                }
+            );
+        }
+        assert!(scan("☎👋 1800005580。").is_empty());
+    }
+
     #[test]
     fn detects_dashed_ssn() {
         assert_eq!(
@@ -4355,11 +4560,70 @@ mod tests {
         assert_eq!(scan("6011111111111117")[0].entity_type, "CREDIT_CARD");
     }
 
+    #[test]
+    fn detects_credit_card_after_separator_without_leading_digit() {
+        for prefix in [".", ",", "no.", "no,"] {
+            for digits in ["4222222222222", "4111111111111111", "4111111111111111110"] {
+                let text = format!("{prefix}{digits}");
+                assert_eq!(
+                    scan(&text),
+                    vec![expected_ascii_finding(
+                        "CREDIT_CARD",
+                        digits,
+                        prefix.len(),
+                        prefix.len() + digits.len(),
+                        "datafog-core/credit-card",
+                    )],
+                    "{text}"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn detects_formatted_credit_card_after_digit_and_separator() {
+        for prefix in ["3.", "3,"] {
+            for card in [
+                "4222-2222-22222",
+                "4111-1111-1111-1111",
+                "4111 1111 1111 1111",
+                "4111-1111-1111-1111110",
+            ] {
+                let text = format!("{prefix}{card}");
+                assert_eq!(
+                    scan(&text)
+                        .into_iter()
+                        .filter(|finding| finding.entity_type == "CREDIT_CARD")
+                        .collect::<Vec<_>>(),
+                    vec![expected_ascii_finding(
+                        "CREDIT_CARD",
+                        card,
+                        prefix.len(),
+                        prefix.len() + card.len(),
+                        "datafog-core/credit-card",
+                    )],
+                    "{text}"
+                );
+            }
+        }
+    }
+
     #[test]
     fn rejects_invalid_credit_card_candidates() {
         assert!(scan("4111-1111-1111-1112").is_empty()); // bad Luhn checksum
         assert!(scan("1234-5678-9012-3452").is_empty()); // unsupported prefix
         assert!(scan("x4111-1111-1111-1111y").is_empty()); // embedded
+        for fraction in [
+            "4222222222222",
+            "4111111111111111",
+            "4111111111111111110",
+            "41111111111111111100",
+        ] {
+            for separator in [".", ","] {
+                let text = format!("3{separator}{fraction}");
+                assert!(scan(&text).is_empty(), "{text}"); // decimal fraction
+            }
+        }
     }
 
     #[test]

datafog_core-0.4.4/crates/core/src/scan_selection_tests.rs

@@ -0,0 +1,155 @@
+use super::*;
+use serde_json::json;
+
+#[test]
+fn selected_scans_match_filtered_defaults_for_every_text_detector() {
+    let fixtures = [
+        include_str!("../../../fixtures/development.jsonl"),
+        include_str!("../../../fixtures/german.jsonl"),
+        include_str!("../../../fixtures/uuid.jsonl"),
+        include_str!("../../../fixtures/credential-overlaps.jsonl"),
+        include_str!("../../../fixtures/credential-uri.jsonl"),
+        include_str!("../../../fixtures/api-key.jsonl"),
+        include_str!("../../../fixtures/bearer-token.jsonl"),
+        include_str!("../../../fixtures/jwt.jsonl"),
+        include_str!("../../../fixtures/npi.jsonl"),
+        include_str!("../../../fixtures/us-routing-number.jsonl"),
+        include_str!("../../../fixtures/private-key.jsonl"),
+        include_str!("../../../fixtures/email-context.jsonl"),
+    ];
+    let mut covered = BTreeSet::new();
+    for fixture in fixtures {
+        for line in fixture.lines() {
+            let row: serde_json::Value = serde_json::from_str(line).unwrap();
+            let text = row["text"].as_str().unwrap();
+            let config = parse_scan_config(row.get("config").unwrap_or(&json!({}))).unwrap();
+            let baseline = scan_with_config(text, &config);
+            for name in capabilities()
+                .supported_entities
+                .into_iter()
+                .filter(|name| name != "PERSON")
+            {
+                let selected = config.clone().with_entities(vec![name.clone()]).unwrap();
+                let expected: Vec<_> = baseline
+                    .iter()
+                    .filter(|finding| finding.entity_type == name)
+                    .cloned()
+                    .collect();
+                if !expected.is_empty() {
+                    covered.insert(name.clone());
+                }
+                assert_eq!(
+                    scan_with_config(text, &selected),
+                    expected,
+                    "{}: {name}",
+                    row["id"]
+                );
+            }
+        }
+    }
+    let expected: BTreeSet<_> = capabilities()
+        .supported_entities
+        .into_iter()
+        .filter(|name| name != "PERSON")
+        .collect();
+    assert_eq!(covered, expected);
+}
+
+#[test]
+fn invalid_text_selections_have_precise_safe_errors() {
+    for (value, path, reason) in [
+        (
+            json!({"entities": []}),
+            "/entities",
+            PrivacyErrorReason::EmptyValue,
+        ),
+        (
+            json!({"entities": null}),
+            "/entities",
+            PrivacyErrorReason::InvalidType,
+        ),
+        (
+            json!({"entities": "EMAIL"}),
+            "/entities",
+            PrivacyErrorReason::InvalidType,
+        ),
+        (
+            json!({"entities": [2]}),
+            "/entities/0",
+            PrivacyErrorReason::InvalidType,
+        ),
+        (
+            json!({"entities": ["EMAIL", "EMAIL"]}),
+            "/entities/1",
+            PrivacyErrorReason::DuplicateValue,
+        ),
+        (
+            json!({"entities": ["PERSON"]}),
+            "/entities/0",
+            PrivacyErrorReason::InvalidValue,
+        ),
+        (
+            json!({"entities": ["NOT_SUPPORTED"]}),
+            "/entities/0",
+            PrivacyErrorReason::InvalidValue,
+        ),
+        (
+            json!({"entities": ["email"]}),
+            "/entities/0",
+            PrivacyErrorReason::InvalidValue,
+        ),
+    ] {
+        let error = parse_scan_config(&value).unwrap_err();
+        assert_eq!(error.code(), PrivacyErrorCode::InvalidConfiguration);
+        assert_eq!(error.path(), Some(path));
+        assert_eq!(error.reason(), Some(reason));
+    }
+    let error = parse_scan_and_transform_config(
+        &json!({"scan": {"entities": []}, "transform": {"default": {"strategy": "redact"}}}),
+    )
+    .unwrap_err();
+    assert_eq!(error.path(), Some("/scan/entities"));
+    assert!(structured::parse_scan_config(&json!({"entities": ["EMAIL"]})).is_err());
+}
+
+#[test]
+fn selection_does_not_activate_uuid_or_locale_detectors() {
+    for (entity, text, active) in [
+        (
+            "UUID",
+            "550e8400-e29b-41d4-a716-446655440000",
+            json!({"detect_uuid": true}),
+        ),
+        (
+            "DE_IBAN",
+            "DE44 5001 0517 5407 3249 31",
+            json!({"locale": "de"}),
+        ),
+    ] {
+        let selected = ScanConfig::new()
+            .with_entities(vec![entity.into()])
+            .unwrap();
+        assert!(scan_with_config(text, &selected).is_empty());
+        let active = parse_scan_config(&active)
+            .unwrap()
+            .with_entities(vec![entity.into()])
+            .unwrap();
+        assert_eq!(scan_with_config(text, &active).len(), 1);
+        let excluded = active.with_entities(vec!["EMAIL".into()]).unwrap();
+        assert!(scan_with_config(text, &excluded).is_empty());
+    }
+}
+
+#[test]
+fn scan_and_transform_selection_preserves_unselected_content_and_unicode_offsets() {
+    let text = "👋 person@example.com / 123-45-6789";
+    let config = parse_scan_and_transform_config(
+        &json!({"scan": {"entities": ["EMAIL"]}, "transform": {"default": {"strategy": "redact"}}}),
+    )
+    .unwrap();
+    let result = scan_and_transform(text, &config).unwrap();
+    assert_eq!(result.text, "👋 [EMAIL] / 123-45-6789");
+    let finding = &scan_with_config(text, config.scan_config())[0];
+    assert_eq!(finding.codepoint_range.start, 2);
+    assert_eq!(finding.byte_range.start, 5);
+}

{datafog_core-0.4.2 → datafog_core-0.4.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "datafog-core"
-version = "0.4.2"
+version = "0.4.4"
 description = "Fast structured PII detection implemented in Rust"
 readme = "README.md"
 license = "MIT"

{datafog_core-0.4.2 → datafog_core-0.4.4}/python/datafog_core/__init__.pyi

@@ -8,11 +8,15 @@ _JsonValue: TypeAlias = (
 )
 _JsonDocument: TypeAlias = list[_JsonValue] | dict[str, _JsonValue]
 
-class _ScanConfig(TypedDict, total=False):
+class _CommonScanConfig(TypedDict, total=False):
     format: Literal["text", "env", "sql"]
     locale: str
     detect_uuid: bool
 
+class _ScanConfig(_CommonScanConfig, total=False):
+    # Select activated text detectors; omission preserves the default scan.
+    entities: list[str] | tuple[str, ...]
+
 class _CapabilityActivationRequired(TypedDict):
     kind: Literal["default", "locale", "config", "structured"]
 
@@ -33,7 +37,7 @@ class _Capabilities(TypedDict):
     locales: dict[str, _LocaleCapability]
     entities: dict[str, _EntityCapability]
 
-class _StructuredScanConfig(_ScanConfig, total=False):
+class _StructuredScanConfig(_CommonScanConfig, total=False):
     discover_person: bool
     mappings: dict[str, Literal["PERSON"]]
     exclude: list[str] | tuple[str, ...]