datafog-core 0.4.3__tar.gz → 0.4.4__tar.gz

{datafog_core-0.4.3 → datafog_core-0.4.4}/Cargo.lock

@@ -80,7 +80,7 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"
 
 [[package]]
 name = "datafog-core"
-version = "0.4.3"
+version = "0.4.4"
 dependencies = [
  "base64",
  "futures",
@@ -94,7 +94,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-core-python"
-version = "0.4.3"
+version = "0.4.4"
 dependencies = [
  "datafog-core",
  "pyo3",
@@ -104,7 +104,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-node"
-version = "0.4.3"
+version = "0.4.4"
 dependencies = [
  "datafog-core",
  "napi",
@@ -115,7 +115,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-wasm"
-version = "0.4.3"
+version = "0.4.4"
 dependencies = [
  "datafog-core",
  "serde",

{datafog_core-0.4.3 → datafog_core-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datafog-core
-Version: 0.4.3
+Version: 0.4.4
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
@@ -38,6 +38,13 @@ existing detectors and returns field paths plus string-local findings. No model
 or dictionary download is needed. See [person-field discovery](docs/guides/person-discovery.mdx)
 for `scan_structured` / `scanStructured` and structured transformation APIs.
 
+Phone candidates without `+` are interpreted as North American numbers: ten
+digits, optionally preceded by `1`, with area and exchange codes starting with
+`2` through `9`. Candidates starting with `+` retain the existing permissive
+7–15 digit check, including shortened synthetic examples such as `+1-555-1002`.
+These checks do not validate country calling codes or assigned phone numbers,
+and numeric identifiers that fit the accepted patterns can still match `PHONE`.
+
 Both ranges use zero-based, end-exclusive offsets. The byte range addresses the
 UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based
 detectors currently report no confidence score. Node.js and browser WASM also
@@ -86,6 +93,19 @@ or `format: "sql"` to preserve email boundaries. See the
 [email source-context policy](docs/guides/email-boundaries.mdx) for supported
 syntax, raw escaped spans, and limitations. Default text scanning is unchanged.
 
+## Credit-card detection and decimal numbers
+
+To reduce decimal false positives, `CREDIT_CARD` detection suppresses contiguous
+digit candidates immediately after a digit followed by `.` or `,`, such as
+`3.4111111111111111` and `3,4111111111111111`. Cards containing spaces or hyphens,
+such as `3.4111-1111-1111-1111`, still undergo the normal card checks.
+
+In plain text, `3,4111111111111111` could also be two CSV fields: `3` and a card
+number. The detector cannot distinguish that structure from a comma-decimal
+number, so it suppresses the card candidate. For CSV input, parse the CSV and
+scan each field separately so a card value is checked without the preceding
+field and delimiter.
+
 ## Packages
 
 | Runtime | Distribution | Import | Status |

{datafog_core-0.4.3/crates/core → datafog_core-0.4.4}/README.md

@@ -15,6 +15,13 @@ existing detectors and returns field paths plus string-local findings. No model
 or dictionary download is needed. See [person-field discovery](docs/guides/person-discovery.mdx)
 for `scan_structured` / `scanStructured` and structured transformation APIs.
 
+Phone candidates without `+` are interpreted as North American numbers: ten
+digits, optionally preceded by `1`, with area and exchange codes starting with
+`2` through `9`. Candidates starting with `+` retain the existing permissive
+7–15 digit check, including shortened synthetic examples such as `+1-555-1002`.
+These checks do not validate country calling codes or assigned phone numbers,
+and numeric identifiers that fit the accepted patterns can still match `PHONE`.
+
 Both ranges use zero-based, end-exclusive offsets. The byte range addresses the
 UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based
 detectors currently report no confidence score. Node.js and browser WASM also
@@ -63,6 +70,19 @@ or `format: "sql"` to preserve email boundaries. See the
 [email source-context policy](docs/guides/email-boundaries.mdx) for supported
 syntax, raw escaped spans, and limitations. Default text scanning is unchanged.
 
+## Credit-card detection and decimal numbers
+
+To reduce decimal false positives, `CREDIT_CARD` detection suppresses contiguous
+digit candidates immediately after a digit followed by `.` or `,`, such as
+`3.4111111111111111` and `3,4111111111111111`. Cards containing spaces or hyphens,
+such as `3.4111-1111-1111-1111`, still undergo the normal card checks.
+
+In plain text, `3,4111111111111111` could also be two CSV fields: `3` and a card
+number. The detector cannot distinguish that structure from a comma-decimal
+number, so it suppresses the card candidate. For CSV input, parse the CSV and
+scan each field separately so a card value is checked without the preceding
+field and delimiter.
+
 ## Packages
 
 | Runtime | Distribution | Import | Status |

{datafog_core-0.4.3 → datafog_core-0.4.4}/bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core-python"
-version = "0.4.3"
+version = "0.4.4"
 edition = "2024"
 rust-version = "1.88"
 description = "Python bindings for datafog-core"

{datafog_core-0.4.3 → datafog_core-0.4.4}/crates/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core"
-version = "0.4.3"
+version = "0.4.4"
 edition = "2024"
 rust-version = "1.88"
 description = "Fast structured PII detection library"

{datafog_core-0.4.3 → datafog_core-0.4.4/crates/core}/README.md

@@ -15,6 +15,13 @@ existing detectors and returns field paths plus string-local findings. No model
 or dictionary download is needed. See [person-field discovery](docs/guides/person-discovery.mdx)
 for `scan_structured` / `scanStructured` and structured transformation APIs.
 
+Phone candidates without `+` are interpreted as North American numbers: ten
+digits, optionally preceded by `1`, with area and exchange codes starting with
+`2` through `9`. Candidates starting with `+` retain the existing permissive
+7–15 digit check, including shortened synthetic examples such as `+1-555-1002`.
+These checks do not validate country calling codes or assigned phone numbers,
+and numeric identifiers that fit the accepted patterns can still match `PHONE`.
+
 Both ranges use zero-based, end-exclusive offsets. The byte range addresses the
 UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based
 detectors currently report no confidence score. Node.js and browser WASM also
@@ -63,6 +70,19 @@ or `format: "sql"` to preserve email boundaries. See the
 [email source-context policy](docs/guides/email-boundaries.mdx) for supported
 syntax, raw escaped spans, and limitations. Default text scanning is unchanged.
 
+## Credit-card detection and decimal numbers
+
+To reduce decimal false positives, `CREDIT_CARD` detection suppresses contiguous
+digit candidates immediately after a digit followed by `.` or `,`, such as
+`3.4111111111111111` and `3,4111111111111111`. Cards containing spaces or hyphens,
+such as `3.4111-1111-1111-1111`, still undergo the normal card checks.
+
+In plain text, `3,4111111111111111` could also be two CSV fields: `3` and a card
+number. The detector cannot distinguish that structure from a comma-decimal
+number, so it suppresses the card candidate. For CSV input, parse the CSV and
+scan each field separately so a card value is checked without the preceding
+field and delimiter.
+
 ## Packages
 
 | Runtime | Distribution | Import | Status |

{datafog_core-0.4.3 → datafog_core-0.4.4}/crates/core/src/lib.rs

@@ -3157,10 +3157,30 @@ fn is_valid_phone(candidate: &str) -> bool {
         .collect();
 
     if candidate.starts_with('+') {
-        (7..=15).contains(&digits.len())
-    } else {
-        digits.len() == 10 || (digits.len() == 11 && digits.starts_with('1'))
+        return (7..=15).contains(&digits.len());
     }
+
+    let national_number = match digits.len() {
+        10 => digits.as_str(),
+        11 if digits.starts_with('1') => &digits[1..],
+        _ => return false,
+    };
+
+    follows_nanp_rule(national_number.as_bytes())
+}
+
+/// Checks that the 1st and 4th digits of a 10-digit North American number are
+/// between 2 and 9. These are the first digits of the area code and the
+/// exchange code, and the North American Numbering Plan does not allow either
+/// to start with 0 or 1.
+///
+/// This is a special case to stop Unix timestamps, such as `1800005580`, being
+/// detected as phone numbers. It only works while timestamps start with 1,
+/// which is true until 2033. It is not full phone number validation. If better
+/// vetting is needed, Google's libphonenumber is a good alternative.
+fn follows_nanp_rule(national_number: &[u8]) -> bool {
+    let is_valid_code_start = |digit: u8| (b'2'..=b'9').contains(&digit);
+    is_valid_code_start(national_number[0]) && is_valid_code_start(national_number[3])
 }
 
 fn detect_phone(text: &str, candidates: &mut Vec<Candidate>) {
@@ -3348,6 +3368,12 @@ fn card_parts_at(bytes: &[u8], start: usize) -> (usize, Vec<u8>) {
     (last_digit_end, digits)
 }
 
+/// A contiguous digit run after `3.` or `3,` may be a decimal fraction.
+/// Comma-separated fields are ambiguous without structured input context.
+fn follows_decimal_separator(bytes: &[u8], start: usize) -> bool {
+    start >= 2 && matches!(bytes[start - 1], b'.' | b',') && bytes[start - 2].is_ascii_digit()
+}
+
 fn detect_credit_card(text: &str, candidates: &mut Vec<Candidate>) {
     let bytes = text.as_bytes();
     let mut start = 0;
@@ -3361,7 +3387,10 @@ fn detect_credit_card(text: &str, candidates: &mut Vec<Candidate>) {
 
         let (end, digits) = card_parts_at(bytes, start);
 
-        if end == start || (end < bytes.len() && bytes[end].is_ascii_alphanumeric()) {
+        if end == start
+            || (end < bytes.len() && bytes[end].is_ascii_alphanumeric())
+            || (follows_decimal_separator(bytes, start) && end - start == digits.len())
+        {
             start += 1;
             continue;
         }
@@ -4365,6 +4394,122 @@ mod tests {
         assert!(scan("order212-555-0100x").is_empty());
     }
 
+    #[test]
+    fn detects_phone_numbers_following_nanp_rule() {
+        for phone in [
+            "2125550100",
+            "212-555-0100",
+            "212.555.0100",
+            "1-212-555-0100",
+            "12125550100",
+            "9995550100",
+            "2122220100",
+        ] {
+            let text = format!("Call {phone}.");
+            assert_eq!(
+                scan(&text),
+                vec![expected_ascii_finding(
+                    "PHONE",
+                    phone,
+                    5,
+                    5 + phone.len(),
+                    "datafog-core/phone",
+                )],
+                "{text}"
+            );
+        }
+    }
+
+    #[test]
+    fn rejects_phone_candidates_breaking_nanp_rule() {
+        for text in [
+            "Call 0125550100.",       // area code starts with 0
+            "Call (112) 555-0100.",   // area code starts with 1
+            "Call 212-055-0100.",     // exchange code starts with 0
+            "Call 1-212-155-0100.",   // exchange code starts with 1
+            "\"start\": 1800005580,", // Unix timestamp
+            "\"end\": 1800021600",    // Unix timestamp
+        ] {
+            assert!(scan(text).is_empty(), "{text}");
+        }
+    }
+
+    #[test]
+    fn preserves_permissive_plus_phone_detection() {
+        for phone in [
+            "+1234567",
+            "+1-555-1002",
+            "+1-112-555-0100",
+            "+1-212-155-0100",
+            "+44 20 7946 0958",
+            "+999222333444555",
+        ] {
+            let text = format!("Call {phone}.");
+            assert_eq!(
+                scan(&text),
+                vec![expected_ascii_finding(
+                    "PHONE",
+                    phone,
+                    5,
+                    5 + phone.len(),
+                    "datafog-core/phone",
+                )],
+                "{text}"
+            );
+        }
+    }
+
+    #[test]
+    fn rejects_plus_phone_candidates_outside_length_or_boundary_rules() {
+        for text in [
+            "Call +123456.",
+            "Call +9992223334445556.",
+            "order+44 20 7946 0958",
+            "Call +44 20 7946 0958x",
+        ] {
+            assert!(scan(text).is_empty(), "{text}");
+        }
+    }
+
+    #[test]
+    fn reports_phone_ranges_in_unicode_text() {
+        for phone in ["212-555-0100", "1-212-555-0100", "+1-555-1002"] {
+            let text = format!("☎👋 {phone}。");
+            let findings = scan(&text);
+            assert_eq!(
+                findings,
+                vec![expected_finding(
+                    "PHONE",
+                    phone,
+                    (8, 8 + phone.len()),
+                    (3, 3 + phone.len()),
+                    "datafog-core/phone",
+                )],
+                "{text}"
+            );
+            let finding = &findings[0];
+            assert_eq!(
+                &text[finding.byte_range.start..finding.byte_range.end],
+                phone
+            );
+            assert_eq!(
+                text.chars()
+                    .skip(finding.codepoint_range.start)
+                    .take(finding.codepoint_range.end - finding.codepoint_range.start)
+                    .collect::<String>(),
+                phone
+            );
+            assert_eq!(
+                utf16_range(&text, finding.byte_range).unwrap(),
+                TextRange {
+                    start: 4,
+                    end: 4 + phone.len(),
+                }
+            );
+        }
+        assert!(scan("☎👋 1800005580。").is_empty());
+    }
+
     #[test]
     fn detects_dashed_ssn() {
         assert_eq!(
@@ -4415,11 +4560,70 @@ mod tests {
         assert_eq!(scan("6011111111111117")[0].entity_type, "CREDIT_CARD");
     }
 
+    #[test]
+    fn detects_credit_card_after_separator_without_leading_digit() {
+        for prefix in [".", ",", "no.", "no,"] {
+            for digits in ["4222222222222", "4111111111111111", "4111111111111111110"] {
+                let text = format!("{prefix}{digits}");
+                assert_eq!(
+                    scan(&text),
+                    vec![expected_ascii_finding(
+                        "CREDIT_CARD",
+                        digits,
+                        prefix.len(),
+                        prefix.len() + digits.len(),
+                        "datafog-core/credit-card",
+                    )],
+                    "{text}"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn detects_formatted_credit_card_after_digit_and_separator() {
+        for prefix in ["3.", "3,"] {
+            for card in [
+                "4222-2222-22222",
+                "4111-1111-1111-1111",
+                "4111 1111 1111 1111",
+                "4111-1111-1111-1111110",
+            ] {
+                let text = format!("{prefix}{card}");
+                assert_eq!(
+                    scan(&text)
+                        .into_iter()
+                        .filter(|finding| finding.entity_type == "CREDIT_CARD")
+                        .collect::<Vec<_>>(),
+                    vec![expected_ascii_finding(
+                        "CREDIT_CARD",
+                        card,
+                        prefix.len(),
+                        prefix.len() + card.len(),
+                        "datafog-core/credit-card",
+                    )],
+                    "{text}"
+                );
+            }
+        }
+    }
+
     #[test]
     fn rejects_invalid_credit_card_candidates() {
         assert!(scan("4111-1111-1111-1112").is_empty()); // bad Luhn checksum
         assert!(scan("1234-5678-9012-3452").is_empty()); // unsupported prefix
         assert!(scan("x4111-1111-1111-1111y").is_empty()); // embedded
+        for fraction in [
+            "4222222222222",
+            "4111111111111111",
+            "4111111111111111110",
+            "41111111111111111100",
+        ] {
+            for separator in [".", ","] {
+                let text = format!("3{separator}{fraction}");
+                assert!(scan(&text).is_empty(), "{text}"); // decimal fraction
+            }
+        }
     }
 
     #[test]

{datafog_core-0.4.3 → datafog_core-0.4.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "datafog-core"
-version = "0.4.3"
+version = "0.4.4"
 description = "Fast structured PII detection implemented in Rust"
 readme = "README.md"
 license = "MIT"