datafog-core 0.4.1__tar.gz → 0.4.3__tar.gz

{datafog_core-0.4.1 → datafog_core-0.4.3}/Cargo.lock

@@ -80,7 +80,7 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"
 
 [[package]]
 name = "datafog-core"
-version = "0.4.1"
+version = "0.4.3"
 dependencies = [
  "base64",
  "futures",
@@ -94,7 +94,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-core-python"
-version = "0.4.1"
+version = "0.4.3"
 dependencies = [
  "datafog-core",
  "pyo3",
@@ -104,7 +104,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-node"
-version = "0.4.1"
+version = "0.4.3"
 dependencies = [
  "datafog-core",
  "napi",
@@ -115,7 +115,7 @@ dependencies = [
 
 [[package]]
 name = "datafog-wasm"
-version = "0.4.1"
+version = "0.4.3"
 dependencies = [
  "datafog-core",
  "serde",

{datafog_core-0.4.1 → datafog_core-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datafog-core
-Version: 0.4.1
+Version: 0.4.3
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
@@ -79,8 +79,12 @@ full-match regex values:
 }
 ```
 
-`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean }, transform: ... }` so
+`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean, format?: "text" | "env" | "sql", entities?: string[] }, transform: ... }` so
 detection settings remain separate from transformation policy.
+For `.env` assignments or standard SQL quoted strings, select `format: "env"`
+or `format: "sql"` to preserve email boundaries. See the
+[email source-context policy](docs/guides/email-boundaries.mdx) for supported
+syntax, raw escaped spans, and limitations. Default text scanning is unchanged.
 
 ## Packages
 
@@ -321,8 +325,28 @@ boundaries, never newlines. Returned text and offsets preserve the source.
 Passport and residence-permit patterns are legacy heuristics with limited
 coverage. See the [German entity reference](docs/reference/german-entities.mdx)
 and [migration differences](docs/guides/migrating-from-datafog-python.mdx).
-Core 0.4.0 is published. The higher-level Python adapter and its dependency
-update remain separate and unreleased in [draft PR #179](https://github.com/DataFog/datafog-python/pull/179).
+The higher-level Python adapter and its dependency update were merged in
+[PR #179](https://github.com/DataFog/datafog-python/pull/179). Its package release is separate from Core.
+
+## Text detector selection
+
+Pass `{"entities":["EMAIL","SSN"]}` to a text scan to run only those
+selected detectors, rather than detecting every type and filtering findings
+later. In Rust, use `ScanConfig::new().with_entities(vec!["EMAIL".into(),
+"SSN".into()])?`. The same selection works inside `scan_and_transform`'s
+`scan` configuration; transformation selection remains independent.
+
+Omitting `entities` preserves existing detection. An explicit selection must be
+a nonempty array of unique, exact canonical text entity names. Unknown names,
+lowercase names, duplicates, and structured-only `PERSON` are rejected using
+configuration errors with `/entities` or `/entities/<index>` paths. Consult
+`capabilities().entities` for types with the `text` scope.
+
+Selection narrows enabled detectors: selecting `UUID` still requires
+`detect_uuid=true`, and German types still require a German locale. Text format
+handling, finding order, and byte/codepoint ranges are unchanged. Structured
+scans do not accept this option. It avoids unselected detector work and findings,
+but does not cap the selected findings or bound the size of a scan response.
 
 ## UUID identifiers
 
@@ -332,7 +356,7 @@ structured scans, independently of locale. It emits `UUID` findings for versions
 sensitivity. See the [UUID reference](docs/reference/uuid.mdx) for boundaries,
 excluded sentinel forms and transformation examples.
 
-**0.4.0 is published** for Rust, Python, Node.js, and WASM. The higher-level Python adapter remains a separate unreleased integration in [draft PR #179](https://github.com/DataFog/datafog-python/pull/179). See the [release notes](docs/releases/0-4-0.mdx), [runtime capabilities](docs/reference/capabilities.mdx), and [0.4.x compatibility policy](docs/reference/compatibility.mdx).
+**0.4.1 is published** for Rust, Python, Node.js, and WASM. The higher-level Python adapter is integrated in [merged PR #179](https://github.com/DataFog/datafog-python/pull/179); its package release is separate. See the [release notes](docs/releases/0-4-1.mdx), [runtime capabilities](docs/reference/capabilities.mdx), and [0.4.x compatibility policy](docs/reference/compatibility.mdx).
 
-The unreleased **0.4.1 candidate** adds three default detectors: provider-prefixed [`API_KEY`](docs/reference/api-keys.mdx), context-required [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), and whole PostgreSQL [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx). The registry reports 23 supported entities and 14 defaults with capability contract 1. See the [candidate release notes](docs/releases/0-4-1.mdx) for exact scopes, preserved legacy Python overlap behavior, and the required exact-wheel Python 4.9 integration gate. Python's default backend is unchanged.
+**Core 0.4.1** adds three default detectors: provider-prefixed [`API_KEY`](docs/reference/api-keys.mdx), context-required [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), and whole PostgreSQL [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx). The registry reports 23 supported entities and 14 defaults with capability contract 1. See the [release notes](docs/releases/0-4-1.mdx) for exact scopes, preserved legacy Python overlap behavior, and the passed local exact-wheel Python 4.9 integration gate. Python's default backend is unchanged.
 

{datafog_core-0.4.1/crates/core → datafog_core-0.4.3}/README.md

@@ -56,8 +56,12 @@ full-match regex values:
 }
 ```
 
-`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean }, transform: ... }` so
+`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean, format?: "text" | "env" | "sql", entities?: string[] }, transform: ... }` so
 detection settings remain separate from transformation policy.
+For `.env` assignments or standard SQL quoted strings, select `format: "env"`
+or `format: "sql"` to preserve email boundaries. See the
+[email source-context policy](docs/guides/email-boundaries.mdx) for supported
+syntax, raw escaped spans, and limitations. Default text scanning is unchanged.
 
 ## Packages
 
@@ -298,8 +302,28 @@ boundaries, never newlines. Returned text and offsets preserve the source.
 Passport and residence-permit patterns are legacy heuristics with limited
 coverage. See the [German entity reference](docs/reference/german-entities.mdx)
 and [migration differences](docs/guides/migrating-from-datafog-python.mdx).
-Core 0.4.0 is published. The higher-level Python adapter and its dependency
-update remain separate and unreleased in [draft PR #179](https://github.com/DataFog/datafog-python/pull/179).
+The higher-level Python adapter and its dependency update were merged in
+[PR #179](https://github.com/DataFog/datafog-python/pull/179). Its package release is separate from Core.
+
+## Text detector selection
+
+Pass `{"entities":["EMAIL","SSN"]}` to a text scan to run only those
+selected detectors, rather than detecting every type and filtering findings
+later. In Rust, use `ScanConfig::new().with_entities(vec!["EMAIL".into(),
+"SSN".into()])?`. The same selection works inside `scan_and_transform`'s
+`scan` configuration; transformation selection remains independent.
+
+Omitting `entities` preserves existing detection. An explicit selection must be
+a nonempty array of unique, exact canonical text entity names. Unknown names,
+lowercase names, duplicates, and structured-only `PERSON` are rejected using
+configuration errors with `/entities` or `/entities/<index>` paths. Consult
+`capabilities().entities` for types with the `text` scope.
+
+Selection narrows enabled detectors: selecting `UUID` still requires
+`detect_uuid=true`, and German types still require a German locale. Text format
+handling, finding order, and byte/codepoint ranges are unchanged. Structured
+scans do not accept this option. It avoids unselected detector work and findings,
+but does not cap the selected findings or bound the size of a scan response.
 
 ## UUID identifiers
 
@@ -309,6 +333,6 @@ structured scans, independently of locale. It emits `UUID` findings for versions
 sensitivity. See the [UUID reference](docs/reference/uuid.mdx) for boundaries,
 excluded sentinel forms and transformation examples.
 
-**0.4.0 is published** for Rust, Python, Node.js, and WASM. The higher-level Python adapter remains a separate unreleased integration in [draft PR #179](https://github.com/DataFog/datafog-python/pull/179). See the [release notes](docs/releases/0-4-0.mdx), [runtime capabilities](docs/reference/capabilities.mdx), and [0.4.x compatibility policy](docs/reference/compatibility.mdx).
+**0.4.1 is published** for Rust, Python, Node.js, and WASM. The higher-level Python adapter is integrated in [merged PR #179](https://github.com/DataFog/datafog-python/pull/179); its package release is separate. See the [release notes](docs/releases/0-4-1.mdx), [runtime capabilities](docs/reference/capabilities.mdx), and [0.4.x compatibility policy](docs/reference/compatibility.mdx).
 
-The unreleased **0.4.1 candidate** adds three default detectors: provider-prefixed [`API_KEY`](docs/reference/api-keys.mdx), context-required [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), and whole PostgreSQL [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx). The registry reports 23 supported entities and 14 defaults with capability contract 1. See the [candidate release notes](docs/releases/0-4-1.mdx) for exact scopes, preserved legacy Python overlap behavior, and the required exact-wheel Python 4.9 integration gate. Python's default backend is unchanged.
+**Core 0.4.1** adds three default detectors: provider-prefixed [`API_KEY`](docs/reference/api-keys.mdx), context-required [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), and whole PostgreSQL [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx). The registry reports 23 supported entities and 14 defaults with capability contract 1. See the [release notes](docs/releases/0-4-1.mdx) for exact scopes, preserved legacy Python overlap behavior, and the passed local exact-wheel Python 4.9 integration gate. Python's default backend is unchanged.

{datafog_core-0.4.1 → datafog_core-0.4.3}/bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core-python"
-version = "0.4.1"
+version = "0.4.3"
 edition = "2024"
 rust-version = "1.88"
 description = "Python bindings for datafog-core"

datafog_core-0.4.3/bindings/python/tests/email_context_conformance.py

@@ -0,0 +1,49 @@
+"""Email source-context fixtures against the installed extension."""
+
+import json
+from pathlib import Path
+
+import datafog_core as api
+
+
+def verify():
+    fixture = Path(__file__).resolve().parents[3] / "fixtures/email-context.jsonl"
+    for line in fixture.read_text().splitlines():
+        row = json.loads(line)
+        text = row["text"]
+        findings = api.scan(text, row["config"])
+        emails = [f for f in findings if f.entity_type == "EMAIL"]
+        assert [
+            dict(text=f.matched_text, start=f.codepoint_range.start, end=f.codepoint_range.end)
+            for f in emails
+        ] == row["matches"], row["id"]
+        for finding in emails:
+            assert text.encode()[finding.byte_range.start:finding.byte_range.end].decode() == finding.matched_text
+            assert text[finding.codepoint_range.start:finding.codepoint_range.end] == finding.matched_text
+        data = {"source": text}
+        located = api.scan_structured(data, row["config"]).findings
+        assert [f.finding for f in located] == findings
+        for strategy in ["redact", "mask", "remove"]:
+            policy = {"default": {"strategy": strategy}, "entities": ["EMAIL"]}
+            envelope = {"scan": row["config"], "transform": policy}
+            result = api.scan_and_transform(text, envelope)
+            assert result.text == row["outputs"][strategy], (row["id"], strategy)
+            assert api.transform(text, findings, policy) == result
+            for record, finding in zip(result.transformations, emails):
+                assert record.source_byte_range == finding.byte_range
+                assert record.source_codepoint_range == finding.codepoint_range
+                assert result.text.encode()[record.output_byte_range.start:record.output_byte_range.end].decode() == record.replacement
+            assert api.scan_and_transform_structured(data, envelope).data == {"source": result.text}
+        if row["config"]["format"] == "text":
+            assert api.scan(text) == findings
+    for invalid in [None, 1, True, {}, "yaml", "ENV"]:
+        for call in [
+            lambda: api.scan("", {"format": invalid}),
+            lambda: api.scan_structured({"source": ""}, {"format": invalid}),
+        ]:
+            try:
+                call()
+            except api.DataFogConfigurationError as error:
+                assert error.path == "/format"
+            else:
+                raise AssertionError("Invalid source format accepted")

{datafog_core-0.4.1 → datafog_core-0.4.3}/bindings/python/tests/test_installed.py

@@ -12,6 +12,7 @@ import jwt_conformance
 import bearer_token_conformance
 import private_key_conformance
 import api_key_conformance
+import email_context_conformance
 import uuid_conformance
 import credential_uri_conformance
 import us_routing_number_conformance
@@ -125,7 +126,44 @@ def verify_structured() -> None:
             raise AssertionError("invalid structured input accepted")
 
 
+def verify_text_selection() -> None:
+    text = "👋 jane@example.com SSN 123-45-6789"
+    complete = scan(text)
+    selected = scan(text, {"entities": ["EMAIL"]})
+    expected = [finding for finding in complete if finding.entity_type == "EMAIL"]
+    assert len(selected) == len(expected) == 1
+    assert selected[0].codepoint_range.start == expected[0].codepoint_range.start
+    assert selected[0].byte_range.start == expected[0].byte_range.start
+    assert selected[0].matched_text == "jane@example.com"
+    transformed = scan_and_transform(text, {
+        "scan": {"entities": ["EMAIL"]},
+        "transform": {"default": {"strategy": "redact"}},
+    })
+    assert "123-45-6789" in transformed.text
+    assert "jane@example.com" not in transformed.text
+    for entities in ([], ["EMAIL", "EMAIL"], ["PERSON"], ["email"], [1], "EMAIL"):
+        try:
+            scan(text, {"entities": entities})
+        except DataFogConfigurationError as error:
+            assert error.code == "invalid_configuration"
+            assert error.path.startswith("/entities")
+        else:
+            raise AssertionError(f"invalid selection accepted: {entities!r}")
+    uuid = "550e8400-e29b-41d4-a716-446655440000"
+    assert not scan(uuid, {"entities": ["UUID"]})
+    assert [item.entity_type for item in scan(uuid, {"entities": ["UUID"], "detect_uuid": True})] == ["UUID"]
+    assert not scan(uuid, {"entities": ["EMAIL"], "detect_uuid": True})
+    assert scan("contact=jane@example.com", {"format": "env", "entities": ["EMAIL"]})[0].matched_text == "jane@example.com"
+    try:
+        scan_structured({"email": "jane@example.com"}, {"entities": ["EMAIL"]})
+    except DataFogConfigurationError as error:
+        assert error.reason == "unknown_field"
+    else:
+        raise AssertionError("text selection was accepted in structured scan")
+
+
 def main() -> None:
+    verify_text_selection()
     capabilities_conformance.verify()
     german_conformance.verify()
     credential_overlap_conformance.verify()
@@ -133,6 +171,7 @@ def main() -> None:
     bearer_token_conformance.verify()
     private_key_conformance.verify()
     api_key_conformance.verify()
+    email_context_conformance.verify()
     uuid_conformance.verify()
     credential_uri_conformance.verify()
     us_routing_number_conformance.verify()

{datafog_core-0.4.1 → datafog_core-0.4.3}/bindings/python/tests/test_typing.py

@@ -42,7 +42,7 @@ config: _TransformationConfig = {
     },
     "allow": {"exact": {"EMAIL": ["safe@example.com"]}, "regex": {"EMAIL": [{"pattern": "safe", "case_sensitive": False}]}},
 }
-findings = scan("jane@example.com", {"locale": "en-US", "detect_uuid": True})
+findings = scan("jane@example.com", {"locale": "en-US", "detect_uuid": True, "format": "env", "entities": ["EMAIL"]})
 assert_type(findings, list[Finding])
 finding = findings[0]
 assert_type(finding.entity_type, str)
@@ -127,9 +127,12 @@ INVALID = """
 from datafog_core import capabilities, scan, transform, scan_structured, PrivacyManager, TransformResult
 capabilities(1)  # error
 scan(123)  # error
+scan("text", {"format": "yaml"})  # error
 scan("text", {"locale": 123})  # error
 scan("text", {"detect_uuid": "true"})  # error
 scan("text", {"max_bytes": 100})  # error
+scan("text", {"entities": "EMAIL"})  # error
+scan_structured({"email": "jane@example.com"}, {"entities": ["EMAIL"]})  # error
 scan("text")[0].entity_type = "OTHER"  # error
 scan("text")[0].byte_range.start = 1  # error
 transform("text", ["not a finding"], {"default": {"strategy": "redact"}})  # error

{datafog_core-0.4.1 → datafog_core-0.4.3}/crates/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core"
-version = "0.4.1"
+version = "0.4.3"
 edition = "2024"
 rust-version = "1.88"
 description = "Fast structured PII detection library"

{datafog_core-0.4.1 → datafog_core-0.4.3/crates/core}/README.md

@@ -56,8 +56,12 @@ full-match regex values:
 }
 ```
 
-`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean }, transform: ... }` so
+`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean, format?: "text" | "env" | "sql", entities?: string[] }, transform: ... }` so
 detection settings remain separate from transformation policy.
+For `.env` assignments or standard SQL quoted strings, select `format: "env"`
+or `format: "sql"` to preserve email boundaries. See the
+[email source-context policy](docs/guides/email-boundaries.mdx) for supported
+syntax, raw escaped spans, and limitations. Default text scanning is unchanged.
 
 ## Packages
 
@@ -298,8 +302,28 @@ boundaries, never newlines. Returned text and offsets preserve the source.
 Passport and residence-permit patterns are legacy heuristics with limited
 coverage. See the [German entity reference](docs/reference/german-entities.mdx)
 and [migration differences](docs/guides/migrating-from-datafog-python.mdx).
-Core 0.4.0 is published. The higher-level Python adapter and its dependency
-update remain separate and unreleased in [draft PR #179](https://github.com/DataFog/datafog-python/pull/179).
+The higher-level Python adapter and its dependency update were merged in
+[PR #179](https://github.com/DataFog/datafog-python/pull/179). Its package release is separate from Core.
+
+## Text detector selection
+
+Pass `{"entities":["EMAIL","SSN"]}` to a text scan to run only those
+selected detectors, rather than detecting every type and filtering findings
+later. In Rust, use `ScanConfig::new().with_entities(vec!["EMAIL".into(),
+"SSN".into()])?`. The same selection works inside `scan_and_transform`'s
+`scan` configuration; transformation selection remains independent.
+
+Omitting `entities` preserves existing detection. An explicit selection must be
+a nonempty array of unique, exact canonical text entity names. Unknown names,
+lowercase names, duplicates, and structured-only `PERSON` are rejected using
+configuration errors with `/entities` or `/entities/<index>` paths. Consult
+`capabilities().entities` for types with the `text` scope.
+
+Selection narrows enabled detectors: selecting `UUID` still requires
+`detect_uuid=true`, and German types still require a German locale. Text format
+handling, finding order, and byte/codepoint ranges are unchanged. Structured
+scans do not accept this option. It avoids unselected detector work and findings,
+but does not cap the selected findings or bound the size of a scan response.
 
 ## UUID identifiers
 
@@ -309,6 +333,6 @@ structured scans, independently of locale. It emits `UUID` findings for versions
 sensitivity. See the [UUID reference](docs/reference/uuid.mdx) for boundaries,
 excluded sentinel forms and transformation examples.
 
-**0.4.0 is published** for Rust, Python, Node.js, and WASM. The higher-level Python adapter remains a separate unreleased integration in [draft PR #179](https://github.com/DataFog/datafog-python/pull/179). See the [release notes](docs/releases/0-4-0.mdx), [runtime capabilities](docs/reference/capabilities.mdx), and [0.4.x compatibility policy](docs/reference/compatibility.mdx).
+**0.4.1 is published** for Rust, Python, Node.js, and WASM. The higher-level Python adapter is integrated in [merged PR #179](https://github.com/DataFog/datafog-python/pull/179); its package release is separate. See the [release notes](docs/releases/0-4-1.mdx), [runtime capabilities](docs/reference/capabilities.mdx), and [0.4.x compatibility policy](docs/reference/compatibility.mdx).
 
-The unreleased **0.4.1 candidate** adds three default detectors: provider-prefixed [`API_KEY`](docs/reference/api-keys.mdx), context-required [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), and whole PostgreSQL [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx). The registry reports 23 supported entities and 14 defaults with capability contract 1. See the [candidate release notes](docs/releases/0-4-1.mdx) for exact scopes, preserved legacy Python overlap behavior, and the required exact-wheel Python 4.9 integration gate. Python's default backend is unchanged.
+**Core 0.4.1** adds three default detectors: provider-prefixed [`API_KEY`](docs/reference/api-keys.mdx), context-required [`BEARER_TOKEN`](docs/reference/bearer-token.mdx), and whole PostgreSQL [`CREDENTIAL_URI`](docs/reference/credential-uri.mdx). The registry reports 23 supported entities and 14 defaults with capability contract 1. See the [release notes](docs/releases/0-4-1.mdx) for exact scopes, preserved legacy Python overlap behavior, and the passed local exact-wheel Python 4.9 integration gate. Python's default backend is unchanged.

{datafog_core-0.4.1 → datafog_core-0.4.3}/crates/core/src/capabilities.rs

@@ -88,6 +88,14 @@ struct Detector {
 }
 
 impl Detector {
+    fn enabled(&self, config: &ScanConfig) -> bool {
+        self.activation.enabled(config)
+            && self
+                .labels
+                .iter()
+                .any(|label| config.includes_entity(label.as_str()))
+    }
+
     fn single(label: Label, detect: fn(&str, &mut Vec<Candidate>), activation: Activation) -> Self {
         Self {
             labels: vec![label],
@@ -147,12 +155,21 @@ pub(super) fn valid_locale(locale: &str) -> bool {
     locale_activation(locale).is_some()
 }
 
-pub(super) fn detect(text: &str, config: &ScanConfig, candidates: &mut Vec<Candidate>) {
-    for detector in REGISTRY
+pub(super) fn text_entity_supported(entity: &str) -> bool {
+    REGISTRY
         .iter()
-        .filter(|detector| detector.activation.enabled(config))
-    {
-        (detector.detect)(text, candidates);
+        .any(|detector| detector.labels.iter().any(|label| label.as_str() == entity))
+}
+
+pub(super) fn detect(text: &str, config: &ScanConfig, candidates: &mut Vec<Candidate>) {
+    for detector in REGISTRY.iter().filter(|detector| detector.enabled(config)) {
+        if detector.labels == [Label::Email] {
+            crate::email_context::detect(text, config.format(), candidates);
+        } else if detector.activation == Activation::German {
+            german::detect_selected(text, config, candidates);
+        } else {
+            (detector.detect)(text, candidates);
+        }
     }
 }
 
@@ -214,3 +231,38 @@ pub fn capabilities() -> Capabilities {
         entities,
     }
 }
+
+#[cfg(test)]
+mod selection_dispatch_tests {
+    use super::*;
+
+    #[test]
+    fn dispatch_skips_every_registry_detector_without_a_selected_label() {
+        let activated = ScanConfig::new()
+            .with_locale("de")
+            .unwrap()
+            .with_uuid_detection(true);
+        for entity in capabilities()
+            .supported_entities
+            .iter()
+            .filter(|name| name.as_str() != "PERSON")
+        {
+            let selected = activated
+                .clone()
+                .with_entities(vec![entity.clone()])
+                .unwrap();
+            let enabled: Vec<_> = REGISTRY
+                .iter()
+                .filter(|detector| detector.enabled(&selected))
+                .collect();
+            assert_eq!(enabled.len(), 1, "{entity}");
+            assert!(
+                enabled[0]
+                    .labels
+                    .iter()
+                    .any(|label| label.as_str() == entity)
+            );
+        }
+        assert!(REGISTRY.iter().all(|detector| detector.enabled(&activated)));
+    }
+}

datafog_core-0.4.3/crates/core/src/email_context.rs

@@ -0,0 +1,172 @@
+//! Delimit original source slices before applying the shared email matcher.
+use crate::{Candidate, ScanFormat, detect_email};
+
+pub(super) fn detect(text: &str, format: ScanFormat, candidates: &mut Vec<Candidate>) {
+    match format {
+        ScanFormat::Text => detect_email(text, candidates),
+        ScanFormat::Env => detect_env(text, candidates),
+        ScanFormat::Sql => detect_sql(text, candidates),
+    }
+}
+
+fn detect_slice(text: &str, start: usize, end: usize, candidates: &mut Vec<Candidate>) {
+    let first = candidates.len();
+    detect_email(&text[start..end], candidates);
+    for candidate in &mut candidates[first..] {
+        candidate.start_byte += start;
+        candidate.end_byte += start;
+    }
+}
+
+fn line_end(text: &str, start: usize) -> usize {
+    text[start..]
+        .find('\n')
+        .map_or(text.len(), |end| start + end)
+}
+
+fn skip_horizontal_space(bytes: &[u8], mut cursor: usize) -> usize {
+    while matches!(bytes.get(cursor), Some(b' ' | b'\t' | b'\r')) {
+        cursor += 1;
+    }
+    cursor
+}
+
+fn env_value_start(line: &str) -> Option<usize> {
+    let bytes = line.as_bytes();
+    let mut cursor = skip_horizontal_space(bytes, 0);
+    if line[cursor..].starts_with("export") && matches!(bytes.get(cursor + 6), Some(b' ' | b'\t')) {
+        cursor = skip_horizontal_space(bytes, cursor + 6);
+    }
+    if !bytes
+        .get(cursor)
+        .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_')
+    {
+        return None;
+    }
+    cursor += 1;
+    while bytes
+        .get(cursor)
+        .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_')
+    {
+        cursor += 1;
+    }
+    cursor = skip_horizontal_space(bytes, cursor);
+    if bytes.get(cursor) != Some(&b'=') {
+        return None;
+    }
+    Some(skip_horizontal_space(bytes, cursor + 1))
+}
+
+// Return the closing delimiter or EOF. No values are decoded, so offsets and
+// escaped local parts remain contiguous spans of the caller's source text.
+fn quote_end(bytes: &[u8], opening: usize, doubled: bool, backslash: bool) -> usize {
+    let quote = bytes[opening];
+    let mut cursor = opening + 1;
+    while cursor < bytes.len() {
+        if backslash && bytes[cursor] == b'\\' {
+            cursor += if cursor + 1 < bytes.len() { 2 } else { 1 };
+        } else if bytes[cursor] == quote {
+            if doubled && bytes.get(cursor + 1) == Some(&quote) {
+                cursor += 2;
+            } else {
+                return cursor;
+            }
+        } else {
+            cursor += 1;
+        }
+    }
+    bytes.len()
+}
+
+fn detect_env(text: &str, candidates: &mut Vec<Candidate>) {
+    let bytes = text.as_bytes();
+    let mut cursor = 0;
+    while cursor < text.len() {
+        let end = line_end(text, cursor);
+        if let Some(value) = env_value_start(&text[cursor..end]) {
+            let start = cursor + value;
+            if matches!(bytes.get(start), Some(b'\'' | b'"')) {
+                let close = quote_end(bytes, start, false, bytes[start] == b'"');
+                detect_slice(text, start + 1, close, candidates);
+                if close == text.len() {
+                    break;
+                }
+                let tail_end = line_end(text, close + 1);
+                detect_env_unquoted(text, close + 1, tail_end, candidates);
+                cursor = tail_end;
+            } else {
+                detect_env_unquoted(text, start, end, candidates);
+                cursor = end;
+            }
+        } else {
+            detect_env_unquoted(text, cursor, end, candidates);
+            cursor = end;
+        }
+        if cursor < text.len() {
+            cursor += 1;
+        }
+    }
+}
+
+fn detect_env_unquoted(text: &str, start: usize, end: usize, candidates: &mut Vec<Candidate>) {
+    let bytes = text.as_bytes();
+    for cursor in start..end {
+        if bytes[cursor] == b'#' && (cursor == start || bytes[cursor - 1].is_ascii_whitespace()) {
+            detect_slice(text, start, cursor, candidates);
+            detect_slice(text, cursor + 1, end, candidates);
+            return;
+        }
+    }
+    detect_slice(text, start, end, candidates);
+}
+
+fn detect_sql(text: &str, candidates: &mut Vec<Candidate>) {
+    let bytes = text.as_bytes();
+    let mut cursor = 0;
+    let mut plain_start = 0;
+    while cursor < bytes.len() {
+        let end = if matches!(bytes[cursor], b'\'' | b'"') {
+            detect_slice(text, plain_start, cursor, candidates);
+            let close = quote_end(bytes, cursor, true, false);
+            detect_slice(text, cursor + 1, close, candidates);
+            if close < bytes.len() {
+                close + 1
+            } else {
+                close
+            }
+        } else if bytes[cursor..].starts_with(b"--") {
+            detect_slice(text, plain_start, cursor, candidates);
+            let end = line_end(text, cursor);
+            detect_slice(text, cursor + 2, end, candidates);
+            end
+        } else if bytes[cursor..].starts_with(b"/*") {
+            detect_slice(text, plain_start, cursor, candidates);
+            let mut end = cursor + 2;
+            let mut comment_start = end;
+            let mut depth = 1;
+            while end < bytes.len() && depth > 0 {
+                if bytes[end..].starts_with(b"/*") {
+                    detect_slice(text, comment_start, end, candidates);
+                    depth += 1;
+                    end += 2;
+                    comment_start = end;
+                } else if bytes[end..].starts_with(b"*/") {
+                    detect_slice(text, comment_start, end, candidates);
+                    depth -= 1;
+                    end += 2;
+                    comment_start = end;
+                } else {
+                    end += 1;
+                }
+            }
+            detect_slice(text, comment_start, end, candidates);
+            end
+        } else {
+            cursor += 1;
+            continue;
+        };
+        cursor = end;
+        plain_start = end;
+    }
+    detect_slice(text, plain_start, text.len(), candidates);
+}

{datafog_core-0.4.1 → datafog_core-0.4.3}/crates/core/src/german.rs

@@ -1,5 +1,5 @@
 //! Locale-gated lexical detection, without official identifier validation.
-use crate::{Candidate, Label};
+use crate::{Candidate, Label, ScanConfig};
 use regex::Regex;
 use std::sync::LazyLock;
 
@@ -71,7 +71,14 @@ fn ascii_boundary(text: &str, start: usize, end: usize) -> bool {
 }
 
 pub(super) fn detect(text: &str, candidates: &mut Vec<Candidate>) {
-    for (label, regex) in DETECTORS.iter() {
+    detect_selected(text, &ScanConfig::default(), candidates);
+}
+
+pub(super) fn detect_selected(text: &str, config: &ScanConfig, candidates: &mut Vec<Candidate>) {
+    for (label, regex) in DETECTORS
+        .iter()
+        .filter(|(label, _)| config.includes_entity(label.as_str()))
+    {
         for captures in regex.captures_iter(text) {
             // Both captures are unconditional in the statically compiled expressions.
             let whole = captures.get(0).expect("whole regex match");