datafog 4.7.0__tar.gz → 4.8.0__tar.gz

{datafog-4.7.0 → datafog-4.8.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: datafog
-Version: 4.7.0
-Summary: Lightning-fast PII detection and anonymization library with 190x performance advantage
+Version: 4.8.0
+Summary: Lightning-fast PII detection and anonymization library, 100x+ faster than NER-based detection (see benchmarks/)
 Author: Sid Mohan
 Author-email: sid@datafog.ai
 Project-URL: Homepage, https://datafog.ai
@@ -126,8 +126,8 @@ values never echoed into logs or transcripts:
 
 - **Claude Code hook** (`datafog-hook`): gates agent tool calls (shell
   commands, web requests, file writes, MCP tools) and warns the model when
-  prompts or tool results carry PII. ~70ms per invocation including process
-  startup. Easiest install is the
+  prompts or tool results carry PII. ~70–90ms per invocation including
+  process startup. Easiest install is the
   [Claude Code plugin](https://github.com/DataFog/datafog-claude-plugin):
 
   ```
@@ -139,7 +139,8 @@ values never echoed into logs or transcripts:
 
 - **LiteLLM guardrail** (`DataFogGuardrail`): redacts or blocks PII in
   requests and responses at the gateway, for any LiteLLM-proxied provider.
-  In-process (~31µs per request), no sidecar service. Setup:
+  In-process (~40µs per message scanned; a request clears the guardrail in
+  well under a millisecond), no sidecar service. Setup:
   [examples/litellm_guardrail/](examples/litellm_guardrail/).
 
 Both default to the high-precision entity set (`EMAIL`, `PHONE`,
@@ -150,6 +151,10 @@ unix timestamps matching as phone numbers) — available in both adapters and
 the API. Presidio-style entity names (`EMAIL_ADDRESS`, `PHONE_NUMBER`,
 `US_SSN`) are accepted as aliases for easy migration.
 
+Every performance number above is reproducible with one command —
+methodology, pinned payloads, and comparisons against Presidio and spaCy
+NER live in [benchmarks/](benchmarks/).
+
 ## Installation
 
 ```bash

{datafog-4.7.0 → datafog-4.8.0}/README.md

@@ -19,8 +19,8 @@ values never echoed into logs or transcripts:
 
 - **Claude Code hook** (`datafog-hook`): gates agent tool calls (shell
   commands, web requests, file writes, MCP tools) and warns the model when
-  prompts or tool results carry PII. ~70ms per invocation including process
-  startup. Easiest install is the
+  prompts or tool results carry PII. ~70–90ms per invocation including
+  process startup. Easiest install is the
   [Claude Code plugin](https://github.com/DataFog/datafog-claude-plugin):
 
   ```
@@ -32,7 +32,8 @@ values never echoed into logs or transcripts:
 
 - **LiteLLM guardrail** (`DataFogGuardrail`): redacts or blocks PII in
   requests and responses at the gateway, for any LiteLLM-proxied provider.
-  In-process (~31µs per request), no sidecar service. Setup:
+  In-process (~40µs per message scanned; a request clears the guardrail in
+  well under a millisecond), no sidecar service. Setup:
   [examples/litellm_guardrail/](examples/litellm_guardrail/).
 
 Both default to the high-precision entity set (`EMAIL`, `PHONE`,
@@ -43,6 +44,10 @@ unix timestamps matching as phone numbers) — available in both adapters and
 the API. Presidio-style entity names (`EMAIL_ADDRESS`, `PHONE_NUMBER`,
 `US_SSN`) are accepted as aliases for easy migration.
 
+Every performance number above is reproducible with one command —
+methodology, pinned payloads, and comparisons against Presidio and spaCy
+NER live in [benchmarks/](benchmarks/).
+
 ## Installation
 
 ```bash

datafog-4.8.0/datafog/__about__.py

@@ -0,0 +1 @@
+__version__ = "4.8.0"

{datafog-4.7.0 → datafog-4.8.0}/datafog/__init__.py

@@ -1,7 +1,8 @@
 """
 DataFog: Lightning-fast PII detection and anonymization library.
 
-Core package provides regex-based PII detection with 190x performance advantage.
+Core package provides regex-based PII detection, 100x+ faster than NER-based
+detection on identical payloads (reproduce: python benchmarks/run.py).
 Optional extras available for advanced features:
 - pip install datafog[nlp] - for spaCy integration
 - pip install datafog[ocr] - for image/OCR processing

{datafog-4.7.0 → datafog-4.8.0}/datafog/__init___lean.py

@@ -7,7 +7,8 @@ only as historical reference until legacy cleanup can remove it safely.
 
 DataFog: Lightning-fast PII detection and anonymization library.
 
-Core package provides regex-based PII detection with 190x performance advantage.
+Core package provides regex-based PII detection, 100x+ faster than NER-based
+detection on identical payloads (reproduce: python benchmarks/run.py).
 Optional extras available for advanced features:
 - pip install datafog[nlp] - for spaCy integration
 - pip install datafog[ocr] - for image/OCR processing

{datafog-4.7.0 → datafog-4.8.0}/datafog/engine.py

@@ -173,6 +173,7 @@ def _suppress_overlapping_entities(entities: list[Entity]) -> list[Entity]:
         key=lambda item: (
             -_entity_length(item),
             -ENTITY_TYPE_PRIORITY.get(item.type, 0),
+            -item.confidence,
             item.start,
             item.end,
             item.type,
@@ -465,7 +466,18 @@ def redact(
     entities: list[Entity],
     strategy: str = "token",
 ) -> RedactResult:
-    """Redact PII entities from text."""
+    """Redact PII entities from text.
+
+    Callers may pass arbitrary entity lists (not just ``scan()`` output), so
+    overlapping or duplicate spans are resolved here: for each overlapping
+    group only one span is redacted — the longest, breaking ties by entity
+    type priority and then confidence. Suppressed spans are omitted from
+    ``RedactResult.entities`` and ``mapping``.
+
+    ``mapping`` for the ``mask`` strategy is keyed by per-type indexed keys
+    (``[EMAIL_MASK_1]``) rather than the mask string, because distinct
+    same-length values produce identical masks and would collide.
+    """
     if not isinstance(text, str):
         raise TypeError("text must be a string")
     if strategy not in {"token", "mask", "hash", "pseudonymize"}:
@@ -481,14 +493,22 @@ def redact(
         for entity in entities
         if 0 <= entity.start < entity.end <= len(text) and entity.text
     ]
-    valid_entities = sorted(
-        valid_entities, key=lambda e: (e.start, e.end), reverse=True
-    )
+    # Returns non-overlapping spans sorted by document position.
+    valid_entities = _suppress_overlapping_entities(valid_entities)
 
+    # Assign replacements in document order so the per-type counters used by
+    # the token/pseudonymize strategies number the first occurrence _1; spans
+    # are applied right-to-left below so earlier offsets stay valid.
+    replacements: list[tuple[Entity, str]] = []
     for entity in valid_entities:
-        original = redacted_text[entity.start : entity.end]
+        original = text[entity.start : entity.end]
+        mapping_key: Optional[str] = None
         if strategy == "mask":
             replacement = "*" * max(len(original), 1)
+            # Distinct same-length values share a mask, so the mask string
+            # cannot key the mapping without collisions.
+            counters[entity.type] = counters.get(entity.type, 0) + 1
+            mapping_key = f"[{entity.type}_MASK_{counters[entity.type]}]"
         elif strategy == "hash":
             digest = hashlib.sha256(original.encode("utf-8")).hexdigest()[:12]
             replacement = f"[{entity.type}_{digest}]"
@@ -504,10 +524,13 @@ def redact(
             counters[entity.type] = counters.get(entity.type, 0) + 1
             replacement = f"[{entity.type}_{counters[entity.type]}]"
 
+        replacements.append((entity, replacement))
+        mapping[mapping_key if mapping_key is not None else replacement] = original
+
+    for entity, replacement in reversed(replacements):
         redacted_text = (
             redacted_text[: entity.start] + replacement + redacted_text[entity.end :]
         )
-        mapping[replacement] = original
 
     return RedactResult(
         redacted_text=redacted_text,

{datafog-4.7.0 → datafog-4.8.0}/datafog.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: datafog
-Version: 4.7.0
-Summary: Lightning-fast PII detection and anonymization library with 190x performance advantage
+Version: 4.8.0
+Summary: Lightning-fast PII detection and anonymization library, 100x+ faster than NER-based detection (see benchmarks/)
 Author: Sid Mohan
 Author-email: sid@datafog.ai
 Project-URL: Homepage, https://datafog.ai
@@ -126,8 +126,8 @@ values never echoed into logs or transcripts:
 
 - **Claude Code hook** (`datafog-hook`): gates agent tool calls (shell
   commands, web requests, file writes, MCP tools) and warns the model when
-  prompts or tool results carry PII. ~70ms per invocation including process
-  startup. Easiest install is the
+  prompts or tool results carry PII. ~70–90ms per invocation including
+  process startup. Easiest install is the
   [Claude Code plugin](https://github.com/DataFog/datafog-claude-plugin):
 
   ```
@@ -139,7 +139,8 @@ values never echoed into logs or transcripts:
 
 - **LiteLLM guardrail** (`DataFogGuardrail`): redacts or blocks PII in
   requests and responses at the gateway, for any LiteLLM-proxied provider.
-  In-process (~31µs per request), no sidecar service. Setup:
+  In-process (~40µs per message scanned; a request clears the guardrail in
+  well under a millisecond), no sidecar service. Setup:
   [examples/litellm_guardrail/](examples/litellm_guardrail/).
 
 Both default to the high-precision entity set (`EMAIL`, `PHONE`,
@@ -150,6 +151,10 @@ unix timestamps matching as phone numbers) — available in both adapters and
 the API. Presidio-style entity names (`EMAIL_ADDRESS`, `PHONE_NUMBER`,
 `US_SSN`) are accepted as aliases for easy migration.
 
+Every performance number above is reproducible with one command —
+methodology, pinned payloads, and comparisons against Presidio and spaCy
+NER live in [benchmarks/](benchmarks/).
+
 ## Installation
 
 ```bash

{datafog-4.7.0 → datafog-4.8.0}/setup.py

@@ -110,7 +110,7 @@ setup(
     version=version,
     author="Sid Mohan",
     author_email="sid@datafog.ai",
-    description="Lightning-fast PII detection and anonymization library with 190x performance advantage",
+    description="Lightning-fast PII detection and anonymization library, 100x+ faster than NER-based detection (see benchmarks/)",
     long_description=long_description,
     long_description_content_type="text/markdown",
     packages=find_packages(exclude=["tests", "tests.*"]),

datafog-4.8.0/tests/test_engine_api.py

@@ -0,0 +1,268 @@
+"""Tests for the internal engine boundary API."""
+
+from __future__ import annotations
+
+import pytest
+
+from datafog.engine import Entity, redact, scan, scan_and_redact
+from datafog.exceptions import EngineNotAvailable
+
+
+def test_scan_regex_detects_structured_entities() -> None:
+    result = scan("Email john@example.com and SSN 123-45-6789", engine="regex")
+
+    entity_types = {entity.type for entity in result.entities}
+    assert "EMAIL" in entity_types
+    assert "SSN" in entity_types
+    assert result.engine_used == "regex"
+
+
+def test_scan_filters_entity_types() -> None:
+    result = scan(
+        "Email john@example.com and SSN 123-45-6789",
+        engine="regex",
+        entity_types=["EMAIL"],
+    )
+    assert result.entities
+    assert {entity.type for entity in result.entities} == {"EMAIL"}
+
+
+def test_scan_invalid_engine_raises_value_error() -> None:
+    with pytest.raises(ValueError, match="engine must be one of"):
+        scan("test", engine="invalid")
+
+
+def test_scan_non_string_raises_type_error() -> None:
+    with pytest.raises(TypeError, match="text must be a string"):
+        scan(None, engine="regex")  # type: ignore[arg-type]
+
+
+@pytest.mark.parametrize("strategy", ["token", "mask", "hash", "pseudonymize"])
+def test_redact_strategies(strategy: str) -> None:
+    text = "Contact john@example.com"
+    entities = [
+        Entity(
+            type="EMAIL",
+            text="john@example.com",
+            start=8,
+            end=24,
+            confidence=1.0,
+            engine="regex",
+        )
+    ]
+
+    result = redact(text=text, entities=entities, strategy=strategy)
+    assert result.redacted_text != text
+    assert result.mapping
+
+
+def _entity_at(text: str, value: str, entity_type: str = "EMAIL") -> Entity:
+    start = text.index(value)
+    return Entity(
+        type=entity_type,
+        text=value,
+        start=start,
+        end=start + len(value),
+        confidence=1.0,
+        engine="regex",
+    )
+
+
+def test_redact_token_numbering_follows_document_order() -> None:
+    text = "first alpha@example.com then beta@example.com end"
+    entities = [
+        _entity_at(text, "alpha@example.com"),
+        _entity_at(text, "beta@example.com"),
+    ]
+
+    result = redact(text=text, entities=entities, strategy="token")
+
+    assert result.redacted_text == "first [EMAIL_1] then [EMAIL_2] end"
+    assert result.mapping == {
+        "[EMAIL_1]": "alpha@example.com",
+        "[EMAIL_2]": "beta@example.com",
+    }
+
+
+def test_redact_pseudonymize_numbering_follows_document_order() -> None:
+    text = "first alpha@example.com then beta@example.com end"
+    entities = [
+        _entity_at(text, "alpha@example.com"),
+        _entity_at(text, "beta@example.com"),
+    ]
+
+    result = redact(text=text, entities=entities, strategy="pseudonymize")
+
+    assert result.redacted_text == "first [EMAIL_PSEUDO_1] then [EMAIL_PSEUDO_2] end"
+    assert result.mapping == {
+        "[EMAIL_PSEUDO_1]": "alpha@example.com",
+        "[EMAIL_PSEUDO_2]": "beta@example.com",
+    }
+
+
+def _span(
+    text: str,
+    start: int,
+    end: int,
+    entity_type: str = "PHONE",
+    confidence: float = 1.0,
+    engine: str = "regex",
+) -> Entity:
+    return Entity(
+        type=entity_type,
+        text=text[start:end],
+        start=start,
+        end=end,
+        confidence=confidence,
+        engine=engine,
+    )
+
+
+@pytest.mark.parametrize("strategy", ["token", "mask", "hash", "pseudonymize"])
+def test_redact_overlapping_spans_match_longest_only(strategy: str) -> None:
+    text = "call 555-000-1111 now"
+    full = _span(text, 5, 17)
+    suffix = _span(text, 9, 17)
+
+    result = redact(text=text, entities=[full, suffix], strategy=strategy)
+    expected = redact(text=text, entities=[full], strategy=strategy)
+
+    assert result.redacted_text == expected.redacted_text
+    assert result.mapping == expected.mapping
+    assert result.entities == [full]
+
+
+@pytest.mark.parametrize("strategy", ["token", "mask", "hash", "pseudonymize"])
+def test_redact_nested_spans_keep_outer(strategy: str) -> None:
+    text = "mail alice@example.com today"
+    outer = _span(text, 5, 22, entity_type="EMAIL")
+    inner = _span(text, 11, 18, entity_type="EMAIL")
+
+    result = redact(text=text, entities=[inner, outer], strategy=strategy)
+    expected = redact(text=text, entities=[outer], strategy=strategy)
+
+    assert result.redacted_text == expected.redacted_text
+    assert result.mapping == expected.mapping
+    assert result.entities == [outer]
+
+
+def test_redact_overlapping_spans_produce_clean_token_output() -> None:
+    text = "call 555-000-1111 now"
+    entities = [_span(text, 5, 17), _span(text, 9, 17)]
+
+    result = redact(text=text, entities=entities, strategy="token")
+
+    assert result.redacted_text == "call [PHONE_1] now"
+    assert result.mapping == {"[PHONE_1]": text[5:17]}
+
+
+def test_redact_duplicate_spans_applied_once() -> None:
+    text = "mail alice@example.com today"
+    entity = _entity_at(text, "alice@example.com")
+
+    result = redact(text=text, entities=[entity, entity], strategy="token")
+
+    assert result.redacted_text == "mail [EMAIL_1] today"
+    assert result.entities == [entity]
+
+
+def test_redact_overlap_tiebreak_prefers_higher_confidence() -> None:
+    text = "Acme Corporation announced"
+    org = _span(text, 0, 16, entity_type="ORGANIZATION", confidence=0.7, engine="spacy")
+    person = _span(text, 0, 16, entity_type="PERSON", confidence=0.9, engine="gliner")
+
+    result = redact(text=text, entities=[org, person], strategy="token")
+
+    assert result.redacted_text == "[PERSON_1] announced"
+    assert result.entities == [person]
+
+
+def test_redact_mask_mapping_distinguishes_same_length_values() -> None:
+    text = "a alice@example.com b bobby@example.com c"
+    entities = [
+        _entity_at(text, "alice@example.com"),
+        _entity_at(text, "bobby@example.com"),
+    ]
+
+    result = redact(text=text, entities=entities, strategy="mask")
+
+    assert result.redacted_text == "a " + "*" * 17 + " b " + "*" * 17 + " c"
+    assert result.mapping == {
+        "[EMAIL_MASK_1]": "alice@example.com",
+        "[EMAIL_MASK_2]": "bobby@example.com",
+    }
+
+
+def test_redact_invalid_strategy_raises_value_error() -> None:
+    with pytest.raises(ValueError, match="strategy must be one of"):
+        redact("test", entities=[], strategy="invalid")
+
+
+def test_redact_ignores_invalid_spans() -> None:
+    text = "hello"
+    entities = [
+        Entity(
+            type="EMAIL",
+            text="x",
+            start=-1,
+            end=2,
+            confidence=1.0,
+            engine="regex",
+        ),
+        Entity(
+            type="EMAIL",
+            text="x",
+            start=2,
+            end=10,
+            confidence=1.0,
+            engine="regex",
+        ),
+    ]
+
+    result = redact(text=text, entities=entities, strategy="token")
+    assert result.redacted_text == text
+    assert result.mapping == {}
+
+
+def test_scan_and_redact_combines_operations() -> None:
+    text = "Call me at (555) 123-4567"
+    result = scan_and_redact(text=text, engine="regex", strategy="token")
+
+    assert result.entities
+    assert "[PHONE_1]" in result.redacted_text
+
+
+@pytest.mark.asyncio
+async def test_scan_from_async_context() -> None:
+    """Verify sync engine API works when called from async code."""
+    result = scan("john@example.com", engine="regex")
+    assert len(result.entities) >= 1
+
+
+def test_gliner_engine_unavailable_raises_clear_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    def _raise(_: str):
+        raise EngineNotAvailable(
+            "GLiNER engine requires the nlp-advanced extra. Install with: pip install datafog[nlp-advanced]"
+        )
+
+    monkeypatch.setattr("datafog.engine._gliner_entities", _raise)
+
+    with pytest.raises(EngineNotAvailable, match="nlp-advanced"):
+        scan("john@example.com", engine="gliner")
+
+
+def test_smart_engine_degrades_to_regex_with_warning(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    def _raise(_: str):
+        raise EngineNotAvailable("not installed")
+
+    monkeypatch.setattr("datafog.engine._gliner_entities", _raise)
+    monkeypatch.setattr("datafog.engine._spacy_entities", _raise)
+
+    with pytest.warns(UserWarning, match="regex only"):
+        result = scan("john@example.com", engine="smart")
+
+    assert any(entity.type == "EMAIL" for entity in result.entities)

datafog-4.7.0/datafog/__about__.py

@@ -1 +0,0 @@
-__version__ = "4.7.0"

datafog-4.7.0/tests/test_engine_api.py

@@ -1,131 +0,0 @@
-"""Tests for the internal engine boundary API."""
-
-from __future__ import annotations
-
-import pytest
-
-from datafog.engine import Entity, redact, scan, scan_and_redact
-from datafog.exceptions import EngineNotAvailable
-
-
-def test_scan_regex_detects_structured_entities() -> None:
-    result = scan("Email john@example.com and SSN 123-45-6789", engine="regex")
-
-    entity_types = {entity.type for entity in result.entities}
-    assert "EMAIL" in entity_types
-    assert "SSN" in entity_types
-    assert result.engine_used == "regex"
-
-
-def test_scan_filters_entity_types() -> None:
-    result = scan(
-        "Email john@example.com and SSN 123-45-6789",
-        engine="regex",
-        entity_types=["EMAIL"],
-    )
-    assert result.entities
-    assert {entity.type for entity in result.entities} == {"EMAIL"}
-
-
-def test_scan_invalid_engine_raises_value_error() -> None:
-    with pytest.raises(ValueError, match="engine must be one of"):
-        scan("test", engine="invalid")
-
-
-def test_scan_non_string_raises_type_error() -> None:
-    with pytest.raises(TypeError, match="text must be a string"):
-        scan(None, engine="regex")  # type: ignore[arg-type]
-
-
-@pytest.mark.parametrize("strategy", ["token", "mask", "hash", "pseudonymize"])
-def test_redact_strategies(strategy: str) -> None:
-    text = "Contact john@example.com"
-    entities = [
-        Entity(
-            type="EMAIL",
-            text="john@example.com",
-            start=8,
-            end=24,
-            confidence=1.0,
-            engine="regex",
-        )
-    ]
-
-    result = redact(text=text, entities=entities, strategy=strategy)
-    assert result.redacted_text != text
-    assert result.mapping
-
-
-def test_redact_invalid_strategy_raises_value_error() -> None:
-    with pytest.raises(ValueError, match="strategy must be one of"):
-        redact("test", entities=[], strategy="invalid")
-
-
-def test_redact_ignores_invalid_spans() -> None:
-    text = "hello"
-    entities = [
-        Entity(
-            type="EMAIL",
-            text="x",
-            start=-1,
-            end=2,
-            confidence=1.0,
-            engine="regex",
-        ),
-        Entity(
-            type="EMAIL",
-            text="x",
-            start=2,
-            end=10,
-            confidence=1.0,
-            engine="regex",
-        ),
-    ]
-
-    result = redact(text=text, entities=entities, strategy="token")
-    assert result.redacted_text == text
-    assert result.mapping == {}
-
-
-def test_scan_and_redact_combines_operations() -> None:
-    text = "Call me at (555) 123-4567"
-    result = scan_and_redact(text=text, engine="regex", strategy="token")
-
-    assert result.entities
-    assert "[PHONE_1]" in result.redacted_text
-
-
-@pytest.mark.asyncio
-async def test_scan_from_async_context() -> None:
-    """Verify sync engine API works when called from async code."""
-    result = scan("john@example.com", engine="regex")
-    assert len(result.entities) >= 1
-
-
-def test_gliner_engine_unavailable_raises_clear_error(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    def _raise(_: str):
-        raise EngineNotAvailable(
-            "GLiNER engine requires the nlp-advanced extra. Install with: pip install datafog[nlp-advanced]"
-        )
-
-    monkeypatch.setattr("datafog.engine._gliner_entities", _raise)
-
-    with pytest.raises(EngineNotAvailable, match="nlp-advanced"):
-        scan("john@example.com", engine="gliner")
-
-
-def test_smart_engine_degrades_to_regex_with_warning(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    def _raise(_: str):
-        raise EngineNotAvailable("not installed")
-
-    monkeypatch.setattr("datafog.engine._gliner_entities", _raise)
-    monkeypatch.setattr("datafog.engine._spacy_entities", _raise)
-
-    with pytest.warns(UserWarning, match="regex only"):
-        result = scan("john@example.com", engine="smart")
-
-    assert any(entity.type == "EMAIL" for entity in result.entities)