euredact 0.1.0__py3-none-any.whl

euredact/__init__.py

@@ -0,0 +1,196 @@
+"""EuRedact -- European PII redaction SDK.
+
+Quick start::
+
+    import euredact
+
+    result = euredact.redact("Jan Janssens, BSN 123456782, woont in Gent.")
+    print(result.redacted_text)
+    print(result.detections)
+
+    # Batch processing:
+    results = euredact.redact_batch(["text one", "text two"], countries=["NL"])
+
+    # Async:
+    result = await euredact.aredact("some text")
+
+    # Available countries:
+    print(euredact.available_countries())  # ['AT', 'BE', 'DE', 'NL', ...]
+"""
+
+from __future__ import annotations
+
+from typing import Iterator
+
+__version__ = "0.1.0"
+
+from euredact.sdk import EuRedact
+from euredact.types import Detection, DetectionSource, EntityType, RedactResult
+
+__all__ = [
+    "__version__",
+    "aredact",
+    "aredact_batch",
+    "available_countries",
+    "Detection",
+    "DetectionSource",
+    "EntityType",
+    "EuRedact",
+    "redact",
+    "redact_batch",
+    "redact_iter",
+    "RedactResult",
+]
+
+# ---------------------------------------------------------------------------
+# Module-level singleton
+# ---------------------------------------------------------------------------
+
+_instance: EuRedact | None = None
+
+
+def _get_instance() -> EuRedact:
+    global _instance
+    if _instance is None:
+        _instance = EuRedact()
+    return _instance
+
+
+# ---------------------------------------------------------------------------
+# Public helpers
+# ---------------------------------------------------------------------------
+
+
+def available_countries() -> list[str]:
+    """Return a sorted list of supported country codes (e.g. ``['AT', 'BE', ...]``)."""
+    from euredact.rules.registry import CountryRegistry
+    return CountryRegistry().available_countries
+
+
+def redact(
+    text: str,
+    *,
+    countries: list[str] | None = None,
+    mode: str = "rules",
+    pseudonymize: bool = False,
+    detect_dates: bool = False,
+    coref: bool = False,
+    coref_model: str = "default",
+    cache: bool = True,
+) -> RedactResult:
+    """Redact PII from text. Main entry point.
+
+    Args:
+        detect_dates: Include date-of-birth / date-of-death detections.
+            Off by default -- bare dates are better handled by the cloud
+            LLM tier. When True, the rule engine applies keyword and
+            structural (JSON/CSV header) checks before emitting a date.
+    """
+    return _get_instance().redact(
+        text,
+        countries=countries,
+        mode=mode,
+        pseudonymize=pseudonymize,
+        detect_dates=detect_dates,
+        coref=coref,
+        coref_model=coref_model,
+        cache=cache,
+    )
+
+
+async def aredact(
+    text: str,
+    *,
+    countries: list[str] | None = None,
+    mode: str = "rules",
+    pseudonymize: bool = False,
+    detect_dates: bool = False,
+    cache: bool = True,
+) -> RedactResult:
+    """Async version of redact().
+
+    Offloads CPU-bound work to a thread pool so it doesn't block the
+    event loop. Safe to ``await`` from multiple concurrent tasks.
+    """
+    return await _get_instance().aredact(
+        text,
+        countries=countries,
+        mode=mode,
+        pseudonymize=pseudonymize,
+        detect_dates=detect_dates,
+        cache=cache,
+    )
+
+
+def redact_batch(
+    texts: list[str],
+    *,
+    countries: list[str] | None = None,
+    mode: str = "rules",
+    pseudonymize: bool = False,
+    detect_dates: bool = False,
+    cache: bool = True,
+) -> list[RedactResult]:
+    """Redact PII from multiple texts at once.
+
+    More efficient than calling ``redact()`` in a loop -- loads country
+    configs once. Returns results in the same order as the input.
+    """
+    return _get_instance().redact_batch(
+        texts,
+        countries=countries,
+        mode=mode,
+        pseudonymize=pseudonymize,
+        detect_dates=detect_dates,
+        cache=cache,
+    )
+
+
+async def aredact_batch(
+    texts: list[str],
+    *,
+    countries: list[str] | None = None,
+    mode: str = "rules",
+    pseudonymize: bool = False,
+    detect_dates: bool = False,
+    cache: bool = True,
+    max_concurrency: int = 4,
+) -> list[RedactResult]:
+    """Async batch redaction with controlled concurrency.
+
+    Processes texts concurrently in a thread pool. ``max_concurrency``
+    limits parallel threads (default 4). Returns results in input order.
+    """
+    return await _get_instance().aredact_batch(
+        texts,
+        countries=countries,
+        mode=mode,
+        pseudonymize=pseudonymize,
+        detect_dates=detect_dates,
+        cache=cache,
+        max_concurrency=max_concurrency,
+    )
+
+
+def redact_iter(
+    texts: Iterator[str],
+    *,
+    countries: list[str] | None = None,
+    mode: str = "rules",
+    pseudonymize: bool = False,
+    detect_dates: bool = False,
+    cache: bool = True,
+) -> Iterator[RedactResult]:
+    """Lazy iterator that yields results one at a time.
+
+    Useful for processing large datasets without loading all results
+    into memory. Loads country configs once on the first item.
+    """
+    return _get_instance().redact_iter(
+        texts,
+        countries=countries,
+        mode=mode,
+        pseudonymize=pseudonymize,
+        detect_dates=detect_dates,
+        cache=cache,
+    )

euredact/cache.py

@@ -0,0 +1,50 @@
+"""LRU result cache with SHA-256 content hashing."""
+
+from __future__ import annotations
+
+import hashlib
+import threading
+from collections import OrderedDict
+
+from euredact.types import RedactResult
+
+
+class ResultCache:
+    """LRU cache keyed on SHA-256 of input text + config hash."""
+
+    def __init__(self, maxsize: int = 1024, enabled: bool = True) -> None:
+        self._maxsize = maxsize
+        self._enabled = enabled
+        self._store: OrderedDict[str, RedactResult] = OrderedDict()
+        self._lock = threading.Lock()
+
+    def key(self, text: str, countries: tuple[str, ...], mode: str) -> str:
+        """Compute cache key from input text and configuration."""
+        raw = f"{text}|{'|'.join(sorted(countries))}|{mode}"
+        return hashlib.sha256(raw.encode()).hexdigest()
+
+    def get(self, key: str) -> RedactResult | None:
+        """Retrieve a cached result, or None on miss."""
+        if not self._enabled:
+            return None
+        with self._lock:
+            if key in self._store:
+                self._store.move_to_end(key)
+                return self._store[key]
+        return None
+
+    def put(self, key: str, result: RedactResult) -> None:
+        """Store a result in the cache."""
+        if not self._enabled:
+            return
+        with self._lock:
+            if key in self._store:
+                self._store.move_to_end(key)
+            self._store[key] = result
+            while len(self._store) > self._maxsize:
+                self._store.popitem(last=False)
+
+    def clear(self) -> None:
+        """Clear the entire cache."""
+        with self._lock:
+            self._store.clear()

euredact/cloud/__init__.py

@@ -0,0 +1 @@
+"""[CLOUD EXTENSION] Cloud tier — stubbed, not implemented."""

euredact/cloud/client.py

@@ -0,0 +1,19 @@
+"""[CLOUD EXTENSION] Placeholder cloud client."""
+
+from __future__ import annotations
+
+
+class NotConfiguredError(Exception):
+    """Raised when cloud tier is used without configuration."""
+
+    def __init__(self) -> None:
+        super().__init__(
+            "Cloud tier not configured. Call euredact.configure(api_key=...) first."
+        )
+
+
+class CloudClient:
+    """Placeholder for the cloud annotation client."""
+
+    def __init__(self) -> None:
+        raise NotConfiguredError()

euredact/cloud/hasher.py

@@ -0,0 +1 @@
+"""[CLOUD EXTENSION] Segment hashing + hash-to-position mapping."""

euredact/cloud/shuffler.py

@@ -0,0 +1 @@
+"""[CLOUD EXTENSION] Cross-client segment shuffling."""

euredact/coref/__init__.py

@@ -0,0 +1 @@
+"""[COREF EXTENSION] Coreference resolution — Pro/Enterprise, not implemented."""

euredact/coref/models.py

@@ -0,0 +1 @@
+"""[COREF EXTENSION] Model registry: manages DistilBERT + TinyBERT variants."""

euredact/coref/resolver.py

@@ -0,0 +1 @@
+"""[COREF EXTENSION] Local coreference model: merges partial entity refs."""

euredact/normalizer.py

@@ -0,0 +1,97 @@
+"""Unicode NFC normalization for input text."""
+
+from __future__ import annotations
+
+import unicodedata
+
+
+def normalize(text: str) -> tuple[str, list[int] | None]:
+    """Normalize text to NFC form before pattern matching.
+
+    Returns the normalized text and an offset mapping from normalized positions
+    to original positions (None if no length change occurred).
+    """
+    normalized = unicodedata.normalize("NFC", text)
+    if len(normalized) == len(text):
+        return normalized, None
+
+    # Build character offset mapping: normalized index -> original index
+    # We do this by normalizing character-by-character
+    mapping: list[int] = []
+    orig_idx = 0
+    for char in text:
+        nfc_char = unicodedata.normalize("NFC", char)
+        for _ in nfc_char:
+            mapping.append(orig_idx)
+        orig_idx += len(char.encode("utf-32-le")) // 4
+        # Actually, we need codepoint-level tracking
+    # Simpler approach: use NFC on full string and build mapping via
+    # decompose-then-compose tracking
+    # For most European text, NFC doesn't change length. When it does,
+    # the mapping lets us translate detection offsets back to original positions.
+    return normalized, _build_offset_mapping(text, normalized)
+
+
+def _build_offset_mapping(original: str, normalized: str) -> list[int]:
+    """Build mapping from normalized character positions to original positions.
+
+    Uses NFD as intermediate to align characters between original and NFC.
+    """
+    nfd_original = unicodedata.normalize("NFD", original)
+    nfd_normalized = unicodedata.normalize("NFD", normalized)
+
+    # Both NFD forms should be identical
+    # Build: original char index -> NFD index range
+    orig_to_nfd: list[int] = []
+    nfd_idx = 0
+    for char in original:
+        nfd_char = unicodedata.normalize("NFD", char)
+        orig_to_nfd.append(nfd_idx)
+        nfd_idx += len(nfd_char)
+
+    # Build: NFD index -> normalized char index
+    nfd_to_norm: list[int] = []
+    nfd_idx = 0
+    for norm_idx, char in enumerate(normalized):
+        nfd_char = unicodedata.normalize("NFD", char)
+        for _ in nfd_char:
+            nfd_to_norm.append(norm_idx)
+        nfd_idx += len(nfd_char)
+
+    # Build: normalized index -> original index
+    # For each normalized position, find which original character maps to it
+    norm_to_orig: list[int] = []
+    orig_char_idx = 0
+    orig_nfd_pos = 0
+    for norm_idx in range(len(normalized)):
+        nfd_of_norm_char = unicodedata.normalize("NFD", normalized[norm_idx])
+        nfd_start = sum(
+            len(unicodedata.normalize("NFD", normalized[i]))
+            for i in range(norm_idx)
+        )
+        # Find which original character this NFD position belongs to
+        target_orig = 0
+        running = 0
+        for oi, oc in enumerate(original):
+            nfd_oc = unicodedata.normalize("NFD", oc)
+            if running + len(nfd_oc) > nfd_start:
+                target_orig = oi
+                break
+            running += len(nfd_oc)
+        norm_to_orig.append(target_orig)
+
+    return norm_to_orig
+
+
+def map_offset_to_original(
+    offset: int, mapping: list[int] | None
+) -> int:
+    """Map a normalized-text offset back to an original-text offset."""
+    if mapping is None:
+        return offset
+    if offset >= len(mapping):
+        # Past end of mapping — extrapolate
+        if mapping:
+            return mapping[-1] + (offset - len(mapping) + 1)
+        return offset
+    return mapping[offset]

euredact/rules/__init__.py

@@ -0,0 +1,5 @@
+"""EuRedact rule engine."""
+
+from euredact.rules.engine import RuleEngine
+
+__all__ = ["RuleEngine"]

euredact/rules/countries/__init__.py

@@ -0,0 +1 @@
+"""Country-specific PII pattern configurations."""

euredact/rules/countries/_base.py

@@ -0,0 +1,31 @@
+"""Base country configuration class."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from euredact.types import EntityType
+
+
+@dataclass
+class PatternDef:
+    """A single pattern definition."""
+
+    entity_type: EntityType
+    pattern: str
+    validator: str | None = None
+    description: str = ""
+    context_keywords: list[str] = field(default_factory=list)
+    requires_context: bool = False
+
+
+@dataclass
+class CountryConfig:
+    """Base class for country-specific PII patterns."""
+
+    code: str = ""
+    name: str = ""
+    patterns: list[PatternDef] = field(default_factory=list)
+
+    def __post_init__(self) -> None:
+        """Override in subclasses to populate patterns."""

euredact/rules/countries/_shared.py

@@ -0,0 +1,171 @@
+"""Shared (EU-wide) PII patterns: email, generic IBAN, international phone, dates, digital IDs."""
+
+from __future__ import annotations
+
+from euredact.rules.countries._base import CountryConfig, PatternDef
+from euredact.types import EntityType
+
+
+class SharedConfig(CountryConfig):
+    """Patterns shared across all EU countries."""
+
+    def __post_init__(self) -> None:
+        self.code = "SHARED"
+        self.name = "Shared EU Patterns"
+        self.patterns = [
+            # --- Email ---
+            PatternDef(
+                entity_type=EntityType.EMAIL,
+                pattern=r"\b[\w._%+\-]+@[\w.\-]+\.[a-zA-Z]{2,}\b",
+                validator=None,
+                description="Email address (RFC 5322 simplified)",
+            ),
+            # --- BIC/SWIFT ---
+            PatternDef(
+                entity_type=EntityType.BIC,
+                pattern=r"\b[A-Z]{4}[A-Z]{2}[A-Z0-9]{2}(?:[A-Z0-9]{3})?\b",
+                validator="bic",
+                description="BIC/SWIFT code (8 or 11 characters)",
+            ),
+            # --- Credit Card (Visa, Mastercard, Amex) ---
+            PatternDef(
+                entity_type=EntityType.CREDIT_CARD,
+                pattern=(
+                    r"\b(?:"
+                    r"4[0-9]{3}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}"  # Visa
+                    r"|5[1-5][0-9]{2}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}[\s\-]?[0-9]{4}"  # MC
+                    r"|3[47][0-9]{2}[\s\-]?[0-9]{6}[\s\-]?[0-9]{5}"  # Amex
+                    r")\b"
+                ),
+                validator="luhn",
+                description="Credit card number (Visa, Mastercard, Amex) with Luhn",
+            ),
+            # --- VIN ---
+            PatternDef(
+                entity_type=EntityType.VIN,
+                pattern=r"\b[A-HJ-NPR-Z0-9]{17}\b",
+                validator="vin",
+                description="Vehicle Identification Number (ISO 3779)",
+            ),
+            # --- IPv4 ---
+            PatternDef(
+                entity_type=EntityType.IP_ADDRESS,
+                pattern=(
+                    r"\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}"
+                    r"(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b"
+                ),
+                validator=None,
+                description="IPv4 address",
+            ),
+            # --- IPv6 ---
+            PatternDef(
+                entity_type=EntityType.IPV6_ADDRESS,
+                pattern=(
+                    r"\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b"
+                    r"|\b(?:[0-9a-fA-F]{1,4}:){1,7}:\b"
+                    r"|\b(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}\b"
+                    r"|\b::(?:[0-9a-fA-F]{1,4}:){0,5}[0-9a-fA-F]{1,4}\b"
+                ),
+                validator=None,
+                description="IPv6 address",
+            ),
+            # --- MAC Address (colon/dash: 00:1A:2B:3C:4D:5E) ---
+            PatternDef(
+                entity_type=EntityType.MAC_ADDRESS,
+                pattern=r"\b(?:[0-9a-fA-F]{2}[:\-]){5}[0-9a-fA-F]{2}\b",
+                validator=None,
+                description="MAC address (colon or dash separated)",
+            ),
+            # MAC Address Cisco format (dot: 0670.3A83.C107)
+            PatternDef(
+                entity_type=EntityType.MAC_ADDRESS,
+                pattern=r"\b[0-9a-fA-F]{4}\.[0-9a-fA-F]{4}\.[0-9a-fA-F]{4}\b",
+                validator=None,
+                description="MAC address (Cisco dot notation)",
+            ),
+            # --- IMEI (15 digits, Luhn check) ---
+            # Luhn + TAC validation is strong enough to not require context
+            PatternDef(
+                entity_type=EntityType.IMEI,
+                pattern=r"\b\d{15}\b",
+                validator="imei",
+                description="IMEI — 15 digits with Luhn check",
+            ),
+            # IMEI with separators: XX-XXXXXX-XXXXXX-X
+            PatternDef(
+                entity_type=EntityType.IMEI,
+                pattern=r"\b\d{2}[\-\s]\d{6}[\-\s]\d{6}[\-\s]\d\b",
+                validator="imei",
+                description="IMEI — formatted with separators",
+            ),
+            # --- GPS Coordinates (decimal degrees) ---
+            # Latitude: -90 to 90, Longitude: -180 to 180, at least 4 decimal places
+            PatternDef(
+                entity_type=EntityType.GPS_COORDINATES,
+                pattern=(
+                    r"-?(?:[1-8]?\d(?:\.\d{4,})|90(?:\.0{4,}))"
+                    r"\s*[,;/]\s*"
+                    r"-?(?:1[0-7]\d(?:\.\d{4,})|180(?:\.0{4,})|\d{1,2}(?:\.\d{4,}))"
+                ),
+                validator=None,
+                description="GPS coordinates — decimal degrees (lat, lon) with 4+ decimals",
+            ),
+            # --- UUID (version 1–5, standard 8-4-4-4-12 hex format) ---
+            PatternDef(
+                entity_type=EntityType.UUID,
+                pattern=r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b",
+                validator=None,
+                description="UUID (RFC 4122, versions 1-5)",
+            ),
+            # --- Social Media Handles (@username) ---
+            # Must be preceded by whitespace or start-of-string, 1-30 alphanumeric/underscore/dot chars
+            PatternDef(
+                entity_type=EntityType.SOCIAL_HANDLE,
+                pattern=r"(?<!\w)@[a-zA-Z][a-zA-Z0-9_.]{1,29}\b",
+                validator=None,
+                description="Social media handle (@username)",
+            ),
+            # --- Date of Birth (EU formats, requires context) ---
+            PatternDef(
+                entity_type=EntityType.DOB,
+                pattern=r"\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b",
+                validator=None,
+                description="Date in DD/MM/YYYY format (EU standard) — requires context",
+                context_keywords=[
+                    "geboren", "geboortedatum", "date de naissance", "né le", "née le",
+                    "né(e) le", "nee le", "nee(e) le",
+                    "date of birth", "DOB", "Geburtsdatum", "geboren am", "geboren op",
+                    "nascido", "nacido", "data di nascita", "nato il", "nata il",
+                    "geb.", "geb.datum", "geb ", "birth date", "birthday",
+                    "naissance", "geboorte", "geburtstag",
+                ],
+                requires_context=True,
+            ),
+            # --- Date of Death (requires context) ---
+            PatternDef(
+                entity_type=EntityType.DATE_OF_DEATH,
+                pattern=r"\b(?:0[1-9]|[12][0-9]|3[01])[/.\-](?:0[1-9]|1[0-2])[/.\-](?:19|20)\d{2}\b",
+                validator=None,
+                description="Date of death — requires context",
+                context_keywords=[
+                    "overleden", "overlijdensdatum", "date de décès", "décédé le",
+                    "date of death", "Sterbedatum", "verstorben am", "gestorven",
+                    "death date", "died on", "mort le", "décès",
+                ],
+                requires_context=True,
+            ),
+            # --- ISO date format YYYY-MM-DD (DOB, requires context) ---
+            PatternDef(
+                entity_type=EntityType.DOB,
+                pattern=r"\b(?:19|20)\d{2}[/.\-](?:0[1-9]|1[0-2])[/.\-](?:0[1-9]|[12][0-9]|3[01])\b",
+                validator=None,
+                description="Date in YYYY-MM-DD format (ISO) — requires context",
+                context_keywords=[
+                    "geboren", "geboortedatum", "date de naissance", "né le", "née le",
+                    "né(e) le", "nee le",
+                    "date of birth", "DOB", "Geburtsdatum", "geboren am", "geboren op",
+                    "geb.", "birth date", "birthday", "naissance", "geboorte",
+                ],
+                requires_context=True,
+            ),
+        ]

euredact/rules/countries/at.py

@@ -0,0 +1,82 @@
+"""Austria (AT) PII patterns."""
+
+from __future__ import annotations
+
+from euredact.rules.countries._base import CountryConfig, PatternDef
+from euredact.types import EntityType
+
+
+class ATConfig(CountryConfig):
+    """Austrian PII patterns: SVNR, IBAN, phone, etc."""
+
+    def __post_init__(self) -> None:
+        self.code = "AT"
+        self.name = "Austria"
+        self.patterns = [
+            # --- SVNR (Sozialversicherungsnummer) — 10 digits ---
+            PatternDef(
+                entity_type=EntityType.NATIONAL_ID,
+                pattern=r"\b\d{4}\s?\d{6}\b",
+                validator="austrian_svnr",
+                description="Austrian SVNR — 10 digits with check digit",
+            ),
+            # --- IBAN ---
+            PatternDef(
+                entity_type=EntityType.IBAN,
+                pattern=r"\bAT\d{2}\s?\d{4}\s?\d{4}\s?\d{4}\s?\d{4}\b",
+                validator="iban",
+                description="Austrian IBAN — AT + 18 digits",
+            ),
+            # --- VAT (UID) ---
+            PatternDef(
+                entity_type=EntityType.VAT,
+                pattern=r"\bATU\d{8}\b",
+                validator=None,
+                description="Austrian VAT (UID) — ATU + 8 digits",
+            ),
+            # --- Phone (national) ---
+            PatternDef(
+                entity_type=EntityType.PHONE,
+                pattern=r"\b0\d{3,4}[\s\-]?\d{5,8}\b",
+                validator=None,
+                description="Austrian national phone",
+            ),
+            # --- Phone (international) ---
+            PatternDef(
+                entity_type=EntityType.PHONE,
+                pattern=r"\+43\s?\d{3,4}[\s\-]?\d{5,8}",
+                validator=None,
+                description="Austrian international phone — +43",
+            ),
+            # --- License Plate ---
+            PatternDef(
+                entity_type=EntityType.LICENSE_PLATE,
+                pattern=r"\b[A-ZÄÖÜ]{1,2}\s?\d{1,5}\s?[A-Z]{1,2}\b",
+                validator=None,
+                description="Austrian license plate — district + number + letters",
+                context_keywords=[
+                    "Kennzeichen", "Nummernschild", "Kfz-Kennzeichen",
+                ],
+                requires_context=True,
+            ),
+            # --- Postal Code ---
+            PatternDef(
+                entity_type=EntityType.POSTAL_CODE,
+                pattern=r"\b[1-9]\d{3}\b",
+                validator=None,
+                description="Austrian postal code — 4 digits",
+                context_keywords=[
+                    "PLZ", "Postleitzahl", "Adresse", "Anschrift",
+                    "Straße", "Str.", "Gasse", "Weg", "Platz",
+                    "Postal:", "Wohnort",
+                ],
+                requires_context=True,
+            ),
+            # Structural: "City, XXXX" or ", XXXX City"
+            PatternDef(
+                entity_type=EntityType.POSTAL_CODE,
+                pattern=r"(?<=, )[1-9]\d{3}(?= [A-Z])",
+                validator=None,
+                description="Austrian postal code — in address structure",
+            ),
+        ]