pseudonymize 0.1.0__py3-none-any.whl

pseudonymize/__init__.py

@@ -0,0 +1,66 @@
+from pseudonymize.adapters import InputAdapter, OutputAdapter
+from pseudonymize.api import generate_key, pseudonymize, redact
+from pseudonymize.backends import (
+    BackendCapabilities,
+    CompositeBackend,
+    DetectionBackend,
+    RulesBackend,
+)
+from pseudonymize.document import (
+    ContentBlock,
+    CSVCellLocation,
+    Document,
+    JSONPathLocation,
+    MetadataValue,
+    SourceLocation,
+    TextOffsetLocation,
+)
+from pseudonymize.engine import ProcessingScope, Pseudonymizer
+from pseudonymize.formats import FileFormat
+from pseudonymize.policy import NetworkPolicy, Policy
+from pseudonymize.processing import (
+    DetectionReport,
+    ProcessingResult,
+    ProcessingStatistics,
+    ProcessingWarning,
+)
+from pseudonymize.resolution import EntityResolver, ExactEntityResolver, ResolvedEntity
+from pseudonymize.result import Detection, EntityType, Replacement, Result
+from pseudonymize.transforms import Alias, TransformationMode
+
+__all__ = [
+    "Alias",
+    "BackendCapabilities",
+    "CSVCellLocation",
+    "CompositeBackend",
+    "ContentBlock",
+    "Detection",
+    "DetectionBackend",
+    "DetectionReport",
+    "Document",
+    "EntityResolver",
+    "EntityType",
+    "ExactEntityResolver",
+    "FileFormat",
+    "InputAdapter",
+    "JSONPathLocation",
+    "MetadataValue",
+    "NetworkPolicy",
+    "OutputAdapter",
+    "Policy",
+    "ProcessingResult",
+    "ProcessingScope",
+    "ProcessingStatistics",
+    "ProcessingWarning",
+    "Pseudonymizer",
+    "Replacement",
+    "ResolvedEntity",
+    "Result",
+    "RulesBackend",
+    "SourceLocation",
+    "TextOffsetLocation",
+    "TransformationMode",
+    "generate_key",
+    "pseudonymize",
+    "redact",
+]

pseudonymize/adapters.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+from typing import Protocol, TypeVar
+
+from pseudonymize.document import Document
+
+SourceT = TypeVar("SourceT", contravariant=True)
+
+
+class InputAdapter(Protocol[SourceT]):
+    def extract(self, source: SourceT) -> Document: ...
+
+
+class OutputAdapter(Protocol):
+    def render(self, document: Document) -> bytes: ...
+
+
+FileInputAdapter = InputAdapter[Path]

pseudonymize/api.py

@@ -0,0 +1,50 @@
+from collections.abc import Sequence
+
+from pseudonymize.backends import DetectionBackend
+from pseudonymize.engine import Pseudonymizer
+from pseudonymize.policy import Policy
+from pseudonymize.transforms import TransformationMode, generate_key
+
+
+def pseudonymize(
+    text: str,
+    *,
+    mode: TransformationMode | str = TransformationMode.NUMBERED,
+    key: bytes | None = None,
+    namespace: str = "default",
+    policy: Policy | None = None,
+    backends: Sequence[DetectionBackend] | None = None,
+) -> str:
+    return (
+        Pseudonymizer(
+            mode=mode,
+            key=key,
+            namespace=namespace,
+            policy=policy,
+            backends=backends,
+        )
+        .process(text)
+        .text
+    )
+
+
+def redact(
+    text: str,
+    *,
+    typed: bool = False,
+    policy: Policy | None = None,
+    backends: Sequence[DetectionBackend] | None = None,
+) -> str:
+    return (
+        Pseudonymizer(
+            mode=TransformationMode.REDACTED,
+            policy=policy,
+            backends=backends,
+            typed_redaction=typed,
+        )
+        .process(text)
+        .text
+    )
+
+
+__all__ = ["generate_key", "pseudonymize", "redact"]

pseudonymize/backends/__init__.py

@@ -0,0 +1,16 @@
+from pseudonymize.backends.base import (
+    BackendCapabilities,
+    DetectionBackend,
+    backend_capabilities,
+)
+from pseudonymize.backends.composite import CompositeBackend, leaf_backends
+from pseudonymize.backends.rules import RulesBackend
+
+__all__ = [
+    "BackendCapabilities",
+    "CompositeBackend",
+    "DetectionBackend",
+    "RulesBackend",
+    "backend_capabilities",
+    "leaf_backends",
+]

pseudonymize/backends/base.py

@@ -0,0 +1,90 @@
+from collections.abc import Sequence
+from dataclasses import dataclass, replace
+from typing import Protocol
+
+from pseudonymize.document import ContentBlock
+from pseudonymize.exceptions import (
+    BackendContractError,
+    BackendExecutionError,
+    InvalidDetectionError,
+    NetworkPolicyError,
+)
+from pseudonymize.policy import NetworkPolicy, Policy
+from pseudonymize.result import Detection, EntityType
+
+
+@dataclass(frozen=True, slots=True)
+class BackendCapabilities:
+    entity_types: frozenset[EntityType]
+    remote: bool = False
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "entity_types", frozenset(self.entity_types))
+        if any(not isinstance(entity_type, EntityType) for entity_type in self.entity_types):
+            raise TypeError("backend entity types must be EntityType values")
+        if not isinstance(self.remote, bool):
+            raise TypeError("backend remote capability must be a boolean")
+
+
+class DetectionBackend(Protocol):
+    @property
+    def name(self) -> str: ...
+
+    @property
+    def capabilities(self) -> BackendCapabilities: ...
+
+    @property
+    def allow_remote_processing(self) -> bool: ...
+
+    def detect(self, block: ContentBlock, policy: Policy) -> Sequence[Detection]: ...
+
+
+def backend_capabilities(backend: DetectionBackend) -> BackendCapabilities:
+    try:
+        capabilities = backend.capabilities
+        allow_remote_processing = backend.allow_remote_processing
+        name = backend.name
+    except Exception:
+        raise BackendContractError(
+            "backend does not declare the required block-aware contract"
+        ) from None
+    if not isinstance(capabilities, BackendCapabilities):
+        raise BackendContractError("backend capabilities are invalid")
+    if not isinstance(allow_remote_processing, bool):
+        raise BackendContractError("backend remote consent must be a boolean")
+    if not isinstance(name, str) or not name:
+        raise BackendContractError("backend name must be a non-empty string")
+    return capabilities
+
+
+def invoke_backend(
+    backend: DetectionBackend, block: ContentBlock, policy: Policy
+) -> tuple[Detection, ...]:
+    capabilities = backend_capabilities(backend)
+    name = backend.name
+    if capabilities.remote:
+        if policy.network_policy is NetworkPolicy.DENY:
+            raise NetworkPolicyError("network policy denies remote processing")
+        if not backend.allow_remote_processing:
+            raise NetworkPolicyError("remote backend lacks explicit consent")
+        if (
+            policy.network_policy is NetworkPolicy.ALLOW_CONFIGURED
+            and name not in policy.allowed_remote_backends
+        ):
+            raise NetworkPolicyError("remote backend is not allowlisted")
+    try:
+        candidates = tuple(backend.detect(block, policy))
+    except TypeError:
+        raise BackendContractError("backend does not implement block-aware detection") from None
+    except Exception:
+        raise BackendExecutionError("backend failed during detection") from None
+    detections: list[Detection] = []
+    for detection in candidates:
+        if not isinstance(detection, Detection):
+            raise BackendContractError("backend returned a value that is not a Detection")
+        if detection.entity_type not in capabilities.entity_types:
+            raise BackendContractError("backend returned an undeclared entity type")
+        if detection.end > len(block.text):
+            raise InvalidDetectionError("backend returned offsets outside the content block")
+        detections.append(detection if detection.backend else replace(detection, backend=name))
+    return tuple(detections)

pseudonymize/backends/composite.py

@@ -0,0 +1,54 @@
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from pseudonymize.backends.base import (
+    BackendCapabilities,
+    DetectionBackend,
+    backend_capabilities,
+    invoke_backend,
+)
+from pseudonymize.document import ContentBlock
+from pseudonymize.policy import Policy
+from pseudonymize.result import Detection
+from pseudonymize.spans import resolve_overlaps
+
+
+@dataclass(frozen=True, slots=True)
+class CompositeBackend:
+    backends: Sequence[DetectionBackend]
+    name: str = "composite"
+    allow_remote_processing: bool = False
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "backends", tuple(self.backends))
+
+    @property
+    def capabilities(self) -> BackendCapabilities:
+        return BackendCapabilities(
+            frozenset(
+                entity_type
+                for backend in self.backends
+                for entity_type in backend_capabilities(backend).entity_types
+            ),
+            any(backend_capabilities(backend).remote for backend in self.backends),
+        )
+
+    def detect(self, block: ContentBlock, policy: Policy) -> tuple[Detection, ...]:
+        candidates = (
+            detection
+            for backend in self.backends
+            for detection in invoke_backend(backend, block, policy)
+            if detection.entity_type in policy.entity_types
+            and detection.confidence >= policy.minimum_confidence
+        )
+        return resolve_overlaps(candidates, policy.detector_priority)
+
+
+def leaf_backends(backends: Sequence[DetectionBackend]) -> tuple[DetectionBackend, ...]:
+    leaves: list[DetectionBackend] = []
+    for backend in backends:
+        if isinstance(backend, CompositeBackend):
+            leaves.extend(leaf_backends(backend.backends))
+        else:
+            leaves.append(backend)
+    return tuple(leaves)

pseudonymize/backends/rules.py

@@ -0,0 +1,43 @@
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from pseudonymize.backends.base import BackendCapabilities
+from pseudonymize.detectors import DEFAULT_DETECTORS, Detector
+from pseudonymize.document import ContentBlock
+from pseudonymize.policy import Policy
+from pseudonymize.result import Detection, EntityType
+
+_RULE_ENTITY_TYPES = frozenset(
+    {
+        EntityType.EMAIL,
+        EntityType.PHONE,
+        EntityType.IP_ADDRESS,
+        EntityType.IBAN,
+        EntityType.PAYMENT_CARD,
+        EntityType.URL_CREDENTIAL,
+        EntityType.SECRET,
+    }
+)
+
+
+@dataclass(frozen=True, slots=True)
+class RulesBackend:
+    detectors: Sequence[Detector] = DEFAULT_DETECTORS
+    name: str = "rules"
+    allow_remote_processing: bool = False
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "detectors", tuple(self.detectors))
+
+    @property
+    def capabilities(self) -> BackendCapabilities:
+        return BackendCapabilities(_RULE_ENTITY_TYPES)
+
+    def detect(self, block: ContentBlock, policy: Policy) -> tuple[Detection, ...]:
+        return tuple(
+            detection
+            for detector in self.detectors
+            for detection in detector.detect(block.text)
+            if detection.entity_type in policy.entity_types
+            and detection.confidence >= policy.minimum_confidence
+        )

pseudonymize/cli.py

@@ -0,0 +1,206 @@
+import argparse
+import base64
+import binascii
+import json
+import os
+import stat
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from pseudonymize.api import generate_key
+from pseudonymize.detectors import DEFAULT_DETECTORS
+from pseudonymize.document import CSVCellLocation, JSONPathLocation, TextOffsetLocation
+from pseudonymize.engine import Data, Pseudonymizer
+from pseudonymize.exceptions import PseudonymizeError
+from pseudonymize.formats import FileFormat
+from pseudonymize.policy import Policy
+from pseudonymize.processing import DetectionReport, ProcessingResult
+from pseudonymize.transforms import TransformationMode
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="pseudonymize")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    subparsers.add_parser("keygen", help="print a new base64-encoded key")
+    subparsers.add_parser("detectors", help="list bundled detectors")
+
+    text_parser = subparsers.add_parser("text", help="process text")
+    text_parser.add_argument("text", help="text to process, or - for standard input")
+    _add_key_options(text_parser, allow_stdin=True)
+    text_parser.add_argument("--namespace", default="default")
+    _add_mode_options(text_parser)
+
+    json_parser = subparsers.add_parser("json", help="process JSON from standard input")
+    _add_key_options(json_parser)
+    json_parser.add_argument("--namespace", default="default")
+    _add_mode_options(json_parser)
+
+    file_parser = subparsers.add_parser("file", help="process a supported file")
+    file_parser.add_argument("source", type=Path)
+    file_parser.add_argument("--output", type=Path)
+    file_parser.add_argument("--format", choices=tuple(FileFormat))
+    file_parser.add_argument("--encoding")
+    file_parser.add_argument("--overwrite", action="store_true")
+    _add_key_options(file_parser, allow_stdin=True)
+    file_parser.add_argument("--namespace", default="default")
+    _add_mode_options(file_parser)
+
+    inspect_parser = subparsers.add_parser(
+        "inspect-file", help="inspect a supported file without writing output"
+    )
+    inspect_parser.add_argument("source", type=Path)
+    inspect_parser.add_argument("--format", choices=tuple(FileFormat))
+    inspect_parser.add_argument("--encoding")
+    return parser
+
+
+def _add_key_options(parser: argparse.ArgumentParser, *, allow_stdin: bool = False) -> None:
+    sources = parser.add_mutually_exclusive_group()
+    sources.add_argument("--key-env", metavar="NAME")
+    sources.add_argument("--key-file", type=Path)
+    sources.add_argument("--key-fd", type=int)
+    if allow_stdin:
+        sources.add_argument("--key-stdin", action="store_true")
+
+
+def _add_mode_options(parser: argparse.ArgumentParser) -> None:
+    modes = parser.add_mutually_exclusive_group()
+    modes.add_argument(
+        "--mode", choices=tuple(TransformationMode), default=TransformationMode.NUMBERED
+    )
+    modes.add_argument(
+        "--redact", action="store_const", const=TransformationMode.REDACTED, dest="mode"
+    )
+    parser.add_argument("--typed-redaction", action="store_true")
+
+
+def _has_key_source(arguments: argparse.Namespace) -> bool:
+    return any(
+        (
+            getattr(arguments, "key_env", None),
+            getattr(arguments, "key_file", None),
+            getattr(arguments, "key_fd", None) is not None,
+            getattr(arguments, "key_stdin", False),
+        )
+    )
+
+
+def _read_key(arguments: argparse.Namespace) -> bytes:
+    encoded: str
+    if getattr(arguments, "key_env", None):
+        encoded = os.environ.get(arguments.key_env, "")
+        if not encoded:
+            raise ValueError(f"environment variable {arguments.key_env!r} is empty or unset")
+    elif getattr(arguments, "key_file", None):
+        path: Path = arguments.key_file
+        if os.name != "nt" and stat.S_IMODE(path.stat().st_mode) & 0o077:
+            raise ValueError("key file must not be accessible by group or other users")
+        encoded = path.read_text(encoding="ascii").strip()
+    elif getattr(arguments, "key_fd", None) is not None:
+        with os.fdopen(os.dup(arguments.key_fd), encoding="ascii") as stream:
+            encoded = stream.read().strip()
+    else:
+        if getattr(arguments, "text", None) == "-":
+            raise ValueError("standard input cannot provide both the key and text")
+        encoded = sys.stdin.read().strip()
+    try:
+        return base64.b64decode(encoded, validate=True)
+    except (binascii.Error, ValueError) as error:
+        raise ValueError("key must be valid base64") from error
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = _parser()
+    arguments = parser.parse_args(argv)
+    if arguments.command == "keygen":
+        print(base64.b64encode(generate_key()).decode("ascii"))
+        return 0
+    if arguments.command == "detectors":
+        print("\n".join(detector.name for detector in DEFAULT_DETECTORS))
+        return 0
+    try:
+        if arguments.command == "inspect-file":
+            inspected = Pseudonymizer().inspect_file(
+                arguments.source,
+                format=arguments.format,
+                encoding=arguments.encoding,
+            )
+            json.dump(_inspection_payload(inspected), sys.stdout, ensure_ascii=False)
+            sys.stdout.write("\n")
+            return 0
+        key = _read_key(arguments) if _has_key_source(arguments) else None
+        engine = Pseudonymizer(
+            mode=arguments.mode,
+            key=key,
+            namespace=arguments.namespace,
+            policy=Policy.llm() if arguments.command == "json" else Policy.default(),
+            typed_redaction=arguments.typed_redaction,
+        )
+        if arguments.command == "text":
+            source = sys.stdin.read() if arguments.text == "-" else arguments.text
+            print(engine.process(source).text)
+            return 0
+        if arguments.command == "file":
+            result = engine.process_file(
+                arguments.source,
+                arguments.output,
+                format=arguments.format,
+                encoding=arguments.encoding,
+                overwrite=arguments.overwrite,
+            )
+            print(result.output)
+            return 0
+        payload = json.load(sys.stdin)
+        output: Data = engine.process_data(payload)
+        json.dump(output, sys.stdout, ensure_ascii=False)
+        sys.stdout.write("\n")
+        return 0
+    except (LookupError, OSError, ValueError, json.JSONDecodeError, PseudonymizeError) as error:
+        parser.exit(2, f"pseudonymize: error: {error}\n")
+
+
+def _inspection_payload(result: ProcessingResult[None]) -> dict[str, object]:
+    return {
+        "detections": [_detection_payload(report) for report in result.detections],
+        "statistics": {
+            "blocks_processed": result.statistics.blocks_processed,
+            "detections_found": result.statistics.detections_found,
+            "replacements_applied": result.statistics.replacements_applied,
+            "backend_invocations": result.statistics.backend_invocations,
+            "local_block_calls": result.statistics.local_block_calls,
+            "remote_block_calls": result.statistics.remote_block_calls,
+        },
+        "warnings": [
+            {"code": warning.code, "message": warning.message, "block_id": warning.block_id}
+            for warning in result.warnings
+        ],
+    }
+
+
+def _detection_payload(report: DetectionReport) -> dict[str, object]:
+    return {
+        "entity_type": report.entity_type.value,
+        "block_id": report.block_id,
+        "location": _location_payload(report),
+        "start": report.start,
+        "end": report.end,
+        "confidence": report.confidence,
+        "backend": report.backend,
+        "detector": report.detector,
+    }
+
+
+def _location_payload(report: DetectionReport) -> dict[str, object]:
+    location = report.location
+    if isinstance(location, TextOffsetLocation):
+        return {"kind": "text_offset", "start": location.start, "end": location.end}
+    if isinstance(location, JSONPathLocation):
+        return {"kind": "json_path", "path": location.path}
+    if isinstance(location, CSVCellLocation):
+        return {"kind": "csv_cell", "row": location.row, "column": location.column}
+    raise TypeError("unsupported report location")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pseudonymize/detectors/__init__.py

@@ -0,0 +1,4 @@
+from pseudonymize.detectors.base import Detector
+from pseudonymize.detectors.registry import DEFAULT_DETECTORS
+
+__all__ = ["DEFAULT_DETECTORS", "Detector"]

pseudonymize/detectors/base.py

@@ -0,0 +1,10 @@
+from typing import Protocol
+
+from pseudonymize.result import Detection
+
+
+class Detector(Protocol):
+    @property
+    def name(self) -> str: ...
+
+    def detect(self, text: str) -> list[Detection]: ...

pseudonymize/detectors/email.py

@@ -0,0 +1,21 @@
+import re
+from dataclasses import dataclass
+
+from pseudonymize.result import Detection, EntityType
+
+_EMAIL = re.compile(
+    r"(?<![\w.+-])[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+@"
+    r"(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,63}(?![\w-])"
+)
+
+
+@dataclass(frozen=True, slots=True)
+class EmailDetector:
+    name: str = "email"
+
+    def detect(self, text: str) -> list[Detection]:
+        return [
+            Detection(EntityType.EMAIL, match.start(), match.end(), 0.99, self.name)
+            for match in _EMAIL.finditer(text)
+            if len(match.group().partition("@")[0]) <= 64 and len(match.group()) <= 254
+        ]

pseudonymize/detectors/iban.py

@@ -0,0 +1,31 @@
+import re
+from dataclasses import dataclass
+
+from pseudonymize.result import Detection, EntityType
+
+_IBAN = re.compile(r"(?<![A-Z0-9])[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]){11,30}(?![A-Z0-9])", re.I)
+
+
+def _valid_mod97(value: str) -> bool:
+    compact = "".join(value.split()).upper()
+    if not 15 <= len(compact) <= 34:
+        return False
+    rearranged = compact[4:] + compact[:4]
+    remainder = 0
+    for character in rearranged:
+        digits = str(ord(character) - 55) if character.isalpha() else character
+        for digit in digits:
+            remainder = (remainder * 10 + int(digit)) % 97
+    return remainder == 1
+
+
+@dataclass(frozen=True, slots=True)
+class IbanDetector:
+    name: str = "iban"
+
+    def detect(self, text: str) -> list[Detection]:
+        return [
+            Detection(EntityType.IBAN, match.start(), match.end(), 1.0, self.name)
+            for match in _IBAN.finditer(text)
+            if _valid_mod97(match.group())
+        ]

pseudonymize/detectors/ip_address.py

@@ -0,0 +1,27 @@
+import ipaddress
+import re
+from dataclasses import dataclass
+
+from pseudonymize.result import Detection, EntityType
+
+_IP_CANDIDATE = re.compile(
+    r"(?<![\w:.])(?:\d{1,3}(?:\.\d{1,3}){3}|[0-9A-Fa-f]{0,4}:[0-9A-Fa-f:]+)"
+    r"(?![\w:]|\.[\w:])"
+)
+
+
+@dataclass(frozen=True, slots=True)
+class IpAddressDetector:
+    name: str = "ip_address"
+
+    def detect(self, text: str) -> list[Detection]:
+        detections: list[Detection] = []
+        for match in _IP_CANDIDATE.finditer(text):
+            try:
+                ipaddress.ip_address(match.group())
+            except ValueError:
+                continue
+            detections.append(
+                Detection(EntityType.IP_ADDRESS, match.start(), match.end(), 0.99, self.name)
+            )
+        return detections

pseudonymize/detectors/payment_card.py

@@ -0,0 +1,33 @@
+import re
+from dataclasses import dataclass
+
+from pseudonymize.result import Detection, EntityType
+
+_CARD = re.compile(r"(?<!\d)(?:\d[ -]?){12,18}\d(?!\d)")
+
+
+def _valid_luhn(value: str) -> bool:
+    digits = [int(character) for character in value if character.isdigit()]
+    if not 13 <= len(digits) <= 19 or len(set(digits)) == 1:
+        return False
+    total = 0
+    parity = len(digits) % 2
+    for index, digit in enumerate(digits):
+        if index % 2 == parity:
+            digit *= 2
+            if digit > 9:
+                digit -= 9
+        total += digit
+    return total % 10 == 0
+
+
+@dataclass(frozen=True, slots=True)
+class PaymentCardDetector:
+    name: str = "payment_card"
+
+    def detect(self, text: str) -> list[Detection]:
+        return [
+            Detection(EntityType.PAYMENT_CARD, match.start(), match.end(), 1.0, self.name)
+            for match in _CARD.finditer(text)
+            if _valid_luhn(match.group())
+        ]