pseudonymize 0.1.0a1__py3-none-any.whl

pseudonymize/__init__.py

@@ -0,0 +1,28 @@
+from pseudonymize.api import generate_key, pseudonymize, redact
+from pseudonymize.backends import CompositeBackend, DetectionBackend, RulesBackend
+from pseudonymize.engine import ProcessingScope, Pseudonymizer
+from pseudonymize.policy import Policy
+from pseudonymize.resolution import EntityResolver, ExactEntityResolver, ResolvedEntity
+from pseudonymize.result import Detection, EntityType, Replacement, Result
+from pseudonymize.transforms import Alias, TransformationMode
+
+__all__ = [
+    "Alias",
+    "CompositeBackend",
+    "Detection",
+    "DetectionBackend",
+    "EntityResolver",
+    "EntityType",
+    "ExactEntityResolver",
+    "Policy",
+    "ProcessingScope",
+    "Pseudonymizer",
+    "Replacement",
+    "ResolvedEntity",
+    "Result",
+    "RulesBackend",
+    "TransformationMode",
+    "generate_key",
+    "pseudonymize",
+    "redact",
+]

pseudonymize/api.py

@@ -0,0 +1,50 @@
+from collections.abc import Sequence
+
+from pseudonymize.backends import DetectionBackend
+from pseudonymize.engine import Pseudonymizer
+from pseudonymize.policy import Policy
+from pseudonymize.transforms import TransformationMode, generate_key
+
+
+def pseudonymize(
+    text: str,
+    *,
+    mode: TransformationMode | str = TransformationMode.NUMBERED,
+    key: bytes | None = None,
+    namespace: str = "default",
+    policy: Policy | None = None,
+    backends: Sequence[DetectionBackend] | None = None,
+) -> str:
+    return (
+        Pseudonymizer(
+            mode=mode,
+            key=key,
+            namespace=namespace,
+            policy=policy,
+            backends=backends,
+        )
+        .process(text)
+        .text
+    )
+
+
+def redact(
+    text: str,
+    *,
+    typed: bool = False,
+    policy: Policy | None = None,
+    backends: Sequence[DetectionBackend] | None = None,
+) -> str:
+    return (
+        Pseudonymizer(
+            mode=TransformationMode.REDACTED,
+            policy=policy,
+            backends=backends,
+            typed_redaction=typed,
+        )
+        .process(text)
+        .text
+    )
+
+
+__all__ = ["generate_key", "pseudonymize", "redact"]

pseudonymize/backends/__init__.py

@@ -0,0 +1,5 @@
+from pseudonymize.backends.base import DetectionBackend, EntityBackend
+from pseudonymize.backends.composite import CompositeBackend
+from pseudonymize.backends.rules import RulesBackend
+
+__all__ = ["CompositeBackend", "DetectionBackend", "EntityBackend", "RulesBackend"]

pseudonymize/backends/base.py

@@ -0,0 +1,14 @@
+from collections.abc import Sequence
+from typing import Protocol
+
+from pseudonymize.result import Detection
+
+
+class DetectionBackend(Protocol):
+    @property
+    def name(self) -> str: ...
+
+    def detect(self, text: str) -> Sequence[Detection]: ...
+
+
+EntityBackend = DetectionBackend

pseudonymize/backends/composite.py

@@ -0,0 +1,17 @@
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from pseudonymize.backends.base import DetectionBackend
+from pseudonymize.result import Detection
+
+
+@dataclass(frozen=True, slots=True)
+class CompositeBackend:
+    backends: Sequence[DetectionBackend]
+    name: str = "composite"
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "backends", tuple(self.backends))
+
+    def detect(self, text: str) -> tuple[Detection, ...]:
+        return tuple(detection for backend in self.backends for detection in backend.detect(text))

pseudonymize/backends/rules.py

@@ -0,0 +1,19 @@
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from pseudonymize.detectors import DEFAULT_DETECTORS, Detector
+from pseudonymize.result import Detection
+
+
+@dataclass(frozen=True, slots=True)
+class RulesBackend:
+    detectors: Sequence[Detector] = DEFAULT_DETECTORS
+    name: str = "rules"
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "detectors", tuple(self.detectors))
+
+    def detect(self, text: str) -> tuple[Detection, ...]:
+        return tuple(
+            detection for detector in self.detectors for detection in detector.detect(text)
+        )

pseudonymize/cli.py

@@ -0,0 +1,125 @@
+import argparse
+import base64
+import binascii
+import json
+import os
+import stat
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from pseudonymize.api import generate_key
+from pseudonymize.detectors import DEFAULT_DETECTORS
+from pseudonymize.engine import Data, Pseudonymizer
+from pseudonymize.exceptions import PseudonymizeError
+from pseudonymize.policy import Policy
+from pseudonymize.transforms import TransformationMode
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="pseudonymize")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    subparsers.add_parser("keygen", help="print a new base64-encoded key")
+    subparsers.add_parser("detectors", help="list bundled detectors")
+
+    text_parser = subparsers.add_parser("text", help="process text")
+    text_parser.add_argument("text", help="text to process, or - for standard input")
+    _add_key_options(text_parser, allow_stdin=True)
+    text_parser.add_argument("--namespace", default="default")
+    _add_mode_options(text_parser)
+
+    json_parser = subparsers.add_parser("json", help="process JSON from standard input")
+    _add_key_options(json_parser)
+    json_parser.add_argument("--namespace", default="default")
+    _add_mode_options(json_parser)
+    return parser
+
+
+def _add_key_options(parser: argparse.ArgumentParser, *, allow_stdin: bool = False) -> None:
+    sources = parser.add_mutually_exclusive_group()
+    sources.add_argument("--key-env", metavar="NAME")
+    sources.add_argument("--key-file", type=Path)
+    sources.add_argument("--key-fd", type=int)
+    if allow_stdin:
+        sources.add_argument("--key-stdin", action="store_true")
+
+
+def _add_mode_options(parser: argparse.ArgumentParser) -> None:
+    modes = parser.add_mutually_exclusive_group()
+    modes.add_argument(
+        "--mode", choices=tuple(TransformationMode), default=TransformationMode.NUMBERED
+    )
+    modes.add_argument(
+        "--redact", action="store_const", const=TransformationMode.REDACTED, dest="mode"
+    )
+    parser.add_argument("--typed-redaction", action="store_true")
+
+
+def _has_key_source(arguments: argparse.Namespace) -> bool:
+    return any(
+        (
+            getattr(arguments, "key_env", None),
+            getattr(arguments, "key_file", None),
+            getattr(arguments, "key_fd", None) is not None,
+            getattr(arguments, "key_stdin", False),
+        )
+    )
+
+
+def _read_key(arguments: argparse.Namespace) -> bytes:
+    encoded: str
+    if getattr(arguments, "key_env", None):
+        encoded = os.environ.get(arguments.key_env, "")
+        if not encoded:
+            raise ValueError(f"environment variable {arguments.key_env!r} is empty or unset")
+    elif getattr(arguments, "key_file", None):
+        path: Path = arguments.key_file
+        if os.name != "nt" and stat.S_IMODE(path.stat().st_mode) & 0o077:
+            raise ValueError("key file must not be accessible by group or other users")
+        encoded = path.read_text(encoding="ascii").strip()
+    elif getattr(arguments, "key_fd", None) is not None:
+        with os.fdopen(os.dup(arguments.key_fd), encoding="ascii") as stream:
+            encoded = stream.read().strip()
+    else:
+        if arguments.text == "-":
+            raise ValueError("standard input cannot provide both the key and text")
+        encoded = sys.stdin.read().strip()
+    try:
+        return base64.b64decode(encoded, validate=True)
+    except (binascii.Error, ValueError) as error:
+        raise ValueError("key must be valid base64") from error
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = _parser()
+    arguments = parser.parse_args(argv)
+    if arguments.command == "keygen":
+        print(base64.b64encode(generate_key()).decode("ascii"))
+        return 0
+    if arguments.command == "detectors":
+        print("\n".join(detector.name for detector in DEFAULT_DETECTORS))
+        return 0
+    try:
+        key = _read_key(arguments) if _has_key_source(arguments) else None
+        engine = Pseudonymizer(
+            mode=arguments.mode,
+            key=key,
+            namespace=arguments.namespace,
+            policy=Policy.llm() if arguments.command == "json" else Policy.default(),
+            typed_redaction=arguments.typed_redaction,
+        )
+        if arguments.command == "text":
+            source = sys.stdin.read() if arguments.text == "-" else arguments.text
+            print(engine.process(source).text)
+            return 0
+        payload = json.load(sys.stdin)
+        output: Data = engine.process_data(payload)
+        json.dump(output, sys.stdout, ensure_ascii=False)
+        sys.stdout.write("\n")
+        return 0
+    except (OSError, ValueError, json.JSONDecodeError, PseudonymizeError) as error:
+        parser.exit(2, f"pseudonymize: error: {error}\n")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pseudonymize/detectors/__init__.py

@@ -0,0 +1,4 @@
+from pseudonymize.detectors.base import Detector
+from pseudonymize.detectors.registry import DEFAULT_DETECTORS
+
+__all__ = ["DEFAULT_DETECTORS", "Detector"]

pseudonymize/detectors/base.py

@@ -0,0 +1,10 @@
+from typing import Protocol
+
+from pseudonymize.result import Detection
+
+
+class Detector(Protocol):
+    @property
+    def name(self) -> str: ...
+
+    def detect(self, text: str) -> list[Detection]: ...

pseudonymize/detectors/email.py

@@ -0,0 +1,21 @@
+import re
+from dataclasses import dataclass
+
+from pseudonymize.result import Detection, EntityType
+
+_EMAIL = re.compile(
+    r"(?<![\w.+-])[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+@"
+    r"(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,63}(?![\w-])"
+)
+
+
+@dataclass(frozen=True, slots=True)
+class EmailDetector:
+    name: str = "email"
+
+    def detect(self, text: str) -> list[Detection]:
+        return [
+            Detection(EntityType.EMAIL, match.start(), match.end(), 0.99, self.name)
+            for match in _EMAIL.finditer(text)
+            if len(match.group().partition("@")[0]) <= 64 and len(match.group()) <= 254
+        ]

pseudonymize/detectors/iban.py

@@ -0,0 +1,31 @@
+import re
+from dataclasses import dataclass
+
+from pseudonymize.result import Detection, EntityType
+
+_IBAN = re.compile(r"(?<![A-Z0-9])[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]){11,30}(?![A-Z0-9])", re.I)
+
+
+def _valid_mod97(value: str) -> bool:
+    compact = "".join(value.split()).upper()
+    if not 15 <= len(compact) <= 34:
+        return False
+    rearranged = compact[4:] + compact[:4]
+    remainder = 0
+    for character in rearranged:
+        digits = str(ord(character) - 55) if character.isalpha() else character
+        for digit in digits:
+            remainder = (remainder * 10 + int(digit)) % 97
+    return remainder == 1
+
+
+@dataclass(frozen=True, slots=True)
+class IbanDetector:
+    name: str = "iban"
+
+    def detect(self, text: str) -> list[Detection]:
+        return [
+            Detection(EntityType.IBAN, match.start(), match.end(), 1.0, self.name)
+            for match in _IBAN.finditer(text)
+            if _valid_mod97(match.group())
+        ]

pseudonymize/detectors/ip_address.py

@@ -0,0 +1,26 @@
+import ipaddress
+import re
+from dataclasses import dataclass
+
+from pseudonymize.result import Detection, EntityType
+
+_IP_CANDIDATE = re.compile(
+    r"(?<![\w:.])(?:\d{1,3}(?:\.\d{1,3}){3}|[0-9A-Fa-f]{0,4}:[0-9A-Fa-f:]+)(?![\w:.])"
+)
+
+
+@dataclass(frozen=True, slots=True)
+class IpAddressDetector:
+    name: str = "ip_address"
+
+    def detect(self, text: str) -> list[Detection]:
+        detections: list[Detection] = []
+        for match in _IP_CANDIDATE.finditer(text):
+            try:
+                ipaddress.ip_address(match.group())
+            except ValueError:
+                continue
+            detections.append(
+                Detection(EntityType.IP_ADDRESS, match.start(), match.end(), 0.99, self.name)
+            )
+        return detections

pseudonymize/detectors/payment_card.py

@@ -0,0 +1,33 @@
+import re
+from dataclasses import dataclass
+
+from pseudonymize.result import Detection, EntityType
+
+_CARD = re.compile(r"(?<!\d)(?:\d[ -]?){12,18}\d(?!\d)")
+
+
+def _valid_luhn(value: str) -> bool:
+    digits = [int(character) for character in value if character.isdigit()]
+    if not 13 <= len(digits) <= 19 or len(set(digits)) == 1:
+        return False
+    total = 0
+    parity = len(digits) % 2
+    for index, digit in enumerate(digits):
+        if index % 2 == parity:
+            digit *= 2
+            if digit > 9:
+                digit -= 9
+        total += digit
+    return total % 10 == 0
+
+
+@dataclass(frozen=True, slots=True)
+class PaymentCardDetector:
+    name: str = "payment_card"
+
+    def detect(self, text: str) -> list[Detection]:
+        return [
+            Detection(EntityType.PAYMENT_CARD, match.start(), match.end(), 1.0, self.name)
+            for match in _CARD.finditer(text)
+            if _valid_luhn(match.group())
+        ]

pseudonymize/detectors/phone.py

@@ -0,0 +1,26 @@
+import re
+from dataclasses import dataclass
+
+from pseudonymize.result import Detection, EntityType
+
+_PHONE = re.compile(r"(?<![\w+])(?:\+\d{1,3}[ .()-]*)?(?:\d[ .()-]*){6,14}\d(?!\w)")
+
+
+def _credible_phone(value: str) -> bool:
+    digits = "".join(character for character in value if character.isdigit())
+    if not 7 <= len(digits) <= 15 or len(set(digits)) == 1:
+        return False
+    separators = sum(character in " .()-" for character in value)
+    return value.startswith("+") or separators >= 2
+
+
+@dataclass(frozen=True, slots=True)
+class PhoneDetector:
+    name: str = "phone"
+
+    def detect(self, text: str) -> list[Detection]:
+        return [
+            Detection(EntityType.PHONE, match.start(), match.end(), 0.86, self.name)
+            for match in _PHONE.finditer(text)
+            if _credible_phone(match.group())
+        ]

pseudonymize/detectors/registry.py

@@ -0,0 +1,18 @@
+from pseudonymize.detectors.base import Detector
+from pseudonymize.detectors.email import EmailDetector
+from pseudonymize.detectors.iban import IbanDetector
+from pseudonymize.detectors.ip_address import IpAddressDetector
+from pseudonymize.detectors.payment_card import PaymentCardDetector
+from pseudonymize.detectors.phone import PhoneDetector
+from pseudonymize.detectors.secret import SecretDetector
+from pseudonymize.detectors.url import UrlDetector
+
+DEFAULT_DETECTORS: tuple[Detector, ...] = (
+    EmailDetector(),
+    IpAddressDetector(),
+    PaymentCardDetector(),
+    IbanDetector(),
+    PhoneDetector(),
+    UrlDetector(),
+    SecretDetector(),
+)

pseudonymize/detectors/secret.py

@@ -0,0 +1,31 @@
+import re
+from dataclasses import dataclass
+
+from pseudonymize.result import Detection, EntityType
+
+_SECRET_PATTERNS = (
+    re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
+    re.compile(r"\bgh[pousr]_[A-Za-z0-9]{36,255}\b"),
+    re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"),
+    re.compile(
+        r"(?i)\b(?:api[_-]?key|client[_-]?secret|password|secret|token)\b\s*[:=]\s*"
+        r"(?P<value>['\"]?[A-Za-z0-9_./+=-]{8,}['\"]?)"
+    ),
+)
+
+
+@dataclass(frozen=True, slots=True)
+class SecretDetector:
+    name: str = "secret"
+
+    def detect(self, text: str) -> list[Detection]:
+        detections: list[Detection] = []
+        for pattern in _SECRET_PATTERNS:
+            for match in pattern.finditer(text):
+                start, end = match.span("value") if "value" in pattern.groupindex else match.span()
+                if text[start : start + 1] in {'"', "'"}:
+                    start += 1
+                if text[end - 1 : end] in {'"', "'"}:
+                    end -= 1
+                detections.append(Detection(EntityType.SECRET, start, end, 0.96, self.name))
+        return detections

pseudonymize/detectors/url.py

@@ -0,0 +1,53 @@
+import re
+import urllib.parse
+from dataclasses import dataclass
+
+from pseudonymize.result import Detection, EntityType
+
+_URL = re.compile(r"https?://[^\s<>\"']+", re.I)
+_SENSITIVE_QUERY_NAMES = frozenset(
+    {"access_token", "api_key", "apikey", "auth", "key", "password", "secret", "token"}
+)
+
+
+@dataclass(frozen=True, slots=True)
+class UrlDetector:
+    name: str = "url"
+
+    def detect(self, text: str) -> list[Detection]:
+        detections: list[Detection] = []
+        for match in _URL.finditer(text):
+            url = match.group().rstrip(".,;:!?)]}")
+            parsed = urllib.parse.urlsplit(url)
+            if parsed.username is not None or parsed.password is not None:
+                authority_start = url.find("//") + 2
+                at = url.find("@", authority_start)
+                detections.append(
+                    Detection(
+                        EntityType.URL_CREDENTIAL,
+                        match.start() + authority_start,
+                        match.start() + at,
+                        1.0,
+                        self.name,
+                    )
+                )
+            query_start = url.find("?") + 1
+            if query_start == 0:
+                continue
+            cursor = query_start
+            for item in url[query_start:].split("&"):
+                name, separator, value = item.partition("=")
+                if separator and urllib.parse.unquote_plus(name).lower() in _SENSITIVE_QUERY_NAMES:
+                    value_start = cursor + len(name) + 1
+                    if value:
+                        detections.append(
+                            Detection(
+                                EntityType.URL_CREDENTIAL,
+                                match.start() + value_start,
+                                match.start() + value_start + len(value),
+                                0.98,
+                                self.name,
+                            )
+                        )
+                cursor += len(item) + 1
+        return detections