sether 0.1.0__py3-none-any.whl

sether/__init__.py

@@ -0,0 +1,148 @@
+"""Sether -- streaming PII redaction for AI applications.
+
+Hide personal data from your AI before it reaches any LLM provider. Sether
+detects sensitive data, swaps each match for a stable token before the request
+leaves your boundary, then restores the originals transparently in the response.
+
+    from sether import Sether
+
+    sether = Sether()
+    safe = sether.redact_sync("my email is alice@example.com")
+    # -> "my email is <EMAIL_...>"
+    back = sether.restore_sync(safe)
+    # -> "my email is alice@example.com"
+
+Named for the Hebrew *sether* -- *the hiding place*. Psalm 32:7.
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from .audit import (
+    DEFAULT_REGULATION_MAPPINGS,
+    AuditEvent,
+    AuditSink,
+    ConsoleAuditSink,
+    MemoryAuditSink,
+    RegulationMapping,
+)
+from .core import Sether
+from .detectors import (
+    Detector,
+    DetectorMatch,
+    address_detector,
+    anthropic_key_detector,
+    aws_access_key_detector,
+    basic_detectors,
+    credit_card_detector,
+    dob_detector,
+    email_detector,
+    github_pat_detector,
+    high_entropy_detector,
+    iban_detector,
+    identity_detectors,
+    ipv4_detector,
+    ipv6_detector,
+    jwt_detector,
+    name_detector,
+    openai_key_detector,
+    passport_detector,
+    phone_detector,
+    secrets_detectors,
+    slack_token_detector,
+    ssn_detector,
+    stripe_key_detector,
+)
+from .middleware import (
+    SetherASGIMiddleware,
+    SetherWSGIMiddleware,
+    wrap_anthropic,
+    wrap_httpx,
+    wrap_openai,
+)
+from .stream import (
+    RedactStream,
+    RestoreStream,
+    SSEStream,
+    aredact_iter,
+    arestore_iter,
+    asse_redact_iter,
+    asse_restore_iter,
+    create_redact_stream,
+    create_restore_stream,
+    create_sse_redact_stream,
+    create_sse_restore_stream,
+    redact_iter,
+    redact_sync,
+    restore_iter,
+    restore_sync,
+    sse_redact_iter,
+    sse_restore_iter,
+)
+from .vault import MemoryVault, Vault
+
+__all__ = [
+    "__version__",
+    # core
+    "Sether",
+    "MemoryVault",
+    "Vault",
+    # one-shot + streaming
+    "redact_sync",
+    "restore_sync",
+    "RedactStream",
+    "RestoreStream",
+    "create_redact_stream",
+    "create_restore_stream",
+    "redact_iter",
+    "aredact_iter",
+    "restore_iter",
+    "arestore_iter",
+    # detectors
+    "Detector",
+    "DetectorMatch",
+    "basic_detectors",
+    "email_detector",
+    "credit_card_detector",
+    "ssn_detector",
+    "ipv4_detector",
+    "ipv6_detector",
+    "iban_detector",
+    "phone_detector",
+    "secrets_detectors",
+    "aws_access_key_detector",
+    "openai_key_detector",
+    "anthropic_key_detector",
+    "github_pat_detector",
+    "slack_token_detector",
+    "stripe_key_detector",
+    "jwt_detector",
+    "high_entropy_detector",
+    "identity_detectors",
+    "name_detector",
+    "dob_detector",
+    "passport_detector",
+    "address_detector",
+    # SSE
+    "SSEStream",
+    "create_sse_redact_stream",
+    "create_sse_restore_stream",
+    "sse_redact_iter",
+    "sse_restore_iter",
+    "asse_redact_iter",
+    "asse_restore_iter",
+    # audit
+    "AuditEvent",
+    "AuditSink",
+    "RegulationMapping",
+    "DEFAULT_REGULATION_MAPPINGS",
+    "ConsoleAuditSink",
+    "MemoryAuditSink",
+    # middleware
+    "wrap_openai",
+    "wrap_anthropic",
+    "wrap_httpx",
+    "SetherASGIMiddleware",
+    "SetherWSGIMiddleware",
+]

sether/audit/__init__.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from .sinks import ConsoleAuditSink, MemoryAuditSink
+from .types import (
+    DEFAULT_REGULATION_MAPPINGS,
+    FRAMEWORKS,
+    AuditEvent,
+    AuditSink,
+    RegulationMapping,
+)
+
+__all__ = [
+    "AuditEvent",
+    "AuditSink",
+    "RegulationMapping",
+    "FRAMEWORKS",
+    "DEFAULT_REGULATION_MAPPINGS",
+    "ConsoleAuditSink",
+    "MemoryAuditSink",
+]

sether/audit/sinks.py

@@ -0,0 +1,48 @@
+"""Reference audit sinks.
+
+Direct port of ``src/audit/console.ts``. Sinks are intentionally simple -- one
+method. Anything more complex (batching, retries, structured forwarding) lives
+in your own adapter.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import Callable, List, Optional
+
+from .types import AuditEvent
+
+
+class ConsoleAuditSink:
+    """Writes one JSON line per event to stderr by default. A template for
+    production sinks (Datadog, Splunk, Logpush, R2, ...)."""
+
+    def __init__(
+        self,
+        write: Optional[Callable[[str], object]] = None,
+        pretty: bool = False,
+    ) -> None:
+        self._write: Callable[[str], object] = write or (lambda line: sys.stderr.write(line))
+        self._pretty = pretty
+
+    def write(self, event: AuditEvent) -> None:
+        data = event.to_dict()
+        text = json.dumps(data, indent=2, ensure_ascii=False) if self._pretty else json.dumps(data, ensure_ascii=False)
+        self._write(text + "\n")
+
+
+class MemoryAuditSink:
+    """Accumulates events in memory. Used by tests and the in-browser sandbox."""
+
+    def __init__(self) -> None:
+        self.events: List[AuditEvent] = []
+
+    def write(self, event: AuditEvent) -> None:
+        self.events.append(event)
+
+    def clear(self) -> None:
+        self.events.clear()
+
+
+__all__ = ["ConsoleAuditSink", "MemoryAuditSink"]

sether/audit/types.py

@@ -0,0 +1,104 @@
+"""Audit event schema.
+
+Direct port of ``src/audit/types.ts``. Every redaction can emit one
+:class:`AuditEvent`. The OSS package provides the type contract and reference
+sinks; the hosted tier ingests these events for retention, compliance
+reporting, and SIEM export. The JSON wire shape is the same on both sides
+(camelCase keys), so a deployment can promote from local-only to hosted without
+reshaping events.
+
+**The original value is never carried in an event -- only its length.**
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Protocol, Tuple, runtime_checkable
+
+# Allowed compliance frameworks (informational -- not enforced at runtime).
+FRAMEWORKS = (
+    "GDPR", "SOC2", "HIPAA", "EU_AI_ACT", "NDPA", "CCPA",
+    "PCI_DSS", "ISO_27001", "OWASP_LLM", "OWASP_AGENTIC",
+)
+
+
+@dataclass(frozen=True)
+class RegulationMapping:
+    framework: str  # one of FRAMEWORKS
+    reference: str  # e.g. "Art. 28", "CC6.7", "164.312"
+
+    def to_dict(self) -> Dict[str, str]:
+        return {"framework": self.framework, "reference": self.reference}
+
+
+@dataclass
+class AuditEvent:
+    """One redaction (or restoration) event."""
+
+    timestamp: str  # ISO-8601 UTC
+    detector: str  # e.g. "EMAIL" or "OPENAI_KEY"
+    value_length: int  # length of the original value (never the value itself)
+    token: str  # the token that replaced the value
+    action: str = "REDACTED"  # "REDACTED" | "RESTORED"
+    tenant_id: Optional[str] = None
+    request_id: Optional[str] = None
+    destination: Optional[str] = None
+    regulation_mappings: Optional[List[RegulationMapping]] = None
+
+    def to_dict(self) -> Dict[str, object]:
+        """Serialise to the camelCase JSON wire shape, omitting unset optionals."""
+        out: Dict[str, object] = {
+            "timestamp": self.timestamp,
+            "detector": self.detector,
+            "valueLength": self.value_length,
+            "token": self.token,
+            "action": self.action,
+        }
+        if self.tenant_id is not None:
+            out["tenantId"] = self.tenant_id
+        if self.request_id is not None:
+            out["requestId"] = self.request_id
+        if self.destination is not None:
+            out["destination"] = self.destination
+        if self.regulation_mappings is not None:
+            out["regulationMappings"] = [m.to_dict() for m in self.regulation_mappings]
+        return out
+
+
+@runtime_checkable
+class AuditSink(Protocol):
+    def write(self, event: AuditEvent) -> None: ...
+
+
+def _m(*pairs: Tuple[str, str]) -> List[RegulationMapping]:
+    return [RegulationMapping(framework, reference) for framework, reference in pairs]
+
+
+# Default regulation mappings per detector type. Each entry is defensible
+# against the actual regulation text.
+DEFAULT_REGULATION_MAPPINGS: Dict[str, List[RegulationMapping]] = {
+    "EMAIL": _m(("GDPR", "Art. 28"), ("SOC2", "CC6.7")),
+    "PHONE": _m(("GDPR", "Art. 28"), ("SOC2", "CC6.7")),
+    "CC": _m(("PCI_DSS", "Req. 3.4"), ("SOC2", "CC6.7")),
+    "SSN": _m(("GDPR", "Art. 9"), ("HIPAA", "164.514")),
+    "IPV4": _m(("GDPR", "Recital 30")),
+    "IPV6": _m(("GDPR", "Recital 30")),
+    "IBAN": _m(("PCI_DSS", "Req. 3.4"), ("GDPR", "Art. 28")),
+    "AWS_KEY": _m(("SOC2", "CC6.1"), ("ISO_27001", "A.9.4.3")),
+    "OPENAI_KEY": _m(("SOC2", "CC6.1")),
+    "ANTHROPIC_KEY": _m(("SOC2", "CC6.1")),
+    "GITHUB_PAT": _m(("SOC2", "CC6.1")),
+    "SLACK_TOKEN": _m(("SOC2", "CC6.1")),
+    "STRIPE_KEY": _m(("PCI_DSS", "Req. 3.5"), ("SOC2", "CC6.1")),
+    "JWT": _m(("SOC2", "CC6.1")),
+    "HIGH_ENTROPY": _m(("SOC2", "CC6.1")),
+}
+
+
+__all__ = [
+    "FRAMEWORKS",
+    "RegulationMapping",
+    "AuditEvent",
+    "AuditSink",
+    "DEFAULT_REGULATION_MAPPINGS",
+]

sether/core.py

@@ -0,0 +1,97 @@
+"""The :class:`Sether` facade -- a shared vault wiring redact and restore."""
+
+from __future__ import annotations
+
+from typing import AsyncIterable, AsyncIterator, Iterable, Iterator, Optional, Sequence
+
+from .detectors.basic import basic_detectors
+from .detectors.types import Detector
+from .stream.redact import (
+    RedactStream,
+    aredact_iter,
+    redact_iter,
+    redact_sync,
+)
+from .stream.restore import (
+    RestoreStream,
+    arestore_iter,
+    restore_iter,
+    restore_sync,
+)
+from .vault.memory import MemoryVault
+from .vault.types import Vault
+
+_DEFAULT_SAFE_DISTANCE = 256
+
+
+class Sether:
+    """Streaming PII redaction layer.
+
+    The same instance shares its vault between redaction and restoration, which
+    is how the redact->restore round-trip identity is preserved across streaming
+    chunks.
+
+    :param detectors: detectors to run. Defaults to ``basic_detectors``.
+    :param vault: token vault. Defaults to a fresh :class:`MemoryVault`.
+    :param safe_distance_bytes: bytes held back at each chunk tail so a pattern
+        crossing a chunk boundary is still detected (default 256).
+    """
+
+    def __init__(
+        self,
+        detectors: Optional[Sequence[Detector]] = None,
+        vault: Optional[Vault] = None,
+        safe_distance_bytes: int = _DEFAULT_SAFE_DISTANCE,
+    ) -> None:
+        self._detectors: Sequence[Detector] = tuple(detectors) if detectors is not None else basic_detectors
+        self._vault: Vault = vault if vault is not None else MemoryVault()
+        self._safe_distance = safe_distance_bytes
+
+    # --- one-shot ---------------------------------------------------------
+
+    def redact_sync(self, text: str) -> str:
+        return redact_sync(text, self._detectors, self._vault)
+
+    def restore_sync(self, text: str) -> str:
+        return restore_sync(text, self._vault)
+
+    # --- synchronous streaming -------------------------------------------
+
+    def redact_stream(self, chunks: Iterable[str]) -> Iterator[str]:
+        return redact_iter(chunks, self._detectors, self._vault, self._safe_distance)
+
+    def restore_stream(self, chunks: Iterable[str]) -> Iterator[str]:
+        return restore_iter(chunks, self._vault)
+
+    # --- asynchronous streaming ------------------------------------------
+
+    def aredact_stream(self, chunks: AsyncIterable[str]) -> AsyncIterator[str]:
+        return aredact_iter(chunks, self._detectors, self._vault, self._safe_distance)
+
+    def arestore_stream(self, chunks: AsyncIterable[str]) -> AsyncIterator[str]:
+        return arestore_iter(chunks, self._vault)
+
+    # --- low-level stateful transforms -----------------------------------
+
+    def new_redact_stream(self) -> RedactStream:
+        return RedactStream(self._detectors, self._vault, self._safe_distance)
+
+    def new_restore_stream(self) -> RestoreStream:
+        return RestoreStream(self._vault)
+
+    # --- accessors --------------------------------------------------------
+
+    @property
+    def vault(self) -> Vault:
+        return self._vault
+
+    @property
+    def detectors(self) -> Sequence[Detector]:
+        return self._detectors
+
+    @property
+    def safe_distance_bytes(self) -> int:
+        return self._safe_distance
+
+
+__all__ = ["Sether"]

sether/detectors/__init__.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from .basic import (
+    basic_detectors,
+    credit_card_detector,
+    email_detector,
+    iban_detector,
+    ipv4_detector,
+    ipv6_detector,
+    is_ipv6_address,
+    phone_detector,
+    ssn_detector,
+)
+from .identity import (
+    address_detector,
+    dob_detector,
+    identity_detectors,
+    name_detector,
+    passport_detector,
+)
+from .secrets import (
+    anthropic_key_detector,
+    aws_access_key_detector,
+    github_pat_detector,
+    high_entropy_detector,
+    jwt_detector,
+    openai_key_detector,
+    secrets_detectors,
+    slack_token_detector,
+    stripe_key_detector,
+)
+from .types import Detector, DetectorMatch
+
+__all__ = [
+    "Detector",
+    "DetectorMatch",
+    # basic
+    "basic_detectors",
+    "email_detector",
+    "credit_card_detector",
+    "ssn_detector",
+    "ipv4_detector",
+    "ipv6_detector",
+    "iban_detector",
+    "phone_detector",
+    "is_ipv6_address",
+    # secrets
+    "secrets_detectors",
+    "aws_access_key_detector",
+    "openai_key_detector",
+    "anthropic_key_detector",
+    "github_pat_detector",
+    "slack_token_detector",
+    "stripe_key_detector",
+    "jwt_detector",
+    "high_entropy_detector",
+    # identity
+    "identity_detectors",
+    "name_detector",
+    "dob_detector",
+    "passport_detector",
+    "address_detector",
+]