ciphergate-cli 0.7.0__py3-none-any.whl

ciphergate/detectors/registry.py

@@ -0,0 +1,194 @@
+"""Builds the per-run detector set from policy settings (no global mutable registry)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ciphergate.detectors.base import Detector, make_finding
+from ciphergate.detectors.custom_regex import CustomRegexDetector
+from ciphergate.detectors.dates import DateDetector
+from ciphergate.detectors.email import EmailDetector, is_reserved_email_domain
+from ciphergate.detectors.field_names import FieldNameDetector, PolicyClassificationDetector
+from ciphergate.detectors.ip_address import IPAddressDetector
+from ciphergate.detectors.network import InternalHostDetector
+from ciphergate.detectors.payment_card import PaymentCardDetector
+from ciphergate.detectors.phone import PhoneDetector
+from ciphergate.detectors.secrets import PEM_BEGIN_RE, PEM_END_RE, SecretDetector
+from ciphergate.models.finding import Finding, ScanItem
+from ciphergate.models.policy import DetectorSettings, PolicyDocument
+
+BUILTIN_DETECTOR_IDS: tuple[str, ...] = (
+    "field_name",
+    "secret_pattern",
+    "luhn_card",
+    "ip_address",
+    "date_pattern",
+    "email_regex",
+    "internal_host",
+    "phone_pattern",
+)
+
+# Phone matching is the least specific pattern; drop phone spans that overlap these detectors.
+_PHONE_SUPPRESSORS = frozenset({"luhn_card", "ip_address", "date_pattern", "secret_pattern"})
+
+
+FIELD_CONTEXT_DETECTORS = frozenset({"field_name", "policy_classification"})
+
+
+@dataclass(slots=True)
+class TextBlockState:
+    """Cross-line state for text/log input (multi-line PEM private key blocks)."""
+
+    in_private_key_block: bool = False
+
+
+def _overlaps(a: tuple[int, int], b: tuple[int, int]) -> bool:
+    return a[0] < b[1] and b[0] < a[1]
+
+
+class DetectorSet:
+    def __init__(self, settings: DetectorSettings, document: PolicyDocument | None = None) -> None:
+        disabled = set(settings.disabled)
+        detectors: list[Detector] = []
+        factories: dict[str, Detector] = {
+            "field_name": FieldNameDetector(),
+            "secret_pattern": SecretDetector(),
+            "luhn_card": PaymentCardDetector(),
+            "ip_address": IPAddressDetector(),
+            "date_pattern": DateDetector(emit_generic=settings.dates_emit_generic),
+            "email_regex": EmailDetector(),
+            "internal_host": InternalHostDetector(settings.internal_host_suffixes),
+            "phone_pattern": PhoneDetector(),
+        }
+        for detector_id in BUILTIN_DETECTOR_IDS:
+            if detector_id not in disabled:
+                detectors.append(factories[detector_id])
+        if document is not None and document.classifications:
+            detectors.append(PolicyClassificationDetector(document.classifications))
+        detectors.extend(CustomRegexDetector(spec) for spec in settings.custom)
+        if settings.presidio.enabled:
+            # Imported lazily: the base install never imports the optional NER code path.
+            from ciphergate.detectors.presidio_ner import PresidioDetector  # noqa: PLC0415
+
+            detectors.append(PresidioDetector(settings.presidio))
+        self.detectors = tuple(detectors)
+        self.ids = tuple(d.id for d in detectors)
+
+    def scan(self, item: ScanItem) -> list[Finding]:
+        findings: list[Finding] = []
+        for detector in self.detectors:
+            findings.extend(detector.detect(item))
+        return self._post_process(findings)
+
+    def scan_field_context(self, item: ScanItem) -> list[Finding]:
+        """Only the detectors that judge a value by its field name/path (logfmt keys)."""
+        findings: list[Finding] = []
+        for detector in self.detectors:
+            if detector.id in FIELD_CONTEXT_DETECTORS:
+                findings.extend(detector.detect(item))
+        return findings
+
+    def scan_text_line(self, item: ScanItem, state: TextBlockState) -> list[Finding]:
+        """Scan one text/log line, tracking multi-line private key blocks across lines."""
+        text = item.text or ""
+        findings = self.scan(item)
+        if "secret_pattern" not in self.ids:
+            return findings
+        if state.in_private_key_block:
+            findings = [f for f in findings if f.category != "credential.secret"]
+            findings.append(
+                make_finding(
+                    item,
+                    category="credential.secret",
+                    detector="secret_pattern",
+                    confidence=0.99,
+                    evidence=("pattern_match", "family:private_key_block"),
+                    span=(0, len(text)) if text else None,
+                )
+            )
+            if PEM_END_RE.search(text):
+                state.in_private_key_block = False
+            return self._post_process(findings)
+        begins = list(PEM_BEGIN_RE.finditer(text))
+        begin = begins[-1] if begins else None
+        if begin is not None and not PEM_END_RE.search(text, begin.end()):
+            state.in_private_key_block = True
+        return findings
+
+    def post_process(self, findings: list[Finding]) -> list[Finding]:
+        """Phone suppression and de-duplication (used for combined logfmt findings)."""
+        return self._post_process(findings)
+
+    @staticmethod
+    def dedupe(findings: list[Finding]) -> list[Finding]:
+        """Drop exact duplicates (same category, detector and span), keeping order."""
+        out: list[Finding] = []
+        seen: set[tuple[str, str, tuple[int, int] | None]] = set()
+        for f in findings:
+            key = (f.category, f.detector, f.location.span)
+            if key not in seen:
+                seen.add(key)
+                out.append(f)
+        return out
+
+    @staticmethod
+    def _post_process(findings: list[Finding]) -> list[Finding]:
+        if not findings:
+            return findings
+        strong = [
+            f.location.span
+            for f in findings
+            if f.detector in _PHONE_SUPPRESSORS and f.location.span is not None
+        ] + [
+            f.location.span
+            for f in findings
+            if f.category == "pii.email"
+            and f.location.span is not None
+            # Field-context findings cover a whole value; they must not hide a phone inside it.
+            and f.detector not in FIELD_CONTEXT_DETECTORS
+        ]
+        out: list[Finding] = []
+        seen: set[tuple[str, str, tuple[int, int] | None]] = set()
+        for f in findings:
+            span = f.location.span
+            if (
+                f.detector == "phone_pattern"
+                and span is not None
+                and any(_overlaps(span, s) for s in strong)
+            ):
+                continue
+            key = (f.category, f.detector, span)
+            if key in seen:
+                continue
+            seen.add(key)
+            out.append(f)
+        return out
+
+
+def unsafe_replacement_reason(value: str) -> str | None:
+    """Why a configured replacement constant looks like a real identifier or secret, if it does."""
+    probe = ScanItem(record_index=0, line=None, field_path="", field_name=None, value=value)
+    secrets = SecretDetector().detect(probe)
+    if secrets:
+        return f"looks like a credential ({secrets[0].evidence[-1]})"
+    if PaymentCardDetector().detect(probe):
+        return "looks like a payment card number"
+    for f in EmailDetector().detect(probe):
+        if "reserved_domain" not in f.evidence:
+            return "email replacements must use a reserved domain such as example.invalid"
+    for f in IPAddressDetector().detect(probe):
+        if "documentation_range" not in f.evidence:
+            return "IP replacements must use a documentation range (192.0.2.0/24, 2001:db8::/32)"
+    for f in PhoneDetector().detect(probe):
+        if f.confidence >= 0.5:
+            return "looks like a real phone number; use an obviously synthetic value"
+    return None
+
+
+__all__ = [
+    "BUILTIN_DETECTOR_IDS",
+    "DetectorSet",
+    "TextBlockState",
+    "is_reserved_email_domain",
+    "unsafe_replacement_reason",
+]

ciphergate/detectors/secrets.py

@@ -0,0 +1,130 @@
+"""Credential and secret detector.
+
+Pattern families: PEM private keys (including multi-line blocks inside one value), bearer/basic
+authorization values, common token formats (AWS access key IDs, GitHub, Slack, Stripe-style,
+Google API keys, JWTs), URL userinfo passwords, connection-string passwords and assignment-style
+``password=...`` / ``api_key: ...`` values. Spans cover the secret part only, so surrounding debug
+context can be kept. Pattern detection has false negatives (e.g. base64-wrapped, split or
+custom-format secrets); it is never proof that content is credential-free.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from ciphergate.detectors.base import MARKER_TOKEN_RE, PSEUDONYM_TOKEN_RE, make_finding
+from ciphergate.models.finding import Finding, ScanItem
+
+PEM_BEGIN_RE = re.compile(r"-----BEGIN (?:[A-Z0-9]{1,16} ){0,3}PRIVATE KEY(?: BLOCK)?-----")
+PEM_END_RE = re.compile(r"-----END (?:[A-Z0-9]{1,16} ){0,3}PRIVATE KEY(?: BLOCK)?-----")
+
+
+@dataclass(frozen=True, slots=True)
+class _Pattern:
+    name: str
+    regex: re.Pattern[str]
+    group: int
+    confidence: float
+
+
+_PATTERNS: tuple[_Pattern, ...] = (
+    _Pattern("bearer_token", re.compile(r"(?i)\bbearer\s+([A-Za-z0-9\-._~+/]{8,2048}=*)"), 1, 0.9),
+    _Pattern(
+        "basic_auth", re.compile(r"(?i)\bbasic\s+([A-Za-z0-9+/]{8,1024}={0,2})(?![\w])"), 1, 0.8
+    ),
+    _Pattern("aws_access_key_id", re.compile(r"\b((?:AKIA|ASIA)[A-Z0-9]{16})\b"), 1, 0.95),
+    _Pattern("github_token", re.compile(r"\b(gh[pousr]_[A-Za-z0-9]{36,255})\b"), 1, 0.95),
+    _Pattern("github_pat", re.compile(r"\b(github_pat_[A-Za-z0-9_]{22,255})\b"), 1, 0.95),
+    _Pattern("slack_token", re.compile(r"\b(xox[abprs]-[A-Za-z0-9-]{10,200})"), 1, 0.95),
+    _Pattern(
+        "stripe_style_key", re.compile(r"\b([sr]k_(?:live|test)_[A-Za-z0-9]{16,128})\b"), 1, 0.9
+    ),
+    _Pattern("google_api_key", re.compile(r"\b(AIza[0-9A-Za-z\-_]{35})(?![\w-])"), 1, 0.9),
+    _Pattern(
+        "jwt",
+        re.compile(
+            r"\b(eyJ[A-Za-z0-9_-]{5,2048}\.eyJ[A-Za-z0-9_-]{5,4096}\.[A-Za-z0-9_-]{5,2048})"
+        ),
+        1,
+        0.85,
+    ),
+    _Pattern(
+        "url_userinfo_password",
+        re.compile(r"\b[a-zA-Z][a-zA-Z0-9+.-]{1,20}://[^\s:/@]{1,128}:([^\s@/]{1,256})@"),
+        1,
+        0.9,
+    ),
+    _Pattern(
+        "connection_string_password",
+        re.compile(r"(?i)(?<![a-z0-9_])(?:password|pwd)\s*=\s*([^;\s\"']{1,256})"),
+        1,
+        0.85,
+    ),
+    _Pattern(
+        "assignment",
+        re.compile(
+            r"(?i)(?<![a-z0-9])"
+            r"(?:password|passwd|passphrase|secret|api[_-]?key|apikey|access[_-]?token|"
+            r"refresh[_-]?token|auth[_-]?token|client[_-]?secret|secret[_-]?key|private[_-]?key|"
+            r"access[_-]?key|token|credentials?)"
+            r"(?:\\?[\"'])?\s{0,3}[:=]\s{0,3}(?:\\?[\"'])?"
+            r"([^\s\"'\\,;&}]{1,512})"
+        ),
+        1,
+        0.8,
+    ),
+)
+
+_PLACEHOLDER_VALUES = frozenset(
+    {"", "null", "none", "nil", "undefined", "true", "false", "redacted", "***", "******", "xxx"}
+)
+
+
+def _is_placeholder(value: str) -> bool:
+    v = value.strip().lower()
+    return (
+        v in _PLACEHOLDER_VALUES
+        or bool(MARKER_TOKEN_RE.fullmatch(value))
+        or bool(PSEUDONYM_TOKEN_RE.fullmatch(value))
+        or set(v) <= {"*"}
+    )
+
+
+class SecretDetector:
+    id = "secret_pattern"
+
+    def detect(self, item: ScanItem) -> list[Finding]:
+        text = item.text
+        if not text or len(text) < 6:
+            return []
+        out: list[Finding] = []
+        covered: list[tuple[int, int]] = []
+        for begin in PEM_BEGIN_RE.finditer(text):
+            closing = PEM_END_RE.search(text, begin.end())
+            span = (begin.start(), closing.end() if closing else len(text))
+            covered.append(span)
+            out.append(self._finding(item, span, "private_key_header", 0.99))
+        for pattern in _PATTERNS:
+            for m in pattern.regex.finditer(text):
+                start, end = m.start(pattern.group), m.end(pattern.group)
+                value = m.group(pattern.group)
+                if _is_placeholder(value):
+                    continue
+                if any(s <= start and end <= e for s, e in covered):
+                    continue
+                covered.append((start, end))
+                out.append(self._finding(item, (start, end), pattern.name, pattern.confidence))
+        return out
+
+    def _finding(
+        self, item: ScanItem, span: tuple[int, int], family: str, confidence: float
+    ) -> Finding:
+        return make_finding(
+            item,
+            category="credential.secret",
+            detector=self.id,
+            confidence=confidence,
+            evidence=("pattern_match", f"family:{family}"),
+            span=span,
+        )

ciphergate/errors.py

@@ -0,0 +1,119 @@
+"""Exception hierarchy with stable error codes and safe messages.
+
+Messages are always authored by CipherGate and must never contain input values, matched text,
+key material, or third-party parser excerpts. Callers chain the original exception with
+``raise ... from exc`` only when it is needed for debugging; the CLI never renders chained messages.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ciphergate.constants import ExitCode
+
+
+class CipherGateError(Exception):
+    """Base class. ``code`` is a stable identifier; ``message`` is safe to display."""
+
+    default_code = "ciphergate_error"
+    exit_code: ExitCode = ExitCode.INTERNAL
+
+    def __init__(self, message: str, *, code: str | None = None) -> None:
+        super().__init__(message)
+        self.code = code or self.default_code
+        self.message = message
+
+    def __str__(self) -> str:
+        return f"{self.code}: {self.message}"
+
+
+class UsageError(CipherGateError):
+    default_code = "usage_error"
+    exit_code = ExitCode.USAGE
+
+
+@dataclass(frozen=True, slots=True)
+class ValidationIssue:
+    """A single policy validation problem: a location in the policy plus a safe message."""
+
+    location: str
+    message: str
+    code: str = "invalid_value"
+
+    def render(self) -> str:
+        return f"{self.location}: {self.message} [{self.code}]"
+
+
+class PolicyValidationError(CipherGateError):
+    default_code = "policy_invalid"
+    exit_code = ExitCode.POLICY_INVALID
+
+    def __init__(
+        self, message: str, *, issues: list[ValidationIssue] | None = None, code: str | None = None
+    ) -> None:
+        super().__init__(message, code=code)
+        self.issues = list(issues or [])
+
+
+class ContextError(CipherGateError):
+    """Required context is missing, unsupported, or conflicts with a policy-pinned value."""
+
+    default_code = "context_error"
+    exit_code = ExitCode.BLOCKED
+
+
+class InputFormatError(CipherGateError):
+    default_code = "input_format_error"
+    exit_code = ExitCode.INPUT_ERROR
+
+
+class InputLimitError(CipherGateError):
+    default_code = "size_limit_exceeded"
+    exit_code = ExitCode.INPUT_ERROR
+
+
+class DetectionError(CipherGateError):
+    default_code = "detection_error"
+    exit_code = ExitCode.INPUT_ERROR
+
+
+class DetectorUnavailableError(CipherGateError):
+    """An optional detector is enabled by the policy but cannot be used (never skipped)."""
+
+    default_code = "detector_unavailable"
+    exit_code = ExitCode.KEY_OR_TRANSFORM_CONFIG
+
+
+class UnsafePathError(CipherGateError):
+    default_code = "unsafe_path"
+    exit_code = ExitCode.OUTPUT_ERROR
+
+
+class OutputWriteError(CipherGateError):
+    default_code = "output_write_error"
+    exit_code = ExitCode.OUTPUT_ERROR
+
+
+class MissingKeyError(CipherGateError):
+    default_code = "key_missing"
+    exit_code = ExitCode.KEY_OR_TRANSFORM_CONFIG
+
+
+class TransformError(CipherGateError):
+    default_code = "transform_error"
+    exit_code = ExitCode.KEY_OR_TRANSFORM_CONFIG
+
+
+class PolicyBlockedError(CipherGateError):
+    default_code = "policy_blocked"
+    exit_code = ExitCode.BLOCKED
+
+
+class RescanFailedError(CipherGateError):
+    default_code = "rescan_failed"
+    exit_code = ExitCode.RESCAN_FAILED
+
+
+class InternalError(CipherGateError):
+    default_code = "internal_error"
+    exit_code = ExitCode.INTERNAL

ciphergate/models/__init__.py

@@ -0,0 +1 @@
+"""Typed data models: findings, context, policy schema, reports."""

ciphergate/models/context.py

@@ -0,0 +1,69 @@
+"""Run context: source environment, target environment, destination, profile, use case."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Literal
+
+from ciphergate.constants import (
+    UNTRUSTED_DESTINATIONS,
+    Destination,
+    DomainProfile,
+    Environment,
+    TargetEnvironment,
+    UseCase,
+)
+
+ContextField = Literal[
+    "environment", "target_environment", "destination", "domain_profile", "use_case"
+]
+CONTEXT_FIELDS: tuple[ContextField, ...] = (
+    "environment",
+    "target_environment",
+    "destination",
+    "domain_profile",
+    "use_case",
+)
+ContextSource = Literal["policy", "cli", "cli_override", "unset"]
+
+
+@dataclass(frozen=True, slots=True)
+class ContextInput:
+    """Context supplied by the caller (CLI flags or API). ``None`` means not supplied."""
+
+    environment: Environment | None = None
+    target_environment: TargetEnvironment | None = None
+    destination: Destination | None = None
+    domain_profile: DomainProfile | None = None
+    use_case: UseCase | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class EffectiveContext:
+    """Context after combining policy pins with caller input. ``None`` means missing."""
+
+    environment: Environment | None = None
+    target_environment: TargetEnvironment | None = None
+    destination: Destination | None = None
+    domain_profile: DomainProfile | None = None
+    use_case: UseCase | None = None
+    sources: dict[str, ContextSource] = field(default_factory=dict)
+
+    @property
+    def is_untrusted(self) -> bool:
+        """True unless the recipient is known and internal. A missing destination is untrusted."""
+        return (
+            self.destination is None
+            or self.destination in UNTRUSTED_DESTINATIONS
+            or self.target_environment in (TargetEnvironment.EXTERNAL, TargetEnvironment.UNKNOWN)
+        )
+
+    def get(self, name: ContextField) -> str | None:
+        value = getattr(self, name)
+        return None if value is None else str(value)
+
+    def to_report(self) -> dict[str, dict[str, str | None]]:
+        return {
+            name: {"value": self.get(name), "source": self.sources.get(name, "unset")}
+            for name in CONTEXT_FIELDS
+        }

ciphergate/models/finding.py

@@ -0,0 +1,108 @@
+"""Detection results. A ``Finding`` never carries the matched value, only where and why."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from ciphergate.constants import confidence_band
+
+
+@dataclass(frozen=True, slots=True)
+class Location:
+    """Structural locator. ``span`` is a character offset range inside the scanned value/line."""
+
+    record_index: int
+    line: int | None
+    field_path: str
+    column_index: int | None = None
+    span: tuple[int, int] | None = None
+    in_key: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        data: dict[str, Any] = {"record": self.record_index, "line": self.line}
+        if self.field_path:
+            data["field_path"] = self.field_path
+        if self.column_index is not None:
+            data["column"] = self.column_index
+        if self.span is not None:
+            data["span"] = [self.span[0], self.span[1]]
+        if self.in_key:
+            data["in_key"] = True
+        return data
+
+
+@dataclass(frozen=True, slots=True)
+class Finding:
+    category: str
+    detector: str
+    confidence: float
+    evidence: tuple[str, ...]
+    location: Location
+    field_name: str | None
+    value_type: str
+
+    @property
+    def band(self) -> str:
+        return confidence_band(self.confidence)
+
+    @property
+    def whole_value(self) -> bool:
+        return self.location.span is None
+
+
+def value_type_name(value: Any) -> str:
+    """Basic JSON-ish type name. Never the value itself."""
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "boolean"
+    if isinstance(value, int):
+        return "integer"
+    if isinstance(value, float):
+        return "float"
+    if isinstance(value, str):
+        return "string"
+    if isinstance(value, list):
+        return "array"
+    if isinstance(value, dict):
+        return "object"
+    return "other"
+
+
+@dataclass(frozen=True, slots=True)
+class ScanItem:
+    """One scalar value (or JSON key) handed to detectors. Held in memory only during a scan."""
+
+    record_index: int
+    line: int | None
+    field_path: str
+    field_name: str | None
+    value: Any
+    column_index: int | None = None
+    in_key: bool = False
+
+    @property
+    def text(self) -> str | None:
+        """Text form used by content detectors. Booleans and nulls carry no scannable content."""
+        if self.value is None or isinstance(self.value, bool):
+            return None
+        if isinstance(self.value, str):
+            return self.value
+        if isinstance(self.value, int | float):
+            return repr(self.value) if isinstance(self.value, float) else str(self.value)
+        return None
+
+    @property
+    def is_empty(self) -> bool:
+        return self.value is None or self.value == ""
+
+    def location(self, span: tuple[int, int] | None = None) -> Location:
+        return Location(
+            record_index=self.record_index,
+            line=self.line,
+            field_path=self.field_path,
+            column_index=self.column_index,
+            span=span,
+            in_key=self.in_key,
+        )