ciphergate-cli 0.7.0__py3-none-any.whl

ciphergate/config.py

@@ -0,0 +1,50 @@
+"""Operational runtime limits. Data-handling decisions live in the policy, never here."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field, replace
+from typing import Literal
+
+PathDisplay = Literal["basename", "redacted", "full"]
+
+MIB = 1024 * 1024
+
+
+@dataclass(frozen=True, slots=True)
+class RuntimeSettings:
+    """Resource limits and presentation options (units: bytes unless stated).
+
+    Every limit fails the run when exceeded; input is never silently truncated.
+    """
+
+    max_input_bytes: int = 100 * MIB
+    max_json_document_bytes: int = 32 * MIB
+    max_record_bytes: int = 1 * MIB
+    max_json_depth: int = 64
+    max_findings: int = 200_000
+    max_report_items: int = 1_000
+    max_batch_files: int = 1_000
+    encoding: str = "utf-8"
+    csv_delimiter: str = ","
+    path_display: PathDisplay = "basename"
+    # Stored for completeness of the run report; never changes decisions.
+    extra: dict[str, str] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        for name in (
+            "max_input_bytes",
+            "max_json_document_bytes",
+            "max_record_bytes",
+            "max_json_depth",
+            "max_findings",
+            "max_report_items",
+            "max_batch_files",
+        ):
+            if getattr(self, name) <= 0:
+                raise ValueError(f"{name} must be a positive integer")
+        if len(self.csv_delimiter) != 1 or self.csv_delimiter in '\r\n"':
+            raise ValueError("csv_delimiter must be a single character other than quote or newline")
+
+    def with_overrides(self, **changes: object) -> RuntimeSettings:
+        clean = {k: v for k, v in changes.items() if v is not None}
+        return replace(self, **clean)  # type: ignore[arg-type]

ciphergate/constants.py

@@ -0,0 +1,274 @@
+"""Stable identifiers shared across CipherGate: context values, categories, actions, codes."""
+
+from __future__ import annotations
+
+from enum import IntEnum, StrEnum
+
+
+class Environment(StrEnum):
+    """Where the source data originated (policy context only; never verified)."""
+
+    DEV = "dev"
+    STG = "stg"
+    PRD = "prd"
+    UNKNOWN = "unknown"
+
+
+class TargetEnvironment(StrEnum):
+    """Where the output will be used, when that is an application environment."""
+
+    DEV = "dev"
+    STG = "stg"
+    PRD = "prd"
+    EXTERNAL = "external"
+    UNKNOWN = "unknown"
+
+
+class Destination(StrEnum):
+    """The actual intended recipient/channel of the output."""
+
+    LOCAL_DEV = "local_dev"
+    APPROVED_INTERNAL = "approved_internal"
+    STAGING_TEST = "staging_test"
+    PRODUCTION_MONITORING = "production_monitoring"
+    EXTERNAL_VENDOR = "external_vendor"
+    EXTERNAL_AI = "external_ai"
+    PUBLIC_REPOSITORY = "public_repository"
+    UNKNOWN = "unknown"
+
+
+# Destinations that receive data outside the organisation's control, or whose trust is unknown.
+UNTRUSTED_DESTINATIONS: frozenset[Destination] = frozenset(
+    {
+        Destination.EXTERNAL_VENDOR,
+        Destination.EXTERNAL_AI,
+        Destination.PUBLIC_REPOSITORY,
+        Destination.UNKNOWN,
+    }
+)
+NON_PRODUCTION_DESTINATIONS: frozenset[Destination] = frozenset(
+    {Destination.LOCAL_DEV, Destination.STAGING_TEST}
+)
+
+
+class DomainProfile(StrEnum):
+    DEVELOPER_LOGS = "developer_logs"
+    TEST_DATASET = "test_dataset"
+    HEALTHCARE = "healthcare"
+    FINANCIAL = "financial"
+    GENERAL = "general"
+
+
+class UseCase(StrEnum):
+    DEBUGGING = "debugging"
+    ANALYTICS = "analytics"
+    TRAINING = "training"
+    EXTERNAL_LLM = "external_llm"
+    SUPPORT_BUNDLE = "support_bundle"
+    TEST_FIXTURE = "test_fixture"
+    MONITORING = "monitoring"
+    INCIDENT_RESPONSE = "incident_response"
+
+
+class InputFormat(StrEnum):
+    CSV = "csv"
+    JSON = "json"
+    JSONL = "jsonl"
+    TEXT = "text"
+    LOGFMT = "logfmt"
+    JSONLOG = "jsonlog"
+    ACCESSLOG = "accesslog"
+    SYSLOG = "syslog"
+    STACKTRACE = "stacktrace"
+
+
+# Line-oriented formats: each line is scanned as text and rewritten span by span.
+LINE_FORMATS = frozenset(
+    {
+        InputFormat.TEXT,
+        InputFormat.LOGFMT,
+        InputFormat.JSONLOG,
+        InputFormat.ACCESSLOG,
+        InputFormat.SYSLOG,
+        InputFormat.STACKTRACE,
+    }
+)
+
+# Findings for stack-trace frames and source lines (--format stacktrace); not a DetectorSet member.
+STACK_TRACE_DETECTOR = "stack_trace"
+
+
+class Action(StrEnum):
+    ALLOW = "allow"
+    REDACT = "redact"
+    REPLACE = "replace"
+    PSEUDONYMIZE = "pseudonymize"
+    GENERALIZE = "generalize"
+    DATE_SHIFT = "date_shift"
+    DROP_FIELD = "drop_field"
+    DROP_RECORD = "drop_record"
+    BLOCK = "block"
+    REVIEW = "review"
+
+
+# Higher rank = more protective. Used to combine several decisions on the same value.
+ACTION_RANK: dict[Action, int] = {
+    Action.BLOCK: 100,
+    Action.REVIEW: 90,
+    Action.DROP_RECORD: 80,
+    Action.DROP_FIELD: 70,
+    Action.REDACT: 60,
+    Action.REPLACE: 50,
+    Action.PSEUDONYMIZE: 40,
+    Action.GENERALIZE: 30,
+    Action.DATE_SHIFT: 20,
+    Action.ALLOW: 0,
+}
+
+# Actions that change a value in place (as opposed to removing it or stopping the run).
+VALUE_TRANSFORM_ACTIONS: frozenset[Action] = frozenset(
+    {Action.REDACT, Action.REPLACE, Action.PSEUDONYMIZE, Action.GENERALIZE, Action.DATE_SHIFT}
+)
+
+CREDENTIAL_CATEGORY = "credential.secret"
+UNKNOWN_CANDIDATE_CATEGORY = "unknown.sensitive_candidate"
+
+BUILTIN_CATEGORIES: frozenset[str] = frozenset(
+    {
+        "credential.secret",
+        "pii.email",
+        "pii.phone",
+        "pii.person_name",
+        "pii.government_id",
+        "pii.payment_card",
+        "pii.bank_account",
+        "pii.address",
+        "pii.date_of_birth",
+        "pii.precise_location",
+        "pii.ip_address",
+        "pii.device_identifier",
+        "pii.customer_identifier",
+        "health.phi",
+        "financial.sensitive",
+        "employment.sensitive",
+        "network.telemetry",
+        "proprietary.source",
+        "security.internal",
+        "location.coarse",
+        "identifier.generic",
+        "unknown.sensitive_candidate",
+        "date.generic",
+    }
+)
+
+# Categories for which an `allow` toward an untrusted destination requires explicit acknowledgement,
+# and which the residual rescan gate fails on by default.
+HIGH_RISK_CATEGORIES: frozenset[str] = frozenset(
+    {
+        "credential.secret",
+        "pii.email",
+        "pii.phone",
+        "pii.person_name",
+        "pii.government_id",
+        "pii.payment_card",
+        "pii.bank_account",
+        "pii.address",
+        "pii.date_of_birth",
+        "pii.precise_location",
+        "pii.customer_identifier",
+        "health.phi",
+        "financial.sensitive",
+        "employment.sensitive",
+        "unknown.sensitive_candidate",
+    }
+)
+
+# Custom categories in these families are treated as high-risk too (e.g. ``pii.member_number``).
+HIGH_RISK_FAMILIES: frozenset[str] = frozenset(
+    {"credential", "pii", "health", "financial", "employment", "unknown"}
+)
+
+# Detectors that must stay enabled for strict or untrusted-destination runs (credential coverage).
+CREDENTIAL_DETECTORS: frozenset[str] = frozenset({"secret_pattern", "field_name"})
+
+# Actions permitted for credential findings. Everything else is a hard-constraint violation.
+CREDENTIAL_PERMITTED_ACTIONS: frozenset[Action] = frozenset(
+    {
+        Action.BLOCK,
+        Action.REVIEW,
+        Action.REDACT,
+        Action.REPLACE,
+        Action.PSEUDONYMIZE,
+        Action.DROP_FIELD,
+        Action.DROP_RECORD,
+    }
+)
+
+
+class ReasonCode(StrEnum):
+    """Stable, machine-readable reason codes. They never contain values."""
+
+    CREDENTIAL_DETECTED_HARD_BLOCK = "credential_detected_hard_block"
+    EXPLICIT_HARD_BLOCK = "explicit_hard_block"
+    ENVIRONMENT_GUARDRAIL = "environment_guardrail"
+    EXPLICIT_POLICY_MATCH = "explicit_policy_match"
+    PROFILE_DEFAULT = "profile_default"
+    FIELD_NAME_SENSITIVE = "field_name_sensitive"
+    VALUE_PATTERN_MATCH = "value_pattern_match"
+    CONFIDENCE_BELOW_RULE_THRESHOLD = "confidence_below_rule_threshold"
+    NO_APPLICABLE_RULE_REQUIRES_REVIEW = "no_applicable_rule_requires_review"
+    NO_APPLICABLE_RULE_BLOCKS = "no_applicable_rule_blocks"
+    HIGH_RISK_ALLOW_REQUIRES_ACKNOWLEDGEMENT = "high_risk_allow_requires_acknowledgement"
+    UNSUPPORTED_TRANSFORM = "unsupported_transform"
+    DESTINATION_REQUIRED_BUT_MISSING = "destination_required_but_missing"
+    ENVIRONMENT_REQUIRED_BUT_MISSING = "environment_required_but_missing"
+    TARGET_ENVIRONMENT_REQUIRED_BUT_MISSING = "target_environment_required_but_missing"
+    ENVIRONMENT_CONTEXT_CONFLICT = "environment_context_conflict"
+    UNSUPPORTED_ENVIRONMENT = "unsupported_environment"
+    UNKNOWN_CONTEXT_REQUIRES_REVIEW = "unknown_context_requires_review"
+    PRD_TO_NONPRODUCTION_BLOCKED = "prd_to_nonproduction_transfer_blocked"
+    PRD_TO_NONPRODUCTION_APPROVED = "prd_to_nonproduction_transfer_approved"
+    STRICT_POLICY_REQUIRED = "strict_policy_required_for_destination"
+    RESCAN_REQUIRED = "rescan_required_for_destination"
+    CREDENTIAL_DETECTION_REQUIRED = "credential_detection_required"
+    KEY_SCAN_REQUIRED = "json_key_scan_required"
+    POLICY_CONFLICT = "policy_conflict"
+    KEY_MISSING = "key_missing"
+    KEY_INVALID = "key_invalid"
+    PARSE_ERROR = "parse_error"
+    SIZE_LIMIT_EXCEEDED = "size_limit_exceeded"
+    RESIDUAL_FINDING = "residual_finding"
+
+
+class ExitCode(IntEnum):
+    OK = 0
+    BLOCKED = 1
+    USAGE = 2
+    POLICY_INVALID = 3
+    INPUT_ERROR = 4
+    OUTPUT_ERROR = 5
+    KEY_OR_TRANSFORM_CONFIG = 6
+    RESCAN_FAILED = 7
+    INTERNAL = 8
+    INTERRUPTED = 130
+
+
+def confidence_band(confidence: float) -> str:
+    """Named band for a confidence estimate. Bands are engineering estimates, not probabilities."""
+    if confidence >= 0.8:
+        return "high"
+    if confidence >= 0.5:
+        return "medium"
+    return "low"
+
+
+REPORT_SCHEMA = "ciphergate.report/v1"
+PSEUDONYM_VERSION = "pz1"
+DEFAULT_KEY_ENV = "CIPHERGATE_HMAC_KEY"
+MIN_KEY_BYTES = 32
+
+DISCLAIMER = (
+    "A successful run means the configured processing completed under the selected policy. "
+    "It does not prove that the output contains no sensitive information; "
+    "detectors can miss values."
+)

ciphergate/detectors/__init__.py

@@ -0,0 +1,5 @@
+"""Deterministic local detectors. Detectors find candidates; they never decide actions."""
+
+from ciphergate.detectors.registry import BUILTIN_DETECTOR_IDS, DetectorSet
+
+__all__ = ["BUILTIN_DETECTOR_IDS", "DetectorSet"]

ciphergate/detectors/base.py

@@ -0,0 +1,53 @@
+"""Detector interface and shared helpers.
+
+Detectors only produce candidate findings (category, evidence, confidence, location). They never
+decide what happens to a value; that is the policy engine's job.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Protocol
+
+from ciphergate.models.finding import Finding, ScanItem, value_type_name
+
+# Output shapes CipherGate itself produces. Detectors skip them so a sanitized file is not
+# re-flagged by its own markers/pseudonyms.
+MARKER_TOKEN_RE = re.compile(r"\[[A-Z][A-Z0-9_]{0,31}(?::[a-z0-9_.]{1,48})?\]")
+PSEUDONYM_TOKEN_RE = re.compile(r"(?:[a-z][a-z0-9_]{0,31})?pz1_[a-z2-7]{32}")
+
+
+def is_ciphergate_token(text: str) -> bool:
+    return bool(MARKER_TOKEN_RE.fullmatch(text) or PSEUDONYM_TOKEN_RE.fullmatch(text))
+
+
+class Detector(Protocol):
+    id: str
+
+    def detect(self, item: ScanItem) -> list[Finding]: ...
+
+
+def make_finding(
+    item: ScanItem,
+    *,
+    category: str,
+    detector: str,
+    confidence: float,
+    evidence: tuple[str, ...],
+    span: tuple[int, int] | None = None,
+) -> Finding:
+    return Finding(
+        category=category,
+        detector=detector,
+        confidence=round(max(0.0, min(1.0, confidence)), 3),
+        evidence=evidence,
+        location=item.location(span),
+        field_name=item.field_name,
+        value_type=value_type_name(item.value),
+    )
+
+
+def preceding_context(text: str, start: int, keywords: tuple[str, ...], window: int = 24) -> bool:
+    """True if a keyword occurs in the ``window`` chars before ``start`` (case-insensitive)."""
+    snippet = text[max(0, start - window) : start].lower()
+    return any(k in snippet for k in keywords)

ciphergate/detectors/custom_regex.py

@@ -0,0 +1,37 @@
+"""User-configured regex detectors (validated by ``regex_safety`` at policy load time)."""
+
+from __future__ import annotations
+
+import re
+
+from ciphergate.detectors.base import is_ciphergate_token, make_finding
+from ciphergate.models.finding import Finding, ScanItem
+from ciphergate.models.policy import CustomDetector
+
+
+class CustomRegexDetector:
+    def __init__(self, spec: CustomDetector) -> None:
+        self.id = f"custom:{spec.id}"
+        self.category = spec.category
+        self.confidence = spec.confidence
+        self._regex = re.compile(spec.pattern, re.IGNORECASE if spec.ignore_case else 0)
+
+    def detect(self, item: ScanItem) -> list[Finding]:
+        text = item.text
+        if not text:
+            return []
+        out: list[Finding] = []
+        for m in self._regex.finditer(text):
+            if m.end() == m.start() or is_ciphergate_token(m.group(0)):
+                continue
+            out.append(
+                make_finding(
+                    item,
+                    category=self.category,
+                    detector=self.id,
+                    confidence=self.confidence,
+                    evidence=("pattern_match", "custom_regex"),
+                    span=(m.start(), m.end()),
+                )
+            )
+        return out

ciphergate/detectors/dates.py

@@ -0,0 +1,92 @@
+"""Date detector. Dates are classified as date of birth only with field or context evidence.
+
+Supported shapes: ISO ``YYYY-MM-DD`` (optionally with time and offset), ``YYYY/MM/DD``,
+``DD/MM/YYYY`` or ``MM/DD/YYYY`` (flagged ``ambiguous_locale``) and ``DD.MM.YYYY``. Without DOB
+evidence, a date is only reported (as ``date.generic``) when ``dates_emit_generic`` is enabled.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import re
+
+from ciphergate.detectors.base import make_finding, preceding_context
+from ciphergate.detectors.field_names import classify_field_name
+from ciphergate.models.finding import Finding, ScanItem
+
+_DATE_RE = re.compile(
+    r"(?<![\d/.-])"
+    r"(?:"
+    r"(?P<iso>(?P<y>\d{4})-(?P<m>\d{2})-(?P<d>\d{2})"
+    r"(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d{1,9})?)?(?:Z|[+-]\d{2}:?\d{2})?)?)"
+    r"|(?P<ymd>(?P<y2>\d{4})/(?P<m2>\d{2})/(?P<d2>\d{2}))"
+    r"|(?P<amb>(?P<a1>\d{1,2})[/.](?P<a2>\d{1,2})[/.](?P<y3>\d{4}))"
+    r")"
+    r"(?![\d/-])"
+)
+_DOB_KEYWORDS = ("dob", "born", "birth", "d.o.b")
+
+
+def _valid(year: int, month: int, day: int) -> bool:
+    try:
+        dt.date(year, month, day)
+    except ValueError:
+        return False
+    return True
+
+
+class DateDetector:
+    id = "date_pattern"
+
+    def __init__(self, emit_generic: bool = False) -> None:
+        self.emit_generic = emit_generic
+
+    def detect(self, item: ScanItem) -> list[Finding]:
+        text = item.text
+        if not text or not isinstance(item.value, str) or sum(c.isdigit() for c in text) < 6:
+            return []
+        field_term = classify_field_name(item.field_name) if item.field_name else None
+        field_is_dob = field_term is not None and field_term.category == "pii.date_of_birth"
+        out: list[Finding] = []
+        for m in _DATE_RE.finditer(text):
+            evidence = ["pattern_match"]
+            if m.group("iso"):
+                ok = _valid(int(m.group("y")), int(m.group("m")), int(m.group("d")))
+            elif m.group("ymd"):
+                ok = _valid(int(m.group("y2")), int(m.group("m2")), int(m.group("d2")))
+            else:
+                a, b, y = int(m.group("a1")), int(m.group("a2")), int(m.group("y3"))
+                ok = _valid(y, b, a) or _valid(y, a, b)
+                if _valid(y, b, a) and _valid(y, a, b) and a != b:
+                    evidence.append("ambiguous_locale")
+            if not ok:
+                continue
+            span = (m.start(), m.end())
+            if field_is_dob:
+                evidence.append("field_name_context")
+            elif preceding_context(text, m.start(), _DOB_KEYWORDS):
+                evidence.append("context_keyword")
+            else:
+                if self.emit_generic:
+                    out.append(
+                        make_finding(
+                            item,
+                            category="date.generic",
+                            detector=self.id,
+                            confidence=0.6,
+                            evidence=tuple(evidence),
+                            span=span,
+                        )
+                    )
+                continue
+            out.append(
+                make_finding(
+                    item,
+                    category="pii.date_of_birth",
+                    detector=self.id,
+                    confidence=0.85,
+                    evidence=tuple(evidence),
+                    span=span,
+                )
+            )
+        return out

ciphergate/detectors/email.py

@@ -0,0 +1,56 @@
+"""Email-like address detector (ASCII local parts and domains; bounded regex)."""
+
+from __future__ import annotations
+
+import re
+
+from ciphergate.detectors.base import make_finding
+from ciphergate.models.finding import Finding, ScanItem
+
+_EMAIL_RE = re.compile(
+    r"(?<![A-Za-z0-9._%+-])"
+    r"([A-Za-z0-9._%+-]{1,64})@((?:[A-Za-z0-9-]{1,63}\.){1,8}[A-Za-z]{2,63})"
+    r"(?![A-Za-z0-9-])"
+)
+
+RESERVED_EMAIL_DOMAINS = ("example.com", "example.org", "example.net")
+RESERVED_EMAIL_TLDS = (".invalid", ".example", ".test", ".localhost")
+
+
+def is_reserved_email_domain(domain: str) -> bool:
+    d = domain.lower()
+    return (
+        d in RESERVED_EMAIL_DOMAINS
+        or any(d.endswith(t) or d == t[1:] for t in RESERVED_EMAIL_TLDS)
+        or any(d.endswith("." + r) for r in RESERVED_EMAIL_DOMAINS)
+    )
+
+
+class EmailDetector:
+    id = "email_regex"
+
+    def detect(self, item: ScanItem) -> list[Finding]:
+        text = item.text
+        if not text or "@" not in text:
+            return []
+        out: list[Finding] = []
+        for m in _EMAIL_RE.finditer(text):
+            local, domain = m.group(1), m.group(2)
+            if local.startswith(".") or local.endswith(".") or ".." in local or ".." in domain:
+                continue
+            if any(label.startswith("-") or label.endswith("-") for label in domain.split(".")):
+                continue
+            evidence = ["pattern_match"]
+            if is_reserved_email_domain(domain):
+                evidence.append("reserved_domain")
+            out.append(
+                make_finding(
+                    item,
+                    category="pii.email",
+                    detector=self.id,
+                    confidence=0.9,
+                    evidence=tuple(evidence),
+                    span=(m.start(), m.end()),
+                )
+            )
+        return out