ciphergate-cli 0.7.0__py3-none-any.whl

ciphergate/detectors/field_names.py

@@ -0,0 +1,220 @@
+"""Field-name heuristics and policy-declared field classifications.
+
+Matching is token-based on a normalized name (camelCase/kebab-case/spaces -> snake_case, lowercase).
+A term matches when the normalized name equals it, or — for suffix-capable terms — ends with
+``_<term>``. The longest matching term wins, so ``server_ip_address`` is an IP address rather
+than a postal address. Names like ``token_count``, ``key_name`` or ``monkey`` deliberately do not
+match: this is a heuristic, not a universal dictionary.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import re
+from dataclasses import dataclass
+
+from ciphergate.detectors.base import is_ciphergate_token, make_finding
+from ciphergate.models.finding import Finding, ScanItem
+from ciphergate.models.policy import Classification
+
+_CAMEL_1 = re.compile(r"([a-z0-9])([A-Z])")
+_CAMEL_2 = re.compile(r"([A-Z]+)([A-Z][a-z])")
+_NON_ALNUM = re.compile(r"[^A-Za-z0-9]+")
+
+
+def normalize_field_name(name: str) -> str:
+    s = _CAMEL_2.sub(r"\1_\2", name)
+    s = _CAMEL_1.sub(r"\1_\2", s)
+    return _NON_ALNUM.sub("_", s).strip("_").lower()
+
+
+@dataclass(frozen=True, slots=True)
+class _Term:
+    term: str
+    category: str
+    confidence: float
+    suffix: bool
+
+
+def _terms(category: str, confidence: float, suffix: str = "", exact: str = "") -> list[_Term]:
+    out = [_Term(t, category, confidence, True) for t in suffix.split()]
+    out += [_Term(t, category, confidence, False) for t in exact.split()]
+    return out
+
+
+_TERMS: list[_Term] = [
+    *_terms(
+        "credential.secret",
+        0.85,
+        suffix="password passwd passphrase pwd secret api_key apikey access_token refresh_token "
+        "auth_token id_token bearer_token session_token private_key client_secret secret_key "
+        "access_key authorization credentials connection_string conn_str x_api_key",
+    ),
+    *_terms("credential.secret", 0.6, exact="token"),
+    *_terms(
+        "pii.email", 0.8, suffix="email e_mail email_address mail_address", exact="emails mail"
+    ),
+    *_terms(
+        "pii.phone",
+        0.75,
+        suffix="phone phone_number mobile mobile_number telephone contact_number fax cell_phone",
+        exact="tel msisdn phones",
+    ),
+    *_terms(
+        "pii.person_name",
+        0.7,
+        suffix="first_name last_name full_name given_name family_name surname middle_name "
+        "patient_name employee_name customer_name contact_name maiden_name",
+    ),
+    *_terms("pii.person_name", 0.45, exact="name"),
+    *_terms("pii.date_of_birth", 0.85, suffix="dob date_of_birth birth_date birthdate birthday"),
+    *_terms(
+        "pii.payment_card",
+        0.85,
+        suffix="card_number cc_number credit_card credit_card_number cvv cvc card_cvv",
+        exact="pan",
+    ),
+    *_terms(
+        "pii.bank_account",
+        0.8,
+        suffix="bank_account bank_account_number account_number iban routing_number sort_code",
+    ),
+    *_terms(
+        "pii.government_id",
+        0.85,
+        suffix="ssn social_security_number national_id tax_id passport_number drivers_license "
+        "national_insurance_number",
+        exact="passport nino",
+    ),
+    *_terms(
+        "health.phi",
+        0.75,
+        suffix="diagnosis medication medications medical_record medical_record_number patient_id "
+        "icd_code procedure_code allergies prescription clinical_notes",
+        exact="mrn patient condition claim claim_id",
+    ),
+    *_terms(
+        "pii.precise_location",
+        0.75,
+        suffix="latitude longitude gps coordinates geo_location geolocation",
+        exact="lat lon lng",
+    ),
+    *_terms(
+        "pii.address",
+        0.7,
+        suffix="address street street_address home_address mailing_address postal_address",
+        exact="postal_code zip zip_code postcode",
+    ),
+    *_terms(
+        "pii.ip_address",
+        0.8,
+        suffix="ip ip_address client_ip remote_addr ipv4 ipv6 source_ip x_forwarded_for",
+    ),
+    *_terms(
+        "pii.device_identifier",
+        0.7,
+        suffix="device_id mac_address imei advertising_id idfa serial_number device_token",
+    ),
+    *_terms(
+        "pii.customer_identifier",
+        0.7,
+        suffix="user_id customer_id account_id member_id subscriber_id username user_name "
+        "remote_user",
+        exact="login",
+    ),
+    *_terms(
+        "employment.sensitive",
+        0.7,
+        suffix="employee_id salary performance_rating hr_case_id",
+    ),
+    *_terms(
+        "financial.sensitive",
+        0.6,
+        suffix="balance account_balance credit_score transaction_amount income",
+    ),
+    *_terms(
+        "network.telemetry",
+        0.6,
+        suffix="trace_id request_id session_id span_id correlation_id hostname",
+        exact="host",
+    ),
+]
+
+
+def classify_field_name(name: str) -> _Term | None:
+    normalized = normalize_field_name(name)
+    if not normalized:
+        return None
+    best: _Term | None = None
+    for term in _TERMS:
+        hit = normalized == term.term or (term.suffix and normalized.endswith("_" + term.term))
+        if hit and (best is None or len(term.term) > len(best.term)):
+            best = term
+    return best
+
+
+def _has_protectable_value(item: ScanItem) -> bool:
+    if item.is_empty:
+        return False
+    return not (isinstance(item.value, str) and is_ciphergate_token(item.value))
+
+
+class FieldNameDetector:
+    id = "field_name"
+
+    def detect(self, item: ScanItem) -> list[Finding]:
+        if item.in_key or not item.field_name or not _has_protectable_value(item):
+            return []
+        term = classify_field_name(item.field_name)
+        if term is None:
+            return []
+        evidence = (
+            ("field_name_match",)
+            if not term.suffix or normalize_field_name(item.field_name) == term.term
+            else ("field_name_match", "field_name_suffix")
+        )
+        return [
+            make_finding(
+                item,
+                category=term.category,
+                detector=self.id,
+                confidence=term.confidence,
+                evidence=evidence,
+            )
+        ]
+
+
+class PolicyClassificationDetector:
+    """Emits findings for fields the policy explicitly classifies (e.g. ``admission_date``)."""
+
+    id = "policy_classification"
+
+    def __init__(self, classifications: list[Classification]) -> None:
+        self._rules = [
+            (
+                c,
+                frozenset(normalize_field_name(n) for n in c.field_names),
+                tuple(c.field_paths),
+            )
+            for c in classifications
+        ]
+
+    def detect(self, item: ScanItem) -> list[Finding]:
+        if item.in_key or not self._rules or not _has_protectable_value(item):
+            return []
+        normalized = normalize_field_name(item.field_name) if item.field_name else ""
+        out: list[Finding] = []
+        for rule, names, paths in self._rules:
+            by_name = bool(normalized) and normalized in names
+            by_path = any(fnmatch.fnmatchcase(item.field_path, p) for p in paths)
+            if by_name or by_path:
+                out.append(
+                    make_finding(
+                        item,
+                        category=rule.category,
+                        detector=self.id,
+                        confidence=rule.confidence,
+                        evidence=("policy_classification", f"classification:{rule.id}"),
+                    )
+                )
+        return out

ciphergate/detectors/ip_address.py

@@ -0,0 +1,85 @@
+"""IPv4/IPv6 detector. Candidates come from bounded regexes; validity comes from ``ipaddress``.
+
+Global addresses are reported as ``pii.ip_address``. Private, loopback, link-local and other
+non-global addresses are reported as ``network.telemetry`` (they can reveal internal topology).
+Version strings such as ``1.2.3.4`` are indistinguishable from IPv4 addresses
+(known false positive).
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import re
+
+from ciphergate.detectors.base import make_finding
+from ciphergate.models.finding import Finding, ScanItem
+
+_V4_RE = re.compile(r"(?<![\d.])((?:\d{1,3}\.){3}\d{1,3})(?![\d.]|\.\d)")
+_V6_RE = re.compile(
+    r"(?<![:\w.])\[?((?:[0-9A-Fa-f]{0,4}:){2,7}[0-9A-Fa-f]{0,4}(?:%[0-9A-Za-z]{1,16})?)\]?(?![:\w])"
+)
+
+DOCUMENTATION_V4 = (
+    ipaddress.ip_network("192.0.2.0/24"),
+    ipaddress.ip_network("198.51.100.0/24"),
+    ipaddress.ip_network("203.0.113.0/24"),
+)
+DOCUMENTATION_V6 = ipaddress.ip_network("2001:db8::/32")
+
+
+def is_documentation_address(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+    if isinstance(addr, ipaddress.IPv4Address):
+        return any(addr in n for n in DOCUMENTATION_V4)
+    return addr in DOCUMENTATION_V6
+
+
+class IPAddressDetector:
+    id = "ip_address"
+
+    def detect(self, item: ScanItem) -> list[Finding]:
+        text = item.text
+        if not text:
+            return []
+        out: list[Finding] = []
+        if "." in text:
+            for m in _V4_RE.finditer(text):
+                try:
+                    addr4 = ipaddress.IPv4Address(m.group(1))
+                except ValueError:
+                    continue
+                out.append(self._finding(item, addr4, (m.start(1), m.end(1)), "ipv4"))
+        if text.count(":") >= 2:
+            for m in _V6_RE.finditer(text):
+                raw = m.group(1)
+                if not any(c in "0123456789abcdefABCDEF" for c in raw.replace(":", "")):
+                    continue
+                try:
+                    addr6 = ipaddress.IPv6Address(raw.split("%", 1)[0])
+                except ValueError:
+                    continue
+                out.append(self._finding(item, addr6, (m.start(1), m.end(1)), "ipv6"))
+        return out
+
+    def _finding(
+        self,
+        item: ScanItem,
+        addr: ipaddress.IPv4Address | ipaddress.IPv6Address,
+        span: tuple[int, int],
+        family: str,
+    ) -> Finding:
+        evidence = ["pattern_match", "stdlib_validated", family]
+        if is_documentation_address(addr):
+            evidence.append("documentation_range")
+        if addr.is_global:
+            category, confidence = "pii.ip_address", 0.85
+        else:
+            category, confidence = "network.telemetry", 0.8
+            evidence.append("non_global_address")
+        return make_finding(
+            item,
+            category=category,
+            detector=self.id,
+            confidence=confidence,
+            evidence=tuple(evidence),
+            span=span,
+        )

ciphergate/detectors/network.py

@@ -0,0 +1,42 @@
+"""Internal hostname detector (configurable suffixes such as ``.internal`` / ``.corp``)."""
+
+from __future__ import annotations
+
+import re
+
+from ciphergate.detectors.base import make_finding
+from ciphergate.models.finding import Finding, ScanItem
+
+
+class InternalHostDetector:
+    id = "internal_host"
+
+    def __init__(self, suffixes: list[str]) -> None:
+        alternatives = "|".join(
+            re.escape(s.lstrip(".")) for s in sorted(suffixes, key=len, reverse=True)
+        )
+        label = r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?"
+        self._regex = (
+            re.compile(
+                rf"(?<![\w.-])((?:{label}\.){{1,10}}(?:{alternatives})|localhost)(?![\w-]|\.\w)",
+                re.IGNORECASE,
+            )
+            if alternatives
+            else re.compile(r"(?<![\w.-])(localhost)(?![\w-]|\.\w)", re.IGNORECASE)
+        )
+
+    def detect(self, item: ScanItem) -> list[Finding]:
+        text = item.text
+        if not text or not isinstance(item.value, str):
+            return []
+        return [
+            make_finding(
+                item,
+                category="network.telemetry",
+                detector=self.id,
+                confidence=0.75,
+                evidence=("pattern_match", "internal_host_suffix"),
+                span=(m.start(1), m.end(1)),
+            )
+            for m in self._regex.finditer(text)
+        ]

ciphergate/detectors/payment_card.py

@@ -0,0 +1,77 @@
+"""Payment-card-like detector: 13-19 digits, Luhn-valid, scored by issuer prefix and context.
+
+A Luhn-valid digit run without a known issuer prefix or context keyword is low confidence:
+long numeric identifiers pass Luhn roughly 10% of the time by chance.
+"""
+
+from __future__ import annotations
+
+import re
+
+from ciphergate.detectors.base import make_finding, preceding_context
+from ciphergate.models.finding import Finding, ScanItem
+
+_CARD_RE = re.compile(r"(?<![0-9A-Za-z_])(\d(?:[ -]?\d){12,18})(?![0-9A-Za-z_])")
+_KEYWORDS = ("card", "visa", "mastercard", "amex", "pan", "cc", "credit", "debit", "payment")
+
+
+def luhn_valid(digits: str) -> bool:
+    total = 0
+    for i, ch in enumerate(reversed(digits)):
+        d = ord(ch) - 48
+        if i % 2 == 1:
+            d *= 2
+            if d > 9:
+                d -= 9
+        total += d
+    return total % 10 == 0
+
+
+def known_issuer_prefix(digits: str) -> bool:
+    if digits[0] == "4" and len(digits) in (13, 16, 19):
+        return True
+    two, four = int(digits[:2]), int(digits[:4])
+    if 51 <= two <= 55 or 2221 <= four <= 2720:
+        return len(digits) == 16
+    if two in (34, 37):
+        return len(digits) == 15
+    if digits.startswith("6011") or two == 65:
+        return len(digits) in (16, 19)
+    return False
+
+
+class PaymentCardDetector:
+    id = "luhn_card"
+
+    def detect(self, item: ScanItem) -> list[Finding]:
+        text = item.text
+        if not text or sum(c.isdigit() for c in text) < 13:
+            return []
+        out: list[Finding] = []
+        for m in _CARD_RE.finditer(text):
+            raw = m.group(1)
+            digits = raw.replace(" ", "").replace("-", "")
+            if not 13 <= len(digits) <= 19 or len(set(digits)) == 1 or not luhn_valid(digits):
+                continue
+            seps = {c for c in raw if c in " -"}
+            if len(seps) > 1:
+                continue
+            evidence = ["pattern_match", "checksum_valid"]
+            confidence = 0.4
+            if known_issuer_prefix(digits):
+                confidence = 0.8
+                evidence.append("issuer_prefix")
+            if preceding_context(text, m.start(), _KEYWORDS, window=32):
+                confidence += 0.15
+                evidence.append("context_keyword")
+            out.append(
+                make_finding(
+                    item,
+                    category="pii.payment_card",
+                    detector=self.id,
+                    confidence=min(confidence, 0.95),
+                    evidence=tuple(evidence),
+                    span=(m.start(1), m.end(1)),
+                )
+            )
+        return out

ciphergate/detectors/phone.py

@@ -0,0 +1,67 @@
+"""Phone-like number detector with conservative, context-dependent confidence.
+
+Locale ambiguity is high: this detector does not validate numbering plans. A leading ``+`` with
+7-15 digits (E.164 length) or a common separator layout gives medium/high confidence; a bare digit
+run is low confidence unless a context keyword (``phone``, ``tel``, ...) precedes it. Spans that
+look like dates, IP addresses or payment cards are suppressed by the registry.
+"""
+
+from __future__ import annotations
+
+import re
+
+from ciphergate.detectors.base import make_finding, preceding_context
+from ciphergate.models.finding import Finding, ScanItem
+
+_PHONE_RE = re.compile(
+    r"(?<![\w+./:-])"
+    r"(\+?\(?\d{1,4}\)?(?:[ .-]?\(?\d{1,5}\)?){1,6})"
+    r"(?:\s?(?:x|ext\.?)\s?\d{1,6})?"
+    r"(?![\w/:-]|\.\d)"
+)
+_DATE_LIKE = re.compile(r"^\d{4}[-./]\d{1,2}[-./]\d{1,2}$|^\d{1,2}[-./]\d{1,2}[-./]\d{2,4}$")
+_SEPARATED = re.compile(r"^\(?\d{2,4}\)?[ .-]\d{2,5}[ .-]\d{2,5}(?:[ .-]\d{2,5})?$")
+_KEYWORDS = ("phone", "tel", "mobile", "cell", "call", "fax", "sms", "whatsapp", "contact")
+
+
+class PhoneDetector:
+    id = "phone_pattern"
+
+    def detect(self, item: ScanItem) -> list[Finding]:
+        text = item.text
+        if not text or sum(c.isdigit() for c in text) < 7:
+            return []
+        out: list[Finding] = []
+        for m in _PHONE_RE.finditer(text):
+            candidate = m.group(0)
+            core = m.group(1)
+            digits = re.sub(r"\D", "", core)
+            if not 7 <= len(digits) <= 15 or len(set(digits)) == 1:
+                continue
+            if _DATE_LIKE.match(core.strip()):
+                continue
+            evidence = ["pattern_match"]
+            if core.startswith("+"):
+                confidence = 0.8 if len(digits) >= 8 else 0.55
+                evidence.append("international_prefix")
+            elif _SEPARATED.match(core.strip()):
+                confidence = 0.6
+                evidence.append("separator_layout")
+            else:
+                confidence = 0.3
+            if preceding_context(text, m.start(), _KEYWORDS):
+                confidence += 0.2
+                evidence.append("context_keyword")
+            if "x" in candidate.lower() or "ext" in candidate.lower():
+                evidence.append("extension")
+            out.append(
+                make_finding(
+                    item,
+                    category="pii.phone",
+                    detector=self.id,
+                    confidence=min(confidence, 0.95),
+                    evidence=tuple(evidence),
+                    span=(m.start(), m.end()),
+                )
+            )
+        return out

ciphergate/detectors/presidio_ner.py

@@ -0,0 +1,112 @@
+"""Optional local NER detector backed by Microsoft Presidio (``pip install 'ciphergate-cli[ner]'``).
+
+Off by default; enabled only by ``detectors.presidio.enabled: true`` in the policy. Safety:
+
+* Never downloads anything. Presidio's spaCy engine downloads a missing model on first use, so
+  the configured spaCy model package must already be importable; otherwise the run fails with
+  ``ner_model_missing`` before Presidio is imported or constructed. (CipherGate does not import
+  network libraries at all, so it cannot police third-party code beyond this check.)
+* Missing package or model fails the run (exit 6); it never silently runs without NER.
+* Like every detector it only reports candidates (entity type, score, span). It cannot allow a
+  value: decisions stay with the policy engine, so explicit block/review rules still win.
+* Presidio errors are replaced by a fixed message; its exception text is never shown.
+"""
+
+from __future__ import annotations
+
+import functools
+import importlib
+import importlib.util
+from typing import Any
+
+from ciphergate.detectors.base import is_ciphergate_token, make_finding
+from ciphergate.errors import DetectionError, DetectorUnavailableError
+from ciphergate.models.finding import Finding, ScanItem
+from ciphergate.models.policy import PresidioSettings
+
+DETECTOR_ID = "presidio_ner"
+PACKAGE = "presidio_analyzer"
+
+
+def _installed(module: str) -> bool:
+    try:
+        return importlib.util.find_spec(module) is not None
+    except (ImportError, ValueError):
+        return False
+
+
+@functools.lru_cache(maxsize=4)
+def load_analyzer(language: str, model: str) -> Any:
+    """Build (once per process) a Presidio AnalyzerEngine on an already-installed spaCy model."""
+    if not _installed(PACKAGE):
+        raise DetectorUnavailableError(
+            "detectors.presidio is enabled but the optional NER extra is not installed "
+            "(pip install 'ciphergate-cli[ner]')",
+            code="ner_not_installed",
+        )
+    if not _installed(model):
+        raise DetectorUnavailableError(
+            "detectors.presidio is enabled but its spaCy model is not installed; CipherGate "
+            "never downloads models, install it yourself (see docs/detector-coverage.md)",
+            code="ner_model_missing",
+        )
+    try:
+        analyzer_mod = importlib.import_module(PACKAGE)
+        nlp_mod = importlib.import_module(f"{PACKAGE}.nlp_engine")
+        provider = nlp_mod.NlpEngineProvider(
+            nlp_configuration={
+                "nlp_engine_name": "spacy",
+                "models": [{"lang_code": language, "model_name": model}],
+            }
+        )
+        return analyzer_mod.AnalyzerEngine(
+            nlp_engine=provider.create_engine(), supported_languages=[language]
+        )
+    except DetectorUnavailableError:
+        raise
+    except Exception:  # noqa: BLE001 - third-party messages may contain anything; replace them
+        raise DetectorUnavailableError(
+            "the optional NER detector could not be initialised", code="ner_init_failed"
+        ) from None
+
+
+class PresidioDetector:
+    id = DETECTOR_ID
+
+    def __init__(self, settings: PresidioSettings, analyzer: Any | None = None) -> None:
+        self.settings = settings
+        self.entities = dict(settings.entities)
+        self.analyzer = analyzer or load_analyzer(settings.language, settings.model)
+
+    def detect(self, item: ScanItem) -> list[Finding]:
+        text = item.text
+        if not text or len(text) < 2 or not any(c.isalpha() for c in text):
+            return []
+        try:
+            results = self.analyzer.analyze(
+                text=text,
+                language=self.settings.language,
+                entities=list(self.entities),
+                score_threshold=self.settings.min_score,
+            )
+        except Exception:  # noqa: BLE001 - never forward third-party messages (may echo input)
+            raise DetectionError("the optional NER detector failed", code="ner_failed") from None
+        out: list[Finding] = []
+        for result in results:
+            category = self.entities.get(str(result.entity_type))
+            start, end = int(result.start), int(result.end)
+            if category is None or not 0 <= start < end <= len(text):
+                continue
+            if is_ciphergate_token(text[start:end].strip("\"'")):
+                continue
+            out.append(
+                make_finding(
+                    item,
+                    category=category,
+                    detector=self.id,
+                    confidence=float(result.score),
+                    evidence=("ner_entity", f"entity:{str(result.entity_type).lower()}"),
+                    span=(start, end),
+                )
+            )
+        return out

ciphergate/detectors/regex_safety.py

@@ -0,0 +1,40 @@
+"""Heuristic checks for user-supplied regular expressions.
+
+Python's ``re`` engine backtracks, so a pathological pattern can take exponential time. These checks
+reject common risky constructs (nested quantifiers, backreferences, empty matches,
+excessive length).
+They are heuristics, NOT a proof of regex safety; inputs are additionally bounded by record limits.
+"""
+
+from __future__ import annotations
+
+import re
+
+MAX_PATTERN_LENGTH = 512
+
+_NESTED_QUANTIFIER = re.compile(r"\((?:[^()\\]|\\.)*[*+}](?:[^()\\]|\\.)*\)\s*(?:[*+]|\{\d*,)")
+_BACKREFERENCE = re.compile(r"\\[1-9]|\(\?P=")
+_UNBOUNDED_WILDCARD_RUN = re.compile(r"(?:\.\*|\.\+)(?:[^()]*)(?:\.\*|\.\+)")
+
+
+def regex_problems(pattern: str, *, strict: bool = True) -> list[str]:
+    """Return a list of safe problem descriptions (never echoes the pattern)."""
+    problems: list[str] = []
+    if len(pattern) > MAX_PATTERN_LENGTH:
+        problems.append(f"pattern longer than {MAX_PATTERN_LENGTH} characters")
+        return problems
+    try:
+        compiled = re.compile(pattern)
+    except re.error:
+        problems.append("pattern does not compile")
+        return problems
+    if compiled.search("") is not None:
+        problems.append("pattern can match the empty string")
+    if strict:
+        if _NESTED_QUANTIFIER.search(pattern):
+            problems.append("nested quantifier (risk of catastrophic backtracking)")
+        if _BACKREFERENCE.search(pattern):
+            problems.append("backreferences are not allowed")
+        if _UNBOUNDED_WILDCARD_RUN.search(pattern):
+            problems.append("multiple unbounded wildcards (risk of excessive backtracking)")
+    return problems