shadowshield 0.4.0__py3-none-any.whl

shadowshield/utils/logging.py

@@ -0,0 +1,89 @@
+"""Structured logging + an append-only JSONL audit sink.
+
+ShadowShield uses :mod:`structlog` so every security event is a structured
+record (easy to ship to a SIEM) rather than an opaque string. The audit sink is
+deliberately simple — newline-delimited JSON — so it works everywhere with no
+infrastructure.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from ..core.config import LoggingConfig
+
+_CONFIGURED = False
+
+
+def configure_logging(config: LoggingConfig) -> Any:
+    """Configure structlog once and return a bound ShadowShield logger.
+
+    Logs are written to **stderr** (never stdout) so ShadowShield can sit on a
+    stdin/stdout model pipeline without corrupting it.
+    """
+    global _CONFIGURED
+    if not _CONFIGURED:
+        structlog.configure(
+            processors=[
+                structlog.contextvars.merge_contextvars,
+                structlog.processors.add_log_level,
+                structlog.processors.TimeStamper(fmt="iso", utc=True),
+                structlog.processors.JSONRenderer(),
+            ],
+            wrapper_class=structlog.make_filtering_bound_logger(
+                getattr(logging, config.level, logging.INFO)
+            ),
+            # stderr, not stdout — keep stdout clean for the host application.
+            logger_factory=structlog.PrintLoggerFactory(file=sys.stderr),
+            cache_logger_on_first_use=True,
+        )
+        _CONFIGURED = True
+    return structlog.get_logger("shadowshield")
+
+
+class AuditLog:
+    """Append-only JSONL audit trail for scan decisions.
+
+    Each call to :meth:`record` writes exactly one JSON object on its own line.
+    When ``redact_payloads`` is set (the default), raw offending text is never
+    written — only a short, truncated preview — so the audit log can't itself
+    become a data-leak vector.
+    """
+
+    def __init__(self, config: LoggingConfig) -> None:
+        self._config = config
+        self._path = Path(config.audit_path) if config.audit_path else None
+        if self._path is not None:
+            self._path.parent.mkdir(parents=True, exist_ok=True)
+        self._logger = configure_logging(config)
+
+    @property
+    def redact(self) -> bool:
+        return self._config.redact_payloads
+
+    def record(self, event: dict[str, Any], *, notable: bool = True) -> None:
+        """Emit a security event to structlog and (if configured) the JSONL file.
+
+        ``notable`` events (a threat was found / the payload wasn't clean) are
+        logged at INFO; clean scans are logged at DEBUG so the default INFO level
+        stays quiet on normal traffic. The JSONL audit file, when configured,
+        records **every** decision regardless — it's the complete trail.
+        """
+        if not self._config.enabled:
+            return
+        if notable:
+            self._logger.info("shadowshield.scan", **event)
+        else:
+            self._logger.debug("shadowshield.scan", **event)
+        if self._path is not None:
+            try:
+                with self._path.open("a", encoding="utf-8") as fh:
+                    fh.write(json.dumps(event, ensure_ascii=False, default=str) + "\n")
+            except OSError as exc:  # never let audit IO break the request path
+                self._logger.warning("shadowshield.audit_write_failed", error=str(exc))

shadowshield/utils/scoring.py

@@ -0,0 +1,42 @@
+"""Aggregation of many per-detector threats into one payload-level verdict.
+
+The aggregation philosophy is **noisy-or with severity weighting**: independent
+detectors each raise the overall suspicion, but no single weak signal can be
+washed out by averaging. A confident HIGH from one detector should dominate even
+if ten other detectors stayed silent — averaging would dilute it, which is the
+wrong failure mode for security.
+"""
+
+from __future__ import annotations
+
+from ..core.types import Severity, Threat
+from .text import truncate  # noqa: F401  (re-exported for convenience)
+
+
+def aggregate_score(threats: list[Threat], weights: dict[str, float] | None = None) -> float:
+    """Combine weighted detector scores via a noisy-or.
+
+    Each threat contributes ``p_i = clamp(score * weight)``. The combined
+    probability that *something* is wrong is ``1 - Π(1 - p_i)`` — monotonic,
+    bounded in ``[0, 1]``, and order-independent. More corroborating detectors
+    push the score up; one strong detector already pushes it high.
+    """
+    if not threats:
+        return 0.0
+    weights = weights or {}
+    complement = 1.0
+    for t in threats:
+        w = weights.get(t.detector, 1.0)
+        p = max(0.0, min(1.0, t.score * w))
+        complement *= 1.0 - p
+    return 1.0 - complement
+
+
+def aggregate_severity(threats: list[Threat], score: float) -> Severity:
+    """The payload severity is the max of the strongest finding and the
+    score-implied band — so a pile of LOW findings that together score HIGH is
+    treated as HIGH, and a lone CRITICAL is never downgraded by a low aggregate.
+    """
+    by_threat = max((t.severity for t in threats), default=Severity.NONE)
+    by_score = Severity.from_score(score)
+    return max(by_threat, by_score)

shadowshield/utils/text.py

@@ -0,0 +1,175 @@
+"""Text normalisation and decoding helpers.
+
+Many prompt-injection evasions rely on the *rendering* of text differing from
+its *bytes*: zero-width characters splitting keywords, homoglyphs, bidirectional
+overrides, fullwidth Unicode, or payloads hidden in base64/hex. The detectors
+operate on a normalised view produced here so a single attack can't dodge every
+detector by changing its surface form.
+
+Nothing in this module *decides* anything — it only exposes the hidden surface.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import re
+import unicodedata
+from dataclasses import dataclass
+
+# Characters used to hide / split / reorder text. Stripping these collapses many
+# obfuscation tricks back to their visible meaning.
+INVISIBLE_CHARS = (
+    "​‌‍‎‏"  # zero-width space/joiners, LRM/RLM
+    "‪‫‬‭‮"  # bidirectional embeddings/overrides
+    "⁠⁡⁢⁣⁤"  # word joiner / invisible operators
+    "⁦⁧⁨⁩"  # isolates
+    ""  # BOM / zero-width no-break space
+)
+_INVISIBLE_RE = re.compile(f"[{re.escape(INVISIBLE_CHARS)}]")
+
+# Homoglyph confusables -> ASCII. Small, high-value subset (Cyrillic/Greek
+# look-alikes commonly used to spell "ignore", "system", etc.).
+_CONFUSABLES = {
+    "а": "a",
+    "е": "e",
+    "о": "o",
+    "р": "p",
+    "с": "c",
+    "х": "x",
+    "у": "y",
+    "і": "i",
+    "ѕ": "s",
+    "Ѕ": "s",
+    "ј": "j",
+    "ԁ": "d",
+    "ո": "n",
+    "ⅼ": "l",
+    "ɡ": "g",
+    "һ": "h",
+    "Α": "A",
+    "Β": "B",
+    "Ε": "E",
+    "Ζ": "Z",
+    "Η": "H",
+    "Ι": "I",
+    "Κ": "K",
+    "Μ": "M",
+    "Ν": "N",
+    "Ο": "O",
+    "Ρ": "P",
+    "Τ": "T",
+    "Υ": "Y",
+    "Χ": "X",
+}
+
+# Lookarounds (not \b) so trailing '=' padding is captured — '=' is a non-word
+# char, so \b after it would drop the padding and break the length check.
+_B64_RE = re.compile(r"(?<![A-Za-z0-9+/])[A-Za-z0-9+/]{16,}={0,2}(?![A-Za-z0-9+/=])")
+_HEX_RE = re.compile(r"\b(?:0x)?[0-9a-fA-F]{16,}\b")
+
+
+@dataclass(slots=True)
+class NormalizedText:
+    """The result of :func:`normalize`.
+
+    Attributes:
+        original: The text exactly as received.
+        normalized: Lower-noise view — invisibles stripped, NFKC-folded,
+            confusables mapped, whitespace collapsed. Detectors match on this.
+        had_invisible: Whether any invisible/bidi control chars were present
+            (a signal in its own right).
+        had_confusables: Whether any homoglyph substitution happened.
+    """
+
+    original: str
+    normalized: str
+    had_invisible: bool
+    had_confusables: bool
+
+
+def normalize(text: str) -> NormalizedText:
+    """Produce a normalised, de-obfuscated view of ``text``.
+
+    Order matters: strip invisibles first (so they can't survive folding), then
+    NFKC-normalise (collapses fullwidth/compatibility forms), then map known
+    confusables, then collapse runs of whitespace.
+    """
+    had_invisible = bool(_INVISIBLE_RE.search(text))
+    stripped = _INVISIBLE_RE.sub("", text)
+
+    folded = unicodedata.normalize("NFKC", stripped)
+
+    had_confusables = any(ch in _CONFUSABLES for ch in folded)
+    if had_confusables:
+        folded = "".join(_CONFUSABLES.get(ch, ch) for ch in folded)
+
+    collapsed = re.sub(r"\s+", " ", folded).strip()
+    return NormalizedText(
+        original=text,
+        normalized=collapsed,
+        had_invisible=had_invisible,
+        had_confusables=had_confusables,
+    )
+
+
+@dataclass(slots=True)
+class DecodedSegment:
+    """A successfully decoded hidden segment found inside text."""
+
+    encoding: str  # "base64" | "hex"
+    source: str  # the encoded substring
+    decoded: str  # the decoded, printable text
+    span: tuple[int, int]
+
+
+def _is_mostly_printable(data: bytes, threshold: float = 0.85) -> bool:
+    if not data:
+        return False
+    printable = sum(1 for b in data if 0x09 <= b <= 0x7E)
+    return printable / len(data) >= threshold
+
+
+def extract_encoded_segments(text: str, *, min_decoded_len: int = 6) -> list[DecodedSegment]:
+    """Find and decode base64/hex blobs that resolve to readable text.
+
+    Only segments that decode to *mostly printable* ASCII are returned — random
+    high-entropy strings (hashes, ids) decode to noise and are ignored, which
+    keeps the false-positive rate low.
+    """
+    out: list[DecodedSegment] = []
+
+    for m in _B64_RE.finditer(text):
+        token = m.group(0)
+        # base64 length must be a multiple of 4 to be valid
+        if len(token) % 4 != 0:
+            continue
+        try:
+            raw = base64.b64decode(token, validate=True)
+        except (binascii.Error, ValueError):
+            continue
+        if _is_mostly_printable(raw):
+            decoded = raw.decode("ascii", errors="replace")
+            if len(decoded.strip()) >= min_decoded_len:
+                out.append(DecodedSegment("base64", token, decoded, m.span()))
+
+    for m in _HEX_RE.finditer(text):
+        token = m.group(0).removeprefix("0x")
+        if len(token) % 2 != 0:
+            continue
+        try:
+            raw = bytes.fromhex(token)
+        except ValueError:
+            continue
+        if _is_mostly_printable(raw):
+            decoded = raw.decode("ascii", errors="replace")
+            if len(decoded.strip()) >= min_decoded_len:
+                out.append(DecodedSegment("hex", m.group(0), decoded, m.span()))
+
+    return out
+
+
+def truncate(text: str, limit: int = 120) -> str:
+    """Shorten text for log lines without leaking the whole payload."""
+    text = text.replace("\n", "\\n")
+    return text if len(text) <= limit else text[: limit - 1] + "…"