patchi 0.7.5__py3-none-any.whl

patchi/core/hosted/anomaly.py

@@ -0,0 +1,298 @@
+"""
+Anomaly detection for Patchi hosted mode.
+
+Two layers:
+  1. Statistical: rolling windows — request rate spikes, error rate spikes,
+     status code distribution shifts, path enumeration patterns.
+  2. ML: IsolationForest on request feature vectors — catches what statistics miss.
+
+Detects: brute force, credential stuffing, scanner sweeps, injection probes,
+         error spikes, path traversal attempts.
+"""
+
+from __future__ import annotations
+
+import collections
+import time
+from dataclasses import dataclass, field
+
+from patchi.core.hosted.log_parsers import LogEntry
+
+
+@dataclass
+class AnomalyFinding:
+    detector: str
+    severity: str  # critical / high / medium
+    title: str
+    detail: str
+    ip: str = ""
+    evidence: dict = field(default_factory=dict)
+    detected_at: float = field(default_factory=time.time)
+
+
+# ── Statistical detectors ──────────────────────────────────────────────────────
+
+
+class _RollingWindow:
+    """Fixed-size time window for rate tracking."""
+
+    def __init__(self, window_secs: int = 60) -> None:
+        self._window = window_secs
+        self._events: collections.deque = collections.deque()
+
+    def add(self, value: float = 1.0) -> None:
+        now = time.time()
+        self._events.append((now, value))
+        self._trim(now)
+
+    def _trim(self, now: float) -> None:
+        while self._events and now - self._events[0][0] > self._window:
+            self._events.popleft()
+
+    def rate(self) -> float:
+        """Events per second in the current window."""
+        if not self._events:
+            return 0.0
+        self._trim(time.time())
+        if not self._events:
+            return 0.0
+        # Compute actual time span of events in window
+        span = self._events[-1][0] - self._events[0][0]
+        if span <= 0:
+            return float(len(self._events))
+        return sum(v for _, v in self._events) / span
+
+    def count(self) -> int:
+        self._trim(time.time())
+        return len(self._events)
+
+
+class StatisticalDetector:
+    """
+    Rolling window detectors for common attack patterns.
+    Maintains per-IP and global counters.
+    Accepts optional config dict for threshold overrides.
+    """
+
+    # Defaults
+    _DEFAULT_RATE_SPIKE_RPS = 50
+    _DEFAULT_BRUTE_FORCE_AUTH = 20
+    _DEFAULT_SCANNER_PATHS = 30
+    _DEFAULT_ERROR_RATE_PCT = 0.40
+    _INJECTION_PATTERNS = [
+        "../",
+        "..\\",
+        "%2e%2e",
+        "'%20OR%20",
+        " OR ",
+        "UNION SELECT",
+        "<script",
+        "javascript:",
+        "${jndi:",
+        "etc/passwd",
+    ]
+    _MAX_IP_TRACKERS = 10_000
+
+    def __init__(self, config: dict | None = None) -> None:
+        hosted = (config or {}).get("hosted", {})
+        self._RATE_SPIKE_RPS = hosted.get("rate_spike_rps", self._DEFAULT_RATE_SPIKE_RPS)
+        self._BRUTE_FORCE_AUTH = hosted.get("brute_force_auth", self._DEFAULT_BRUTE_FORCE_AUTH)
+        self._SCANNER_PATHS = hosted.get("scanner_paths", self._DEFAULT_SCANNER_PATHS)
+        self._ERROR_RATE_PCT = hosted.get("error_rate_pct", self._DEFAULT_ERROR_RATE_PCT)
+        self._whitelist: set[str] = set(hosted.get("ip_whitelist", []))
+
+        self._global_rate = _RollingWindow(60)
+        self._error_count = _RollingWindow(60)
+        self._total_count = _RollingWindow(60)
+        self._ip_auth_hits: dict[str, _RollingWindow] = {}
+        self._ip_paths: dict[str, set] = {}
+        self._ip_path_times: dict[str, float] = {}
+        self._ip_lru_order: list[str] = []
+
+    def feed(self, entry: LogEntry) -> list[AnomalyFinding]:
+        findings: list[AnomalyFinding] = []
+
+        # Skip whitelisted IPs
+        if entry.ip and entry.ip in self._whitelist:
+            return findings
+
+        self._global_rate.add()
+        self._total_count.add()
+        if entry.status >= 400:
+            self._error_count.add()
+
+        # Rate spike
+        if self._global_rate.rate() > self._RATE_SPIKE_RPS:
+            findings.append(
+                AnomalyFinding(
+                    detector="rate_spike",
+                    severity="high",
+                    title=f"Request rate spike: {self._global_rate.rate():.0f} req/s",
+                    detail="Sustained high request rate — possible DDoS or scanner.",
+                )
+            )
+
+        # Error rate
+        total = self._total_count.count()
+        if total > 20:
+            err_pct = self._error_count.count() / total
+            if err_pct > self._ERROR_RATE_PCT:
+                findings.append(
+                    AnomalyFinding(
+                        detector="error_spike",
+                        severity="medium",
+                        title=f"High error rate: {err_pct:.0%}",
+                        detail=f"{self._error_count.count()} errors in last 60s.",
+                    )
+                )
+
+        # Per-IP checks
+        if entry.ip:
+            findings.extend(self._check_ip(entry))
+
+        # Injection patterns in path
+        path_lower = entry.path.lower()
+        for pattern in self._INJECTION_PATTERNS:
+            if pattern.lower() in path_lower:
+                findings.append(
+                    AnomalyFinding(
+                        detector="injection_probe",
+                        severity="critical",
+                        title=f"Injection probe detected: {pattern}",
+                        detail=f"IP {entry.ip} sent probe in path: {entry.path[:120]}",
+                        ip=entry.ip,
+                        evidence={"path": entry.path, "pattern": pattern},
+                    )
+                )
+
+        return findings
+
+    def _check_ip(self, entry: LogEntry) -> list[AnomalyFinding]:
+        findings: list[AnomalyFinding] = []
+        ip = entry.ip
+
+        # LRU eviction if tracking too many IPs
+        if ip not in self._ip_auth_hits and len(self._ip_auth_hits) >= self._MAX_IP_TRACKERS:
+            self._evict_lru()
+
+        # Brute force on auth endpoints
+        auth_paths = ("/auth/login", "/login", "/signin", "/api/token", "/admin")
+        if any(entry.path.startswith(p) for p in auth_paths) and entry.method == "POST":
+            if ip not in self._ip_auth_hits:
+                self._ip_auth_hits[ip] = _RollingWindow(60)
+                self._ip_lru_order.append(ip)
+            self._ip_auth_hits[ip].add()
+            if self._ip_auth_hits[ip].count() >= self._BRUTE_FORCE_AUTH:
+                findings.append(
+                    AnomalyFinding(
+                        detector="brute_force",
+                        severity="critical",
+                        title=f"Brute force: {ip} → {entry.path}",
+                        detail=f"{self._ip_auth_hits[ip].count()} auth attempts in 60s.",
+                        ip=ip,
+                        evidence={"count": self._ip_auth_hits[ip].count()},
+                    )
+                )
+
+        # Path enumeration (scanner sweep)
+        now = time.time()
+        last_reset = self._ip_path_times.get(ip, 0)
+        if now - last_reset > 60:
+            self._ip_paths[ip] = set()
+            self._ip_path_times[ip] = now
+            if ip not in self._ip_lru_order:
+                self._ip_lru_order.append(ip)
+        self._ip_paths[ip].add(entry.path)
+        if len(self._ip_paths[ip]) >= self._SCANNER_PATHS:
+            findings.append(
+                AnomalyFinding(
+                    detector="scanner_sweep",
+                    severity="high",
+                    title=f"Scanner sweep: {ip} probing {len(self._ip_paths[ip])} paths",
+                    detail="Rapid path enumeration — possible automated scanner.",
+                    ip=ip,
+                    evidence={"distinct_paths": len(self._ip_paths[ip])},
+                )
+            )
+
+        return findings
+
+    def _evict_lru(self) -> None:
+        """Evict oldest tracked IPs to bound memory usage."""
+        while len(self._ip_auth_hits) >= self._MAX_IP_TRACKERS and self._ip_lru_order:
+            old_ip = self._ip_lru_order.pop(0)
+            self._ip_auth_hits.pop(old_ip, None)
+            self._ip_paths.pop(old_ip, None)
+            self._ip_path_times.pop(old_ip, None)
+
+
+class MLDetector:
+    """
+    IsolationForest anomaly detector on request feature vectors.
+    Trains on a baseline window, then flags outliers.
+    Requires sklearn — gracefully no-ops if not available.
+    """
+
+    MIN_SAMPLES = 100  # minimum before ML kicks in
+    _MAX_SAMPLES = 10_000  # cap memory usage
+
+    def __init__(self) -> None:
+        self._samples: list[list[float]] = []
+        self._model = None
+        self._trained = False
+
+    def _to_vector(self, entry: LogEntry) -> list[float]:
+        """Convert a log entry to a numeric feature vector."""
+        return [
+            float(entry.status),
+            float(len(entry.path)),
+            float(entry.path.count("/")),
+            float(1 if entry.method == "POST" else 0),
+            float(1 if entry.status >= 500 else 0),
+            float(1 if entry.status == 404 else 0),
+            float(len(entry.path.split("?")[1]) if "?" in entry.path else 0),
+        ]
+
+    def feed(self, entry: LogEntry) -> list[AnomalyFinding]:
+        try:
+            from sklearn.ensemble import IsolationForest
+        except ImportError:
+            return []
+
+        vec = self._to_vector(entry)
+        self._samples.append(vec)
+
+        # Cap memory usage
+        if len(self._samples) > self._MAX_SAMPLES:
+            self._samples = self._samples[-self._MAX_SAMPLES :]
+
+        if len(self._samples) < self.MIN_SAMPLES:
+            return []
+
+        # Retrain every 500 samples
+        if len(self._samples) % 500 == 0 or not self._trained:
+            self._model = IsolationForest(contamination=0.05, random_state=42, n_estimators=50)
+            self._model.fit(self._samples[-1000:])
+            self._trained = True
+
+        if not self._model:
+            return []
+
+        score = self._model.decision_function([vec])[0]
+        pred = self._model.predict([vec])[0]
+
+        if pred == -1 and score < -0.2:
+            return [
+                AnomalyFinding(
+                    detector="ml_outlier",
+                    severity="medium",
+                    title="ML anomaly detected",
+                    detail=(
+                        f"Request to {entry.path} from {entry.ip} has anomaly score {score:.3f}."
+                    ),
+                    ip=entry.ip,
+                    evidence={"score": round(score, 3), "status": entry.status},
+                )
+            ]
+
+        return []

patchi/core/hosted/audit_log.py

@@ -0,0 +1,102 @@
+"""
+Rotating audit log for Patchi hosted mode.
+
+Writes structured JSON-lines to .patchi/hosted/audit.log.
+Rotates when file exceeds MAX_SIZE_BYTES, keeping MAX_BACKUPS old files.
+Each entry: {timestamp, event, actor, data}.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import time
+from pathlib import Path
+
+_LOG_FILE = ".patchi/hosted/audit.log"
+MAX_SIZE_BYTES = 5 * 1024 * 1024  # 5 MB
+MAX_BACKUPS = 3
+
+
+_log = logging.getLogger("patchi.core.audit_log")
+
+
+def _log_path(root: Path) -> Path:
+    return root / _LOG_FILE
+
+
+def _rotate(path: Path) -> None:
+    """Rotate old log files, keeping MAX_BACKUPS copies."""
+    # Shift existing backups down (remove oldest first to avoid conflicts)
+    for i in range(MAX_BACKUPS - 1, 0, -1):
+        src = Path(f"{path}.{i}")
+        dst = Path(f"{path}.{i + 1}")
+        if dst.exists():
+            dst.unlink()
+        if src.exists():
+            src.rename(dst)
+    # Move current log to .1
+    if path.exists():
+        dst = Path(f"{path}.1")
+        if dst.exists():
+            dst.unlink()
+        path.rename(dst)
+
+
+def write(root: Path, event: str, actor: str = "patchi", data: dict | None = None) -> None:
+    """
+    Append one audit entry. Rotates if log exceeds MAX_SIZE_BYTES.
+    Never raises — audit logging must not crash the caller.
+    """
+    try:
+        path = _log_path(root)
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        if path.exists() and path.stat().st_size >= MAX_SIZE_BYTES:
+            _rotate(path)
+
+        entry = json.dumps(
+            {
+                "timestamp": time.time(),
+                "event": event,
+                "actor": actor,
+                "data": data or {},
+            }
+        )
+        with path.open("a", encoding="utf-8") as f:
+            f.write(entry + "\n")
+    except Exception as e:
+        _log.warning("write failed: %s", e)
+
+
+def read_recent(root: Path, limit: int = 50) -> list[dict]:
+    """Return the most recent `limit` audit entries, newest first."""
+    path = _log_path(root)
+    if not path.exists():
+        return []
+    try:
+        lines = path.read_text(encoding="utf-8").strip().splitlines()
+        entries = []
+        for line in reversed(lines[-limit * 2 :]):
+            try:
+                entries.append(json.loads(line))
+                if len(entries) >= limit:
+                    break
+            except json.JSONDecodeError:
+                continue
+        return entries
+    except OSError:
+        return []
+
+
+def clear(root: Path) -> int:
+    """Delete current log and all backups. Returns number of files removed."""
+    path = _log_path(root)
+    removed = 0
+    for p in [path] + [Path(f"{path}.{i}") for i in range(1, MAX_BACKUPS + 2)]:
+        try:
+            if p.exists():
+                p.unlink()
+                removed += 1
+        except OSError:
+            pass
+    return removed

patchi/core/hosted/compliance_report.py

@@ -0,0 +1,148 @@
+"""
+Compliance Report Generator — Evidence packs for hosted mode.
+
+Aggregates scan results, health score, audit log, and domain control status
+into a single structured report suitable for auditors / customers.
+Deterministic: built entirely from data already on disk (no AI required).
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import UTC, datetime
+from pathlib import Path
+
+_log = logging.getLogger("patchi.hosted.compliance")
+
+# Map Patchi security agents to compliance framework domains
+_FRAMEWORK_MAPS: dict[str, dict[str, list[str]]] = {
+    "owasp-asvs": {
+        "V1: Architecture": ["AppMapperAgent", "BlastRadiusAgent", "PlanAuditorAgent"],
+        "V2: Authentication": [
+            "AuthenticationAuditAgent",
+            "JWTSecurityAgent",
+            "SessionManagementAgent",
+            "SecretScanner",
+        ],
+        "V3: Session Management": ["SessionManagementAgent", "JWTSecurityAgent"],
+        "V4: Access Control": ["AuthZAgent", "InjectionAgent"],
+        "V5: Validation & Encoding": ["InjectionAgent", "TaintAnalyzer", "SensitiveDataAgent"],
+        "V6: Stored Cryptography": ["CryptoAgent", "SecretsGuard"],
+        "V7: Error Handling & Logging": ["HistoryAgent", "GovernanceAgent"],
+        "V8: Data Protection": ["PrivacyAgent", "SensitiveDataAgent"],
+        "V9: Communications": ["NetworkAgent", "SSRFProtectionAgent", "CORSAuditor"],
+        "V10: Malicious Code": ["SupplyChainAgent", "DependencyVulnerabilityAgent"],
+        "V12: Files & Resources": ["MisconfigAgent", "ConfigAuditAgent"],
+        "V14: Configuration": [
+            "ConfigAuditAgent",
+            "HeaderAuditAgent",
+            "RateLimitAuditor",
+            "EnvVarValidator",
+        ],
+    },
+    "pci-dss": {
+        "Req 2: Secure Configs": ["ConfigAuditAgent", "MisconfigAgent", "HeaderAuditAgent"],
+        "Req 3: Protect Stored Data": ["SecretScanner", "SensitiveDataAgent", "CryptoAgent"],
+        "Req 4: Encrypt Transmission": ["NetworkAgent", "CryptoAgent"],
+        "Req 5: Malware Protection": ["SupplyChainAgent"],
+        "Req 6: Secure Development": [
+            "TaintAnalyzer",
+            "InjectionAgent",
+            "DependencyVulnerabilityAgent",
+        ],
+        "Req 8: Identify & Auth": ["AuthenticationAuditAgent", "JWTSecurityAgent"],
+        "Req 10: Log & Monitor": ["HistoryAgent", "GovernanceAgent", "EnvVarValidator"],
+    },
+}
+
+
+def generate_report(root: Path, standard: str = "owasp-asvs") -> dict:
+    """Build a structured compliance evidence report from disk state.
+
+    Returns {standard, generated_at, controls: [...], summary: {...}} where
+    each control row carries pass/fail/pending status plus the agent evidence
+    behind it. Never raises — missing data becomes 'pending'.
+    """
+    from patchi.core import memory as mem
+
+    framework = _FRAMEWORK_MAPS.get(standard)
+    if framework is None:
+        return {"error": f"Unknown standard '{standard}'. Supported: {', '.join(_FRAMEWORK_MAPS)}"}
+
+    scans = mem.get_scan_results(root)
+    brain = mem.get_brain(root)
+    health = brain.get("health_score", {})
+
+    def _agent_findings(agent: str) -> list[dict]:
+        return [f for f in scans.get(agent, {}).get("findings", []) if isinstance(f, dict)]
+
+    def _severity_counts(findings: list[dict]) -> dict[str, int]:
+        counts: dict[str, int] = {}
+        for f in findings:
+            sev = f.get("severity", "info")
+            counts[sev] = counts.get(sev, 0) + 1
+        return counts
+
+    sections: list[dict] = []
+    total_pass = total_fail = total_pending = 0
+
+    for section_name, agents in framework.items():
+        rows = []
+        for agent in agents:
+            findings = _agent_findings(agent)
+            counts = _severity_counts(findings)
+
+            if agent not in scans:
+                status, evidence = "pending", f"{agent} has not run yet"
+            elif counts.get("critical", 0) or counts.get("high", 0):
+                status = "fail"
+                evidence = (
+                    f"{counts.get('critical', 0)} critical / {counts.get('high', 0)} high "
+                    f"finding(s) from {agent}"
+                )
+            elif findings:
+                status = "pass_with_notes"
+                evidence = f"{len(findings)} low/medium finding(s) from {agent}"
+            else:
+                status = "pass"
+                evidence = f"{agent} ran clean"
+
+            if status == "fail":
+                total_fail += 1
+            elif status == "pending":
+                total_pending += 1
+            else:
+                total_pass += 1
+
+            rows.append({"control_agent": agent, "status": status, "evidence": evidence})
+
+        sections.append(
+            {
+                "section": section_name,
+                "controls": rows,
+                "failed": sum(1 for r in rows if r["status"] == "fail"),
+                "total": len(rows),
+            }
+        )
+
+    return {
+        "standard": standard,
+        "generated_at": datetime.now(UTC).isoformat(),
+        "project": {
+            "purpose": brain.get("project_purpose", ""),
+            "domain": brain.get("project_domain", ""),
+            "framework": brain.get("framework", ""),
+            "file_count": brain.get("file_count", 0),
+        },
+        "health_score": health.get("total"),
+        "summary": {
+            "sections": len(sections),
+            "controls_total": sum(s["total"] for s in sections),
+            "controls_failed": total_fail,
+            "controls_passed": total_pass,
+            "controls_pending": total_pending,
+            # Simple posture rollout: pass ratio over evaluated controls
+            "posture_pct": round(total_pass / max(1, total_pass + total_fail) * 100, 1),
+        },
+        "sections": sections,
+    }