nat-engine 1__py3-none-any.whl

mannf/core/anomaly/enhanced_detector.py

@@ -0,0 +1,541 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Enhanced Anomaly Detection with adaptive thresholds and classification.
+
+Learns the *normal* response profile for each endpoint and scores incoming
+responses on how anomalous they are.  Anomalies are classified into:
+
+* ``latency_spike``      — Response time significantly above learned baseline.
+* ``status_code_shift``  — Unexpected status code pattern change.
+* ``payload_anomaly``    — Response body structure deviates from learned norm.
+* ``error_rate_surge``   — Sudden increase in error rate.
+
+Thresholds adapt automatically based on the rolling baseline rather than being
+hardcoded, which prevents false positives on naturally slow endpoints.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import statistics
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+
+class AnomalyType(str, Enum):
+    """Taxonomy of detectable anomaly types."""
+
+    LATENCY_SPIKE = "latency_spike"
+    STATUS_CODE_SHIFT = "status_code_shift"
+    PAYLOAD_ANOMALY = "payload_anomaly"
+    ERROR_RATE_SURGE = "error_rate_surge"
+
+
+class Severity(str, Enum):
+    """Alert severity levels."""
+
+    INFO = "info"
+    WARNING = "warning"
+    CRITICAL = "critical"
+
+
+@dataclass
+class AnomalyAlert:
+    """A structured anomaly alert produced by the detector.
+
+    Attributes
+    ----------
+    endpoint:
+        The API endpoint where the anomaly was detected.
+    anomaly_type:
+        Classification of the detected anomaly.
+    severity:
+        How severe the anomaly is (info / warning / critical).
+    score:
+        Anomaly score in [0.0, 1.0]; 1.0 = maximally anomalous.
+    description:
+        Human-readable explanation of what was detected.
+    detected_at:
+        ISO 8601 timestamp of when the alert was generated.
+    scan_id:
+        Optional scan identifier linking the alert to a specific run.
+    details:
+        Additional structured information about the anomaly.
+    """
+
+    endpoint: str
+    anomaly_type: AnomalyType
+    severity: Severity
+    score: float
+    description: str
+    detected_at: str = field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )
+    scan_id: Optional[str] = None
+    details: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "endpoint": self.endpoint,
+            "anomaly_type": self.anomaly_type,
+            "severity": self.severity,
+            "score": round(self.score, 4),
+            "description": self.description,
+            "detected_at": self.detected_at,
+            "scan_id": self.scan_id,
+            "details": self.details,
+        }
+
+
+# ---------------------------------------------------------------------------
+# Per-endpoint baseline statistics
+# ---------------------------------------------------------------------------
+
+@dataclass
+class _EndpointBaseline:
+    """Rolling baseline stats for a single endpoint."""
+
+    latencies: List[float] = field(default_factory=list)
+    status_codes: List[int] = field(default_factory=list)
+    payload_sizes: List[int] = field(default_factory=list)
+    error_flags: List[bool] = field(default_factory=list)
+    seen_status_codes: Set[int] = field(default_factory=set)
+
+    # Adaptive thresholds (recomputed periodically)
+    latency_mean: float = 0.0
+    latency_std: float = 0.0
+    error_rate: float = 0.0
+    payload_size_mean: float = 0.0
+    payload_size_std: float = 0.0
+
+    _window: int = 50  # rolling window size
+
+    def update(
+        self,
+        latency_ms: float,
+        status_code: int,
+        payload_size: int,
+        is_error: bool,
+    ) -> None:
+        def _push(lst: list, val: Any) -> None:
+            lst.append(val)
+            if len(lst) > self._window:
+                lst.pop(0)
+
+        _push(self.latencies, latency_ms)
+        _push(self.status_codes, status_code)
+        _push(self.payload_sizes, payload_size)
+        _push(self.error_flags, is_error)
+        self.seen_status_codes.add(status_code)
+
+        if len(self.latencies) >= 3:
+            self.latency_mean = statistics.mean(self.latencies)
+            self.latency_std = statistics.stdev(self.latencies) if len(self.latencies) > 1 else 0.0
+        if self.payload_sizes:
+            self.payload_size_mean = statistics.mean(self.payload_sizes)
+            self.payload_size_std = (
+                statistics.stdev(self.payload_sizes) if len(self.payload_sizes) > 1 else 0.0
+            )
+        if self.error_flags:
+            self.error_rate = sum(self.error_flags) / len(self.error_flags)
+
+    @property
+    def is_warmed_up(self) -> bool:
+        """True once the baseline has enough data to compute thresholds."""
+        return len(self.latencies) >= 5
+
+
+# ---------------------------------------------------------------------------
+# Main detector
+# ---------------------------------------------------------------------------
+
+class EnhancedAnomalyDetector:
+    """Adaptive anomaly detector for API test responses.
+
+    Learns per-endpoint baseline response profiles and scores each new
+    response on how anomalous it is.  Anomaly thresholds adapt automatically
+    as the baseline evolves.
+
+    Parameters
+    ----------
+    latency_sigma_threshold:
+        Number of standard deviations above the mean latency to trigger a
+        ``latency_spike`` warning.  Default is 2σ (≈95th percentile).
+    latency_critical_sigma:
+        Standard deviations threshold for a *critical* latency spike.
+    error_rate_warning_threshold:
+        Error rate (0–1) at which an ``error_rate_surge`` warning fires.
+    error_rate_critical_threshold:
+        Error rate at which a *critical* surge fires.
+    payload_sigma_threshold:
+        Standard deviations from mean payload size to flag a payload anomaly.
+    alert_history_limit:
+        Maximum number of alerts retained in memory.
+    """
+
+    def __init__(
+        self,
+        latency_sigma_threshold: float = 2.0,
+        latency_critical_sigma: float = 4.0,
+        error_rate_warning_threshold: float = 0.3,
+        error_rate_critical_threshold: float = 0.6,
+        payload_sigma_threshold: float = 3.0,
+        alert_history_limit: int = 500,
+    ) -> None:
+        self._lat_warn_sigma = latency_sigma_threshold
+        self._lat_crit_sigma = latency_critical_sigma
+        self._err_warn = error_rate_warning_threshold
+        self._err_crit = error_rate_critical_threshold
+        self._payload_sigma = payload_sigma_threshold
+        self._alert_limit = alert_history_limit
+
+        self._baselines: Dict[str, _EndpointBaseline] = defaultdict(
+            _EndpointBaseline
+        )
+        self._alerts: List[AnomalyAlert] = []
+
+    # ------------------------------------------------------------------
+    # Core observe API
+    # ------------------------------------------------------------------
+
+    def observe(
+        self,
+        endpoint: str,
+        latency_ms: float,
+        status_code: int,
+        payload: Any = None,
+        is_error: bool = False,
+        scan_id: Optional[str] = None,
+    ) -> Tuple[float, List[AnomalyAlert]]:
+        """Record a new response observation and return anomaly signals.
+
+        Parameters
+        ----------
+        endpoint:
+            Target endpoint identifier.
+        latency_ms:
+            Response latency in milliseconds.
+        status_code:
+            HTTP status code returned.
+        payload:
+            Response body (used for payload size estimation).
+        is_error:
+            Whether this response counts as an error.
+        scan_id:
+            Optional scan identifier for alert attribution.
+
+        Returns
+        -------
+        (anomaly_score, new_alerts)
+            *anomaly_score* is in [0.0, 1.0].
+            *new_alerts* contains any new :class:`AnomalyAlert` objects
+            generated by this observation.
+        """
+        payload_size = self._estimate_payload_size(payload)
+        baseline = self._baselines[endpoint]
+
+        # Compute anomaly scores *before* updating baseline to avoid
+        # the new observation polluting its own threshold check
+        new_alerts: List[AnomalyAlert] = []
+        composite_score = 0.0
+
+        if baseline.is_warmed_up:
+            lat_score, lat_alert = self._check_latency(
+                endpoint, latency_ms, baseline, scan_id
+            )
+            sc_score, sc_alert = self._check_status_code(
+                endpoint, status_code, baseline, scan_id
+            )
+            pl_score, pl_alert = self._check_payload(
+                endpoint, payload_size, baseline, scan_id
+            )
+            err_score, err_alert = self._check_error_rate(
+                endpoint, is_error, baseline, scan_id
+            )
+
+            for alert in [lat_alert, sc_alert, pl_alert, err_alert]:
+                if alert is not None:
+                    new_alerts.append(alert)
+                    self._store_alert(alert)
+
+            composite_score = min(
+                1.0, (lat_score + sc_score + pl_score + err_score) / 4.0
+            )
+
+        # Now update baseline
+        baseline.update(latency_ms, status_code, payload_size, is_error)
+
+        return composite_score, new_alerts
+
+    # ------------------------------------------------------------------
+    # Anomaly score (no alert side-effect)
+    # ------------------------------------------------------------------
+
+    def score(
+        self,
+        endpoint: str,
+        latency_ms: float,
+        status_code: int,
+        payload: Any = None,
+        is_error: bool = False,
+    ) -> float:
+        """Return anomaly score for a response without updating the baseline."""
+        baseline = self._baselines.get(endpoint)
+        if baseline is None or not baseline.is_warmed_up:
+            return 0.0
+
+        lat_score, _ = self._check_latency(endpoint, latency_ms, baseline, None)
+        sc_score, _ = self._check_status_code(endpoint, status_code, baseline, None)
+        pl_score, _ = self._check_payload(
+            endpoint, self._estimate_payload_size(payload), baseline, None
+        )
+        err_score, _ = self._check_error_rate(endpoint, is_error, baseline, None)
+
+        return min(1.0, (lat_score + sc_score + pl_score + err_score) / 4.0)
+
+    # ------------------------------------------------------------------
+    # Latency check
+    # ------------------------------------------------------------------
+
+    def _check_latency(
+        self,
+        endpoint: str,
+        latency_ms: float,
+        baseline: _EndpointBaseline,
+        scan_id: Optional[str],
+    ) -> Tuple[float, Optional[AnomalyAlert]]:
+        mean = baseline.latency_mean
+        std = baseline.latency_std
+
+        # When std is zero (all observations identical) use a relative floor
+        # so minor fluctuations don't trigger false positives.
+        if std < 1e-6:
+            # When all observations are identical (std = 0), apply a relative
+            # floor of 10 % of the mean (min 1 ms) to prevent every minor
+            # fluctuation from triggering a false positive.
+            _RELATIVE_FLOOR = 0.10  # 10 % of mean
+            _ABS_MIN_MS = 1.0       # absolute minimum floor (milliseconds)
+            std = max(_ABS_MIN_MS, mean * _RELATIVE_FLOOR)
+
+        deviation = (latency_ms - mean) / std
+
+        if deviation < self._lat_warn_sigma:
+            return 0.0, None
+
+        # Normalize: at warning threshold → 0.3, at critical → 0.8, beyond → 1.0
+        score = min(
+            1.0,
+            0.3 + 0.5 * (deviation - self._lat_warn_sigma)
+            / max(1.0, self._lat_crit_sigma - self._lat_warn_sigma),
+        )
+        severity = (
+            Severity.CRITICAL if deviation >= self._lat_crit_sigma else Severity.WARNING
+        )
+        alert = AnomalyAlert(
+            endpoint=endpoint,
+            anomaly_type=AnomalyType.LATENCY_SPIKE,
+            severity=severity,
+            score=round(score, 4),
+            description=(
+                f"Latency {latency_ms:.1f}ms is {deviation:.1f}σ above the "
+                f"baseline mean of {mean:.1f}ms (σ={std:.1f}ms)"
+            ),
+            scan_id=scan_id,
+            details={
+                "latency_ms": latency_ms,
+                "baseline_mean_ms": round(mean, 2),
+                "baseline_std_ms": round(std, 2),
+                "sigma_deviation": round(deviation, 2),
+            },
+        )
+        return score, alert
+
+    # ------------------------------------------------------------------
+    # Status code check
+    # ------------------------------------------------------------------
+
+    def _check_status_code(
+        self,
+        endpoint: str,
+        status_code: int,
+        baseline: _EndpointBaseline,
+        scan_id: Optional[str],
+    ) -> Tuple[float, Optional[AnomalyAlert]]:
+        if status_code in baseline.seen_status_codes:
+            return 0.0, None
+
+        # Novel status code — severity depends on the code family
+        if status_code >= 500:
+            score, severity = 0.9, Severity.CRITICAL
+        elif status_code >= 400:
+            score, severity = 0.6, Severity.WARNING
+        else:
+            score, severity = 0.3, Severity.INFO
+
+        alert = AnomalyAlert(
+            endpoint=endpoint,
+            anomaly_type=AnomalyType.STATUS_CODE_SHIFT,
+            severity=severity,
+            score=score,
+            description=(
+                f"Unexpected status code {status_code} — not seen in baseline "
+                f"(known codes: {sorted(baseline.seen_status_codes)})"
+            ),
+            scan_id=scan_id,
+            details={
+                "status_code": status_code,
+                "known_status_codes": sorted(baseline.seen_status_codes),
+            },
+        )
+        return score, alert
+
+    # ------------------------------------------------------------------
+    # Payload size check
+    # ------------------------------------------------------------------
+
+    def _check_payload(
+        self,
+        endpoint: str,
+        payload_size: int,
+        baseline: _EndpointBaseline,
+        scan_id: Optional[str],
+    ) -> Tuple[float, Optional[AnomalyAlert]]:
+        mean = baseline.payload_size_mean
+        std = baseline.payload_size_std if baseline.payload_size_std > 0 else 1.0
+        deviation = abs(payload_size - mean) / std
+
+        if deviation < self._payload_sigma:
+            return 0.0, None
+
+        score = min(1.0, 0.4 + 0.3 * (deviation - self._payload_sigma) / self._payload_sigma)
+        severity = Severity.CRITICAL if deviation >= self._payload_sigma * 2 else Severity.WARNING
+        alert = AnomalyAlert(
+            endpoint=endpoint,
+            anomaly_type=AnomalyType.PAYLOAD_ANOMALY,
+            severity=severity,
+            score=round(score, 4),
+            description=(
+                f"Payload size {payload_size} bytes deviates {deviation:.1f}σ from "
+                f"baseline mean {mean:.0f}±{std:.0f} bytes"
+            ),
+            scan_id=scan_id,
+            details={
+                "payload_size": payload_size,
+                "baseline_mean_bytes": round(mean, 0),
+                "baseline_std_bytes": round(std, 0),
+                "sigma_deviation": round(deviation, 2),
+            },
+        )
+        return score, alert
+
+    # ------------------------------------------------------------------
+    # Error rate check
+    # ------------------------------------------------------------------
+
+    def _check_error_rate(
+        self,
+        endpoint: str,
+        is_error: bool,
+        baseline: _EndpointBaseline,
+        scan_id: Optional[str],
+    ) -> Tuple[float, Optional[AnomalyAlert]]:
+        if not is_error:
+            return 0.0, None
+
+        # Compute a provisional error rate including this observation
+        n = len(baseline.error_flags) + 1
+        errors = sum(baseline.error_flags) + 1
+        current_rate = errors / n
+
+        if current_rate < self._err_warn:
+            return 0.0, None
+
+        if current_rate >= self._err_crit:
+            score, severity = 0.9, Severity.CRITICAL
+        else:
+            score = 0.3 + 0.6 * (current_rate - self._err_warn) / max(
+                0.01, self._err_crit - self._err_warn
+            )
+            severity = Severity.WARNING
+
+        alert = AnomalyAlert(
+            endpoint=endpoint,
+            anomaly_type=AnomalyType.ERROR_RATE_SURGE,
+            severity=severity,
+            score=round(score, 4),
+            description=(
+                f"Error rate surging: {current_rate:.1%} "
+                f"(baseline: {baseline.error_rate:.1%}, "
+                f"threshold: {self._err_warn:.0%})"
+            ),
+            scan_id=scan_id,
+            details={
+                "current_error_rate": round(current_rate, 4),
+                "baseline_error_rate": round(baseline.error_rate, 4),
+                "warning_threshold": self._err_warn,
+                "critical_threshold": self._err_crit,
+            },
+        )
+        return score, alert
+
+    # ------------------------------------------------------------------
+    # Alert management
+    # ------------------------------------------------------------------
+
+    def _store_alert(self, alert: AnomalyAlert) -> None:
+        self._alerts.append(alert)
+        if len(self._alerts) > self._alert_limit:
+            self._alerts.pop(0)
+
+    def get_alerts(
+        self,
+        endpoint: Optional[str] = None,
+        scan_id: Optional[str] = None,
+        severity: Optional[Severity] = None,
+    ) -> List[AnomalyAlert]:
+        """Return stored anomaly alerts, optionally filtered."""
+        alerts = self._alerts
+        if endpoint is not None:
+            alerts = [a for a in alerts if a.endpoint == endpoint]
+        if scan_id is not None:
+            alerts = [a for a in alerts if a.scan_id == scan_id]
+        if severity is not None:
+            alerts = [a for a in alerts if a.severity == severity]
+        return list(alerts)
+
+    def get_alerts_dict(
+        self,
+        endpoint: Optional[str] = None,
+        scan_id: Optional[str] = None,
+        severity: Optional[Severity] = None,
+    ) -> List[Dict[str, Any]]:
+        """Return alerts as a list of JSON-serializable dicts."""
+        return [a.to_dict() for a in self.get_alerts(endpoint, scan_id, severity)]
+
+    def clear_alerts(self) -> None:
+        """Remove all stored alerts."""
+        self._alerts.clear()
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _estimate_payload_size(payload: Any) -> int:
+        if payload is None:
+            return 0
+        if isinstance(payload, (bytes, bytearray)):
+            return len(payload)
+        if isinstance(payload, str):
+            return len(payload.encode("utf-8"))
+        try:
+            return len(json.dumps(payload).encode("utf-8"))
+        except (TypeError, ValueError):
+            return len(str(payload).encode("utf-8"))

mannf/core/browser/__init__.py

@@ -0,0 +1,63 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Browser agent utilities for functional/UI testing."""
+
+from __future__ import annotations
+
+from mannf.core.browser.dom_snapshot import DOMElement, DOMSnapshot, build_selector, capture_dom_snapshot
+from mannf.core.browser.visual_comparer import (
+    VisualComparisonResult,
+    compare_screenshots,
+    load_baseline,
+    save_baseline,
+)
+from mannf.core.browser.accessibility_scanner import (
+    AccessibilityScanResult,
+    AccessibilityViolation,
+    scan_accessibility,
+)
+from mannf.core.browser.performance_metrics import (
+    PerformanceMetric,
+    PerformanceResult,
+    evaluate_metrics,
+    get_threshold_violations,
+    DEFAULT_THRESHOLDS,
+)
+from mannf.core.browser.scenario_generator import ScenarioConfigGenerator
+from mannf.core.browser.security_scenario_generator import SecurityScenarioGenerator
+from mannf.core.browser.reflection_analyzer import ReflectionAnalyzer, analyze_security_results
+from mannf.core.browser.ingestor_bridge import IngestorBridge
+
+__all__ = [
+    "DOMElement",
+    "DOMSnapshot",
+    "build_selector",
+    "capture_dom_snapshot",
+    # Visual comparison
+    "VisualComparisonResult",
+    "compare_screenshots",
+    "load_baseline",
+    "save_baseline",
+    # Accessibility scanning
+    "AccessibilityScanResult",
+    "AccessibilityViolation",
+    "scan_accessibility",
+    # Performance metrics
+    "PerformanceMetric",
+    "PerformanceResult",
+    "evaluate_metrics",
+    "get_threshold_violations",
+    "DEFAULT_THRESHOLDS",
+    # Scenario generation
+    "ScenarioConfigGenerator",
+    # Security scenario generation
+    "SecurityScenarioGenerator",
+    # Reflection analysis
+    "ReflectionAnalyzer",
+    "analyze_security_results",
+    # Ingestor bridge
+    "IngestorBridge",
+]