nat-engine 1__py3-none-any.whl

mannf/core/prioritization/belief_prioritizer.py

@@ -0,0 +1,231 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Belief-Driven Test Prioritizer.
+
+Uses BDI agent belief states to rank test cases by predicted fault likelihood,
+concentrating testing effort on the highest-risk endpoints first.
+
+Supported strategies
+--------------------
+- ``risk_first``   — Sort by descending risk score (default).
+- ``round_robin``  — Cycle evenly across all endpoints.
+- ``exploration``  — ε-greedy: exploit high-risk endpoints most of the time,
+                     but occasionally explore lower-risk ones to avoid blind
+                     spots (mirrors the BDI exploration quota from §4.5).
+"""
+
+from __future__ import annotations
+
+import logging
+import random
+from collections import defaultdict
+from enum import Enum
+from typing import Dict, List, Optional
+
+from mannf.core.agents.belief_state import BeliefState
+from mannf.core.testing.models import TestCase, TestResult
+
+logger = logging.getLogger(__name__)
+
+
+class PrioritizationStrategy(str, Enum):
+    """Available test prioritization strategies."""
+
+    RISK_FIRST = "risk_first"
+    ROUND_ROBIN = "round_robin"
+    EXPLORATION = "exploration"
+
+
+class BeliefPrioritizer:
+    """Ranks and prioritizes test cases using BDI agent belief states.
+
+    Parameters
+    ----------
+    strategy:
+        Which prioritization strategy to apply.
+    exploration_epsilon:
+        For the ``exploration`` strategy — probability of picking a
+        non-highest-risk endpoint (0 = fully exploit, 1 = fully random).
+    epsilon_decay:
+        Multiplicative decay applied to *exploration_epsilon* after each
+        :meth:`record_result` call.
+    min_epsilon:
+        Floor for *exploration_epsilon* after decay.
+    """
+
+    def __init__(
+        self,
+        strategy: PrioritizationStrategy | str = PrioritizationStrategy.RISK_FIRST,
+        exploration_epsilon: float = 0.2,
+        epsilon_decay: float = 0.99,
+        min_epsilon: float = 0.05,
+    ) -> None:
+        self.strategy = PrioritizationStrategy(strategy)
+        self.epsilon = exploration_epsilon
+        self.epsilon_decay = epsilon_decay
+        self.min_epsilon = min_epsilon
+
+        # Historical fault tracking per endpoint target
+        self._fault_counts: Dict[str, int] = defaultdict(int)
+        self._run_counts: Dict[str, int] = defaultdict(int)
+
+        # Round-robin index per endpoint (for round_robin strategy)
+        self._rr_index: int = 0
+
+    # ------------------------------------------------------------------
+    # Core prioritization
+    # ------------------------------------------------------------------
+
+    def prioritize(
+        self,
+        test_cases: List[TestCase],
+        belief_states: Optional[List[BeliefState]] = None,
+    ) -> List[TestCase]:
+        """Return *test_cases* re-ordered by the configured strategy.
+
+        Parameters
+        ----------
+        test_cases:
+            Unordered list of test cases to prioritize.
+        belief_states:
+            Current BDI agent belief states used to derive per-endpoint risk
+            scores.  If *None*, only historical fault rates are used.
+
+        Returns
+        -------
+        List[TestCase]
+            A new list with the same elements, sorted highest-risk first
+            (or in the strategy-appropriate order).
+        """
+        if not test_cases:
+            return []
+
+        risk_scores = self._compute_risk_scores(test_cases, belief_states)
+
+        if self.strategy == PrioritizationStrategy.RISK_FIRST:
+            return self._sort_risk_first(test_cases, risk_scores)
+        if self.strategy == PrioritizationStrategy.ROUND_ROBIN:
+            return self._sort_round_robin(test_cases)
+        if self.strategy == PrioritizationStrategy.EXPLORATION:
+            return self._sort_exploration(test_cases, risk_scores)
+
+        # Fallback
+        return test_cases[:]
+
+    def get_risk_scores(
+        self,
+        test_cases: List[TestCase],
+        belief_states: Optional[List[BeliefState]] = None,
+    ) -> Dict[str, float]:
+        """Return per-endpoint risk scores (0.0–1.0) for the given test cases."""
+        return self._compute_risk_scores(test_cases, belief_states)
+
+    # ------------------------------------------------------------------
+    # Learning
+    # ------------------------------------------------------------------
+
+    def record_result(self, result: TestResult, target: str) -> None:
+        """Update historical fault rates with the observed test result.
+
+        Parameters
+        ----------
+        result:
+            Outcome of the executed test.
+        target:
+            Endpoint target string (e.g. ``"/users"``).
+        """
+        self._run_counts[target] += 1
+        if not result.passed or result.error:
+            self._fault_counts[target] += 1
+
+        # Decay exploration rate
+        self.epsilon = max(self.min_epsilon, self.epsilon * self.epsilon_decay)
+
+    def historical_fault_rate(self, target: str) -> float:
+        """Return the observed fault rate for *target* (0.0–1.0)."""
+        runs = self._run_counts.get(target, 0)
+        if runs == 0:
+            return 0.5  # Uninformed prior
+        return self._fault_counts.get(target, 0) / runs
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _compute_risk_scores(
+        self,
+        test_cases: List[TestCase],
+        belief_states: Optional[List[BeliefState]],
+    ) -> Dict[str, float]:
+        """Compute a composite risk score per unique endpoint target."""
+        targets = {tc.target for tc in test_cases}
+        scores: Dict[str, float] = {}
+
+        for target in targets:
+            # Component 1: historical fault rate
+            hist_rate = self.historical_fault_rate(target)
+
+            # Component 2: BDI belief state signal (average across agents)
+            belief_signal = 0.5  # uninformed prior
+            if belief_states:
+                signals = []
+                for bs in belief_states:
+                    val = bs.fault_likelihood.get(target)
+                    if val is not None:
+                        signals.append(val)
+                if signals:
+                    belief_signal = sum(signals) / len(signals)
+
+            # Component 3: test case priority field (if set by upstream)
+            tc_priorities = [
+                tc.priority for tc in test_cases if tc.target == target
+            ]
+            tc_priority = (
+                sum(tc_priorities) / len(tc_priorities) if tc_priorities else 0.5
+            )
+
+            # Weighted composite (belief state has highest weight)
+            score = 0.4 * belief_signal + 0.4 * hist_rate + 0.2 * tc_priority
+            scores[target] = min(1.0, max(0.0, score))
+
+        return scores
+
+    def _sort_risk_first(
+        self, test_cases: List[TestCase], risk_scores: Dict[str, float]
+    ) -> List[TestCase]:
+        return sorted(
+            test_cases,
+            key=lambda tc: risk_scores.get(tc.target, 0.5),
+            reverse=True,
+        )
+
+    def _sort_round_robin(self, test_cases: List[TestCase]) -> List[TestCase]:
+        """Interleave test cases across endpoints in a round-robin fashion."""
+        by_target: Dict[str, List[TestCase]] = defaultdict(list)
+        for tc in test_cases:
+            by_target[tc.target].append(tc)
+
+        ordered_targets = sorted(by_target.keys())
+        result: List[TestCase] = []
+        while any(by_target[t] for t in ordered_targets):
+            for t in ordered_targets:
+                if by_target[t]:
+                    result.append(by_target[t].pop(0))
+        return result
+
+    def _sort_exploration(
+        self, test_cases: List[TestCase], risk_scores: Dict[str, float]
+    ) -> List[TestCase]:
+        """ε-greedy: exploit high-risk ordering most of the time."""
+        if random.random() < self.epsilon:
+            # Explore: random shuffle
+            shuffled = test_cases[:]
+            random.shuffle(shuffled)
+            logger.debug("BeliefPrioritizer: exploration — shuffled %d tests", len(shuffled))
+            return shuffled
+        # Exploit: risk-first
+        logger.debug("BeliefPrioritizer: exploitation — risk-first ordering")
+        return self._sort_risk_first(test_cases, risk_scores)

mannf/core/prioritization/risk_scorer.py

@@ -0,0 +1,430 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Endpoint Risk Scoring Engine.
+
+Computes and maintains per-endpoint composite risk scores by combining:
+
+* **Historical fault rate** — percentage of past test failures per endpoint.
+* **Belief state confidence** — BDI agent confidence in fault likelihood.
+* **Response time anomalies** — endpoints with increasing latency trends.
+* **Error pattern diversity** — endpoints showing varied error types.
+* **Connection-oriented metrics** — disconnect rate and message latency for
+  WebSocket and gRPC streaming endpoints.
+
+The :meth:`get_risk_report` method returns a structured, JSON-serializable
+report suitable for the REST API and future dashboard.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from collections import defaultdict
+from typing import Any, Dict, List, Optional, Tuple
+
+from mannf.core.agents.belief_state import BeliefState
+from mannf.core.testing.models import TestResult
+
+# Sliding window sizes for history tracking
+_LATENCY_WINDOW = 50
+_MSG_LATENCY_WINDOW = 50
+_SCORE_HISTORY_WINDOW = 10
+
+
+class TrendDirection(str):
+    """Trend direction constants."""
+
+    IMPROVING = "improving"
+    DEGRADING = "degrading"
+    STABLE = "stable"
+
+
+class RiskScorer:
+    """Computes and tracks per-endpoint composite risk scores.
+
+    Parameters
+    ----------
+    belief_weight:
+        Weight for the BDI belief state signal (0–1).
+    fault_rate_weight:
+        Weight for the historical fault rate signal (0–1).
+    latency_weight:
+        Weight for the response-time anomaly signal (0–1).
+    diversity_weight:
+        Weight for the error pattern diversity signal (0–1).
+    connection_weight:
+        Weight for the connection-oriented signal (WebSocket disconnect rate,
+        gRPC stream errors) (0–1).  Defaults to 0.10.
+
+    The five weights should sum to 1.0 for a normalized composite score;
+    they are automatically re-normalized if they don't.
+    """
+
+    def __init__(
+        self,
+        belief_weight: float = 0.30,
+        fault_rate_weight: float = 0.30,
+        latency_weight: float = 0.20,
+        diversity_weight: float = 0.10,
+        connection_weight: float = 0.10,
+    ) -> None:
+        total = belief_weight + fault_rate_weight + latency_weight + diversity_weight + connection_weight
+        if total <= 0:
+            total = 1.0
+        self._w_belief = belief_weight / total
+        self._w_fault = fault_rate_weight / total
+        self._w_latency = latency_weight / total
+        self._w_diversity = diversity_weight / total
+        self._w_connection = connection_weight / total
+
+        # Per-endpoint counters
+        self._run_counts: Dict[str, int] = defaultdict(int)
+        self._fault_counts: Dict[str, int] = defaultdict(int)
+
+        # Response time history (sliding window of last 50 samples)
+        self._latency_history: Dict[str, List[float]] = defaultdict(list)
+        self._latency_window = _LATENCY_WINDOW
+
+        # Error type sets per endpoint (tracks diversity)
+        self._error_types: Dict[str, set] = defaultdict(set)
+
+        # Snapshot of the last computed risk scores
+        self._last_scores: Dict[str, float] = {}
+
+        # Score history for trend detection (last 10 snapshots per endpoint)
+        self._score_history: Dict[str, List[float]] = defaultdict(list)
+
+        # Connection-oriented metrics (WebSocket / gRPC streaming)
+        self._disconnect_counts: Dict[str, int] = defaultdict(int)
+        self._message_latency_history: Dict[str, List[float]] = defaultdict(list)
+        self._msg_latency_window = _MSG_LATENCY_WINDOW
+
+    # ------------------------------------------------------------------
+    # Ingestion
+    # ------------------------------------------------------------------
+
+    def record(
+        self,
+        endpoint: str,
+        result: TestResult,
+        belief_states: Optional[List[BeliefState]] = None,
+    ) -> float:
+        """Record a test result and return the updated risk score for *endpoint*.
+
+        Parameters
+        ----------
+        endpoint:
+            Target endpoint identifier (e.g. ``"/users GET"``).
+        result:
+            Test execution outcome.
+        belief_states:
+            Optional BDI belief states from the current agents.
+
+        Returns
+        -------
+        float
+            Updated composite risk score in [0.0, 1.0].
+        """
+        self._run_counts[endpoint] += 1
+
+        # Track faults
+        if not result.passed or result.error:
+            self._fault_counts[endpoint] += 1
+            if result.error:
+                # Capture error type from message prefix (first word)
+                err_type = result.error.split()[0] if result.error else "unknown"
+                self._error_types[endpoint].add(err_type)
+
+        # Track latency
+        latency_ms = result.execution_time_ms
+        history = self._latency_history[endpoint]
+        history.append(latency_ms)
+        if len(history) > self._latency_window:
+            history.pop(0)
+
+        score = self._compute(endpoint, belief_states)
+        self._last_scores[endpoint] = score
+
+        hist = self._score_history[endpoint]
+        hist.append(score)
+        if len(hist) > 10:
+            hist.pop(0)
+
+        return score
+
+    def record_disconnect(self, endpoint: str) -> None:
+        """Record an unexpected connection disconnect for *endpoint*.
+
+        Should be called by :class:`~mannf.product.integrations.websocket_sut.WebSocketSUT`
+        and :class:`~mannf.product.integrations.grpc_sut.GrpcSUT` when a
+        connection drops unexpectedly.
+
+        Parameters
+        ----------
+        endpoint:
+            The WebSocket channel path or gRPC method path.
+        """
+        self._disconnect_counts[endpoint] += 1
+
+    def record_message_latency(self, endpoint: str, latency_ms: float) -> None:
+        """Record a send→receive message latency sample for *endpoint*.
+
+        Used to track WebSocket and gRPC streaming message round-trip times
+        independently from HTTP request latencies.
+
+        Parameters
+        ----------
+        endpoint:
+            The WebSocket channel path or gRPC method path.
+        latency_ms:
+            Observed latency in milliseconds.
+        """
+        hist = self._message_latency_history[endpoint]
+        hist.append(latency_ms)
+        if len(hist) > self._msg_latency_window:
+            hist.pop(0)
+
+    # ------------------------------------------------------------------
+    # Scoring
+    # ------------------------------------------------------------------
+
+    def score(
+        self,
+        endpoint: str,
+        belief_states: Optional[List[BeliefState]] = None,
+    ) -> float:
+        """Return the current composite risk score for *endpoint* (0.0–1.0)."""
+        return self._compute(endpoint, belief_states)
+
+    def _compute(
+        self,
+        endpoint: str,
+        belief_states: Optional[List[BeliefState]],
+    ) -> float:
+        fault_rate = self._fault_rate(endpoint)
+        belief_signal = self._belief_signal(endpoint, belief_states)
+        latency_signal = self._latency_signal(endpoint)
+        diversity_signal = self._diversity_signal(endpoint)
+        connection_signal = self._connection_signal(endpoint)
+
+        score = (
+            self._w_belief * belief_signal
+            + self._w_fault * fault_rate
+            + self._w_latency * latency_signal
+            + self._w_diversity * diversity_signal
+            + self._w_connection * connection_signal
+        )
+        return min(1.0, max(0.0, score))
+
+    def _fault_rate(self, endpoint: str) -> float:
+        runs = self._run_counts.get(endpoint, 0)
+        if runs == 0:
+            return 0.5  # uninformed prior
+        return self._fault_counts.get(endpoint, 0) / runs
+
+    def _belief_signal(
+        self,
+        endpoint: str,
+        belief_states: Optional[List[BeliefState]],
+    ) -> float:
+        if not belief_states:
+            return self._fault_rate(endpoint)  # fall back to fault rate
+        signals = [
+            bs.fault_likelihood.get(endpoint, 0.5)
+            for bs in belief_states
+        ]
+        return sum(signals) / len(signals)
+
+    def _latency_signal(self, endpoint: str) -> float:
+        """Normalised latency anomaly score: 0 = stable, 1 = rapidly growing."""
+        history = self._latency_history.get(endpoint, [])
+        if len(history) < 4:
+            return 0.0
+        # Split into two halves and compare means
+        mid = len(history) // 2
+        first_half = history[:mid]
+        second_half = history[mid:]
+        mean_first = sum(first_half) / len(first_half)
+        mean_second = sum(second_half) / len(second_half)
+        if mean_first <= 0:
+            return 0.0
+        growth_ratio = (mean_second - mean_first) / mean_first
+        # Normalize: 50% growth → score of 0.5; cap at 1.0
+        return min(1.0, max(0.0, growth_ratio))
+
+    def _diversity_signal(self, endpoint: str) -> float:
+        """Error type diversity → 0 (no errors) to 1.0 (many distinct types)."""
+        n_types = len(self._error_types.get(endpoint, set()))
+        if n_types == 0:
+            return 0.0
+        # log scale: 1 type → ~0.3, 3 types → ~0.8, 7+ types → 1.0
+        return min(1.0, math.log(n_types + 1) / math.log(8))
+
+    def _connection_signal(self, endpoint: str) -> float:
+        """Connection-oriented risk signal (0 = healthy, 1 = high disconnect/latency).
+
+        Combines:
+        * **Disconnect rate** — fraction of ``record_disconnect`` calls relative
+          to total runs.  A service that drops 50% of connections scores 0.5
+          on this dimension.
+        * **Message latency growth** — like ``_latency_signal`` but applied to
+          the message-latency samples recorded via
+          :meth:`record_message_latency`.
+        """
+        runs = self._run_counts.get(endpoint, 0)
+        disconnect_count = self._disconnect_counts.get(endpoint, 0)
+        disconnect_rate = disconnect_count / max(runs, 1)
+
+        # Message latency growth signal
+        msg_history = self._message_latency_history.get(endpoint, [])
+        msg_latency_signal = 0.0
+        if len(msg_history) >= 4:
+            mid = len(msg_history) // 2
+            first_half = msg_history[:mid]
+            second_half = msg_history[mid:]
+            mean_first = sum(first_half) / len(first_half)
+            mean_second = sum(second_half) / len(second_half)
+            if mean_first > 0:
+                growth = (mean_second - mean_first) / mean_first
+                msg_latency_signal = min(1.0, max(0.0, growth))
+
+        return min(1.0, (disconnect_rate + msg_latency_signal) / 2.0)
+
+    # ------------------------------------------------------------------
+    # Trend detection
+    # ------------------------------------------------------------------
+
+    def trend(self, endpoint: str) -> str:
+        """Return trend direction for *endpoint*: improving / degrading / stable."""
+        history = self._score_history.get(endpoint, [])
+        if len(history) < 3:
+            return TrendDirection.STABLE
+        recent = history[-3:]
+        older = history[:-3] or [recent[0]]
+        mean_recent = sum(recent) / len(recent)
+        mean_older = sum(older) / len(older)
+        delta = mean_recent - mean_older
+        if delta > 0.05:
+            return TrendDirection.DEGRADING
+        if delta < -0.05:
+            return TrendDirection.IMPROVING
+        return TrendDirection.STABLE
+
+    # ------------------------------------------------------------------
+    # Report
+    # ------------------------------------------------------------------
+
+    def _confidence_interval(self, endpoint: str) -> Tuple[float, float]:
+        """Compute a 95% confidence interval for the risk score of *endpoint*.
+
+        Uses ``mean ± 1.96 * std / √n`` from the score history window.
+
+        Edge cases:
+        - ``n == 0``: return (0.0, 1.0) — widest possible range.
+        - ``n == 1``: return (score, score) — no variance estimable.
+        - ``std == 0``: return (score, score) — all identical scores.
+
+        Returns
+        -------
+        tuple[float, float]
+            ``(ci_low, ci_high)`` clamped to [0.0, 1.0].
+        """
+        history = self._score_history.get(endpoint, [])
+        n = len(history)
+        if n == 0:
+            return (0.0, 1.0)
+        if n == 1:
+            s = history[0]
+            return (s, s)
+        mean = sum(history) / n
+        variance = sum((x - mean) ** 2 for x in history) / (n - 1)  # sample variance
+        std = math.sqrt(variance)
+        if std == 0.0:
+            return (mean, mean)
+        margin = 1.96 * std / math.sqrt(n)
+        return (
+            max(0.0, mean - margin),
+            min(1.0, mean + margin),
+        )
+
+    def get_risk_report(
+        self,
+        belief_states: Optional[List[BeliefState]] = None,
+    ) -> List[Dict[str, Any]]:
+        """Return a structured risk report for all tracked endpoints.
+
+        Returns
+        -------
+        list of dict
+            Each entry contains:
+            ``endpoint``, ``risk_score``, ``fault_rate``, ``belief_signal``,
+            ``latency_signal``, ``diversity_signal``, ``connection_signal``,
+            ``disconnect_count``, ``trend``, ``run_count``, ``fault_count``,
+            ``confidence_interval_low``, ``confidence_interval_high``,
+            ``sample_count``.
+            Sorted by descending risk score.
+        """
+        all_endpoints = set(self._run_counts.keys())
+
+        report = []
+        for ep in all_endpoints:
+            fault_rate = self._fault_rate(ep)
+            belief_signal = self._belief_signal(ep, belief_states)
+            latency_signal = self._latency_signal(ep)
+            diversity_signal = self._diversity_signal(ep)
+            connection_signal = self._connection_signal(ep)
+            risk_score = (
+                self._w_belief * belief_signal
+                + self._w_fault * fault_rate
+                + self._w_latency * latency_signal
+                + self._w_diversity * diversity_signal
+                + self._w_connection * connection_signal
+            )
+            risk_score = min(1.0, max(0.0, risk_score))
+            self._last_scores[ep] = risk_score
+
+            # Confidence interval: mean ± 1.96 * std / √n from score history
+            ci_low, ci_high = self._confidence_interval(ep)
+
+            report.append(
+                {
+                    "endpoint": ep,
+                    "risk_score": round(risk_score, 4),
+                    "fault_rate": round(fault_rate, 4),
+                    "belief_signal": round(belief_signal, 4),
+                    "latency_signal": round(latency_signal, 4),
+                    "diversity_signal": round(diversity_signal, 4),
+                    "connection_signal": round(connection_signal, 4),
+                    "disconnect_count": self._disconnect_counts.get(ep, 0),
+                    "trend": self.trend(ep),
+                    "run_count": self._run_counts.get(ep, 0),
+                    "fault_count": self._fault_counts.get(ep, 0),
+                    "confidence_interval_low": round(ci_low, 4),
+                    "confidence_interval_high": round(ci_high, 4),
+                    "sample_count": len(self._score_history.get(ep, [])),
+                }
+            )
+
+        report.sort(key=lambda x: x["risk_score"], reverse=True)
+        return report
+
+    def to_json(self, belief_states: Optional[List[BeliefState]] = None) -> str:
+        """Return the risk report serialized as a JSON string."""
+        return json.dumps(self.get_risk_report(belief_states), indent=2)
+
+    # ------------------------------------------------------------------
+    # Convenience accessors
+    # ------------------------------------------------------------------
+
+    def top_k(self, k: int, belief_states: Optional[List[BeliefState]] = None) -> List[Tuple[str, float]]:
+        """Return the *k* highest-risk endpoints as (endpoint, score) tuples."""
+        report = self.get_risk_report(belief_states)
+        return [(r["endpoint"], r["risk_score"]) for r in report[:k]]
+
+    @property
+    def all_endpoints(self) -> List[str]:
+        """All endpoints for which data has been recorded."""
+        return list(self._run_counts.keys())

mannf/core/reporting/__init__.py

@@ -0,0 +1,12 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""NAT reporting sub-package – unified test report generation."""
+
+from __future__ import annotations
+
+from mannf.core.reporting.unified_report import UnifiedReportGenerator
+
+__all__ = ["UnifiedReportGenerator"]