nat-engine 1__py3-none-any.whl

mannf/core/agents/oracle_agent.py

@@ -0,0 +1,150 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Oracle agent.
+
+The :class:`OracleAgent` acts as the *test oracle*: it receives
+:attr:`MessageType.TEST_RESULT` messages from the orchestrator and uses its
+neural network to predict whether a test outcome should be considered a
+genuine defect or an expected/acceptable failure.
+
+It publishes :attr:`MessageType.TEST_VERDICT` messages that the orchestrator
+records in the test suite and uses to update the adaptive controller.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, Optional, Set
+
+import numpy as np
+
+from mannf.core.agents.base import BaseAgent
+from mannf.core.messaging.bus import MessageBus
+from mannf.core.messaging.messages import Message, MessageType
+from mannf.core.neural import NeuralNetwork
+from mannf.core.testing.models import TestCase, TestResult
+
+logger = logging.getLogger(__name__)
+
+_INPUT_DIM = 5   # features derived from a TestResult
+
+
+class OracleAgent(BaseAgent):
+    """Classifies test outcomes as genuine defects or expected failures.
+
+    The oracle uses a binary classifier neural network that is trained online:
+    initially it trusts all raw pass/fail flags, but as it accumulates
+    experience it learns patterns (e.g. transient latency spikes should not
+    be classified as defects).
+
+    Verdict payload format::
+
+        {
+            "test_case_id": str,
+            "is_defect": bool,
+            "confidence": float,   # [0, 1]
+            "raw_passed": bool,
+        }
+    """
+
+    def __init__(self, agent_id: str, bus: MessageBus) -> None:
+        network = NeuralNetwork(
+            layer_sizes=[_INPUT_DIM, 16, 8, 1],
+            hidden_activation="relu",
+            learning_rate=0.005,
+        )
+        super().__init__(agent_id, bus, network)
+        self._verdict_count = 0
+        self._defect_count = 0
+
+    @property
+    def subscribed_types(self) -> Set[MessageType]:
+        return {MessageType.TEST_RESULT}
+
+    # ------------------------------------------------------------------
+    # Message handling
+    # ------------------------------------------------------------------
+
+    async def _handle_message(self, message: Message) -> None:
+        if message.type == MessageType.TEST_RESULT:
+            payload = message.payload
+            result: TestResult = payload.get("result") if isinstance(payload, dict) else payload
+
+            if not isinstance(result, TestResult):
+                logger.warning("OracleAgent: unexpected payload type %s", type(result))
+                return
+
+            test_case: Optional[TestCase] = (
+                payload.get("test_case") if isinstance(payload, dict) else None
+            )
+            verdict = self._evaluate(result, test_case)
+            await self._publish(
+                Message(MessageType.TEST_VERDICT, self.agent_id, verdict)
+            )
+
+    # ------------------------------------------------------------------
+    # Evaluation
+    # ------------------------------------------------------------------
+
+    def _evaluate(
+        self,
+        result: TestResult,
+        test_case: Optional[TestCase] = None,
+    ) -> Dict[str, Any]:
+        """Return a verdict dictionary."""
+        features = self._extract_features(result, test_case)
+        assert self.network is not None
+        raw_score = float(self.network.predict(features)[0])
+        # Sigmoid to get probability of being a defect
+        confidence = 1.0 / (1.0 + np.exp(-raw_score))
+        is_defect = bool(confidence >= 0.5)
+
+        # Online training: use raw pass/fail as ground truth label
+        label = np.array([0.0 if result.passed else 1.0])
+        self.network.train_step(features, label)
+
+        self._verdict_count += 1
+        if is_defect:
+            self._defect_count += 1
+
+        return {
+            "test_case_id": result.test_case_id,
+            "is_defect": is_defect,
+            "confidence": round(float(confidence), 4),
+            "raw_passed": result.passed,
+            "error": result.error,
+        }
+
+    @staticmethod
+    def _extract_features(
+        result: TestResult,
+        test_case: Optional[TestCase],
+    ) -> np.ndarray:
+        """Encode result + optional test_case into a feature vector."""
+        vec = np.zeros(_INPUT_DIM)
+        vec[0] = 0.0 if result.passed else 1.0
+        vec[1] = 1.0 if result.error else 0.0
+        vec[2] = min(result.execution_time_ms / 500.0, 1.0)  # normalised
+        if test_case is not None:
+            vec[3] = test_case.priority
+            vec[4] = len(test_case.inputs) / 10.0
+        return vec
+
+    # ------------------------------------------------------------------
+    # Stats
+    # ------------------------------------------------------------------
+
+    @property
+    def verdict_count(self) -> int:
+        return self._verdict_count
+
+    @property
+    def defect_count(self) -> int:
+        return self._defect_count
+
+    @property
+    def defect_rate(self) -> float:
+        return self._defect_count / self._verdict_count if self._verdict_count else 0.0

mannf/core/agents/performance_testing_agent.py

@@ -0,0 +1,279 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""PerformanceTestingAgent – BDI agent that measures Web Vitals performance.
+
+Listens for :attr:`~mannf.core.messaging.messages.MessageType.INTERACTION_RESULT`
+messages published by browser agents and evaluates any performance metrics
+embedded in the payload.  Metrics can also be deposited programmatically via
+:meth:`register_metrics`.
+
+After each evaluation the agent publishes:
+
+* :attr:`~MessageType.PERFORMANCE_METRICS_CAPTURED` — always, with all metric
+  ratings.
+* :attr:`~MessageType.PERFORMANCE_THRESHOLD_VIOLATED` — only when one or more
+  metrics are rated ``"poor"``.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from mannf.core.agents.bdi_agent import BDIAgent
+from mannf.core.browser.performance_metrics import (
+    PerformanceResult,
+    evaluate_metrics,
+    get_threshold_violations,
+)
+from mannf.core.messaging.bus import MessageBus
+from mannf.core.messaging.messages import Message, MessageType
+
+logger = logging.getLogger(__name__)
+
+
+class PerformanceTestingAgent(BDIAgent):
+    """BDI agent that evaluates Web Vitals performance metrics.
+
+    Parameters
+    ----------
+    agent_id:
+        Unique identifier for this agent.
+    bus:
+        Shared message bus.
+    service_names:
+        Services in the system under test (forwarded to :class:`BDIAgent`).
+    custom_thresholds:
+        Optional threshold overrides mapping metric name to
+        ``(threshold_good, threshold_poor, unit)``.  Any metric name absent
+        from this dict falls back to :data:`~mannf.core.browser.performance_metrics.DEFAULT_THRESHOLDS`.
+    min_performance_score:
+        Minimum passing performance score in the range 0–100.  Evaluations
+        whose :attr:`~PerformanceResult.performance_score` is below this
+        value are counted as ``"failed"`` in :meth:`get_performance_report`.
+    """
+
+    def __init__(
+        self,
+        agent_id: str,
+        bus: MessageBus,
+        service_names: list[str],
+        custom_thresholds: dict[str, tuple[float, float, str]] | None = None,
+        min_performance_score: float = 70.0,
+    ) -> None:
+        super().__init__(agent_id=agent_id, bus=bus, service_names=service_names)
+
+        self._custom_thresholds = custom_thresholds
+        self._min_performance_score = min_performance_score
+
+        # url -> raw metrics deposited by callers / orchestrator
+        self._metrics_registry: dict[str, dict[str, float]] = {}
+
+        # Evaluated results accumulated across all processed URLs
+        self._results: list[PerformanceResult] = []
+
+    # ------------------------------------------------------------------
+    # BDIAgent contract
+    # ------------------------------------------------------------------
+
+    @property
+    def subscribed_types(self) -> set[MessageType]:
+        return {
+            MessageType.INTERACTION_RESULT,
+            MessageType.SCREENSHOT_CAPTURED,
+            MessageType.BELIEF_UPDATE,
+        }
+
+    async def _handle_message(self, message: Message) -> None:
+        if message.type == MessageType.INTERACTION_RESULT:
+            await self._on_interaction_result(message)
+        elif message.type == MessageType.SCREENSHOT_CAPTURED:
+            # No-op — retained for future enrichment.
+            pass
+        elif message.type == MessageType.BELIEF_UPDATE:
+            payload = message.payload or {}
+            peer_beliefs = payload.get("beliefs", {})
+            if peer_beliefs:
+                self._absorb_peer_beliefs(message.sender_id, peer_beliefs)
+
+    # ------------------------------------------------------------------
+    # Metrics registry
+    # ------------------------------------------------------------------
+
+    def register_metrics(self, url: str, raw_metrics: dict[str, float]) -> None:
+        """Deposit raw performance metrics for the given URL.
+
+        The metrics will be evaluated when
+        :meth:`evaluate_url_performance` is called, or automatically when an
+        :attr:`~MessageType.INTERACTION_RESULT` message arrives for *url*.
+
+        Parameters
+        ----------
+        url:
+            The page URL the metrics were collected from.
+        raw_metrics:
+            Dict mapping metric names to measured values.
+        """
+        self._metrics_registry[url] = raw_metrics
+
+    # ------------------------------------------------------------------
+    # Message handlers
+    # ------------------------------------------------------------------
+
+    async def _on_interaction_result(self, message: Message) -> None:
+        payload = message.payload or {}
+        url: str = payload.get("url", "")
+        performance_data: dict[str, float] | None = payload.get("performance")
+
+        if performance_data:
+            await self.evaluate_url_performance(url, performance_data)
+        elif url in self._metrics_registry:
+            await self.evaluate_url_performance(url, self._metrics_registry[url])
+
+    # ------------------------------------------------------------------
+    # Core evaluation logic
+    # ------------------------------------------------------------------
+
+    async def evaluate_url_performance(
+        self, url: str, raw_metrics: dict[str, float]
+    ) -> PerformanceResult:
+        """Evaluate metrics for *url* and publish results on the bus.
+
+        Parameters
+        ----------
+        url:
+            Page URL (used for logging and the published payload).
+        raw_metrics:
+            Dict mapping metric names to measured values.
+
+        Returns
+        -------
+        PerformanceResult
+            The evaluation outcome (also stored internally for reporting).
+        """
+        result = evaluate_metrics(raw_metrics, url=url, custom_thresholds=self._custom_thresholds)
+        self._results.append(result)
+
+        passed = result.performance_score >= self._min_performance_score
+        violations = get_threshold_violations(result)
+
+        logger.info(
+            "PerformanceTestingAgent %s: url=%s score=%.1f poor_count=%d %s",
+            self.agent_id,
+            url,
+            result.performance_score,
+            result.poor_count,
+            "PASS" if passed else "FAIL",
+        )
+
+        await self._publish(
+            Message(
+                MessageType.PERFORMANCE_METRICS_CAPTURED,
+                sender_id=self.agent_id,
+                payload={
+                    "url": url,
+                    "performance_score": result.performance_score,
+                    "page_load_time_ms": result.page_load_time_ms,
+                    "passed": passed,
+                    "good_count": result.good_count,
+                    "poor_count": result.poor_count,
+                    "metrics": [
+                        {
+                            "name": m.name,
+                            "value": m.value,
+                            "unit": m.unit,
+                            "rating": m.rating,
+                            "threshold_good": m.threshold_good,
+                            "threshold_poor": m.threshold_poor,
+                        }
+                        for m in result.metrics
+                    ],
+                },
+            )
+        )
+
+        if violations:
+            await self._publish(
+                Message(
+                    MessageType.PERFORMANCE_THRESHOLD_VIOLATED,
+                    sender_id=self.agent_id,
+                    payload={
+                        "url": url,
+                        "violations": [
+                            {
+                                "name": m.name,
+                                "value": m.value,
+                                "unit": m.unit,
+                                "threshold_poor": m.threshold_poor,
+                            }
+                            for m in violations
+                        ],
+                    },
+                )
+            )
+
+        return result
+
+    # ------------------------------------------------------------------
+    # Reporting
+    # ------------------------------------------------------------------
+
+    def get_performance_report(self) -> dict[str, Any]:
+        """Return a summary of all performance evaluations performed.
+
+        Returns
+        -------
+        dict
+            ``{total_evaluations, passed, failed, avg_performance_score,
+               threshold_violations, results: [...]}``
+        """
+        total = len(self._results)
+        passed_count = sum(
+            1 for r in self._results
+            if r.performance_score >= self._min_performance_score
+        )
+        failed_count = total - passed_count
+        avg_score = (
+            round(
+                sum(r.performance_score for r in self._results) / total,
+                1,
+            )
+            if total > 0
+            else 100.0
+        )
+        all_violations = [
+            {"url": r.url, "metric": m.name, "value": m.value, "unit": m.unit}
+            for r in self._results
+            for m in get_threshold_violations(r)
+        ]
+
+        return {
+            "total_evaluations": total,
+            "passed": passed_count,
+            "failed": failed_count,
+            "avg_performance_score": avg_score,
+            "threshold_violations": all_violations,
+            "results": [
+                {
+                    "url": r.url,
+                    "performance_score": r.performance_score,
+                    "page_load_time_ms": r.page_load_time_ms,
+                    "passed": r.performance_score >= self._min_performance_score,
+                    "good_count": r.good_count,
+                    "poor_count": r.poor_count,
+                    "metrics": [
+                        {
+                            "name": m.name,
+                            "value": m.value,
+                            "unit": m.unit,
+                            "rating": m.rating,
+                        }
+                        for m in r.metrics
+                    ],
+                }
+                for r in self._results
+            ],
+        }

mannf/core/agents/planner_agent.py

@@ -0,0 +1,128 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Planner Agent – generates candidate test objectives.
+
+The PlannerAgent is responsible for formulating testing tasks and submitting
+them to the CoordinatorAgent for allocation.  It uses its Predictor network
+and current belief state to identify the most fault-prone services (Thesis
+§4.2, §4.4).
+
+Task announcement payload::
+
+    {
+        "task_id": str,
+        "target": str,          # service name
+        "priority": float,      # expected informational gain
+        "risk_estimate": float, # NN-derived fault-proneness score
+        "beliefs": dict,        # snapshot of planner's beliefs
+    }
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import Any, Dict, List, Set
+
+import numpy as np
+
+from mannf.core.agents.bdi_agent import BDIAgent
+from mannf.core.messaging.bus import MessageBus
+from mannf.core.messaging.messages import Message, MessageType
+from mannf.core.testing.models import TestCase
+
+logger = logging.getLogger(__name__)
+
+
+class PlannerAgent(BDIAgent):
+    """Formulates test objectives and requests CoordinatorAgent to schedule them.
+
+    On each idle cycle the planner consults its belief state and NN to identify
+    high-risk services and emits TASK_ANNOUNCEMENT messages.
+    """
+
+    def __init__(
+        self,
+        agent_id: str,
+        bus: MessageBus,
+        service_names: List[str],
+        tasks_per_cycle: int = 2,
+    ) -> None:
+        super().__init__(agent_id, bus, service_names)
+        self._tasks_per_cycle = tasks_per_cycle
+        self._announced_tasks: Dict[str, Dict[str, Any]] = {}
+
+    @property
+    def subscribed_types(self) -> Set[MessageType]:
+        return {
+            MessageType.BELIEF_UPDATE,      # absorb peer beliefs
+            MessageType.CONTRACT_RESULT,    # learn from execution outcomes
+        }
+
+    # ------------------------------------------------------------------
+    # Message handling
+    # ------------------------------------------------------------------
+
+    async def _handle_message(self, message: Message) -> None:
+        if message.type == MessageType.BELIEF_UPDATE:
+            payload = message.payload or {}
+            self._absorb_peer_beliefs(message.sender_id, payload.get("beliefs", {}))
+
+        elif message.type == MessageType.CONTRACT_RESULT:
+            self._process_contract_result(message.payload)
+            # Re-announce tasks immediately after a contract completes
+            # so the pipeline stays full without waiting for idle timeout
+            await self._announce_tasks()
+
+    async def _on_idle(self) -> None:
+        self._deliberate()
+        await self._announce_tasks()
+
+    # ------------------------------------------------------------------
+    # Task announcement
+    # ------------------------------------------------------------------
+
+    async def _announce_tasks(self) -> None:
+        """Announce the top-N tasks based on current belief priorities."""
+        targets = self.beliefs.top_k(self._tasks_per_cycle)
+        for target in targets:
+            risk = self.beliefs.fault_likelihood.get(target, 0.5)
+            task = {
+                "task_id": str(uuid.uuid4()),
+                "target": target,
+                "priority": risk,
+                "risk_estimate": risk,
+                "beliefs": self.beliefs.snapshot(),
+                "planner_id": self.agent_id,
+            }
+            self._announced_tasks[task["task_id"]] = task
+            await self._publish(
+                Message(MessageType.TASK_ANNOUNCEMENT, self.agent_id, task)
+            )
+            logger.debug(
+                "PlannerAgent %s: announced task for %s (risk=%.3f)",
+                self.agent_id, target, risk,
+            )
+
+    # ------------------------------------------------------------------
+    # Learning from results
+    # ------------------------------------------------------------------
+
+    def _process_contract_result(self, payload: Dict[str, Any]) -> None:
+        if not payload:
+            return
+        target = payload.get("target", "")
+        passed = payload.get("passed", True)
+        error = payload.get("error")
+        if not target:
+            return
+        # Evidence: failures/errors increase fault-likelihood
+        evidence = 0.0 if passed else (0.5 if error else 1.0)
+        self._update_belief_from_result(target, evidence)
+        logger.debug(
+            "PlannerAgent %s: belief for %s updated to %.3f",
+            self.agent_id, target, self.beliefs.fault_likelihood.get(target, 0.5),
+        )