nat-engine 1__py3-none-any.whl

mannf/core/agents/test_agent.py

@@ -0,0 +1,249 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Test-generation agent.
+
+The :class:`TestAgent` uses its neural network to generate prioritised test
+cases for a given set of distributed service endpoints.  It listens for
+:attr:`MessageType.TEST_REQUEST` messages and responds with
+:attr:`MessageType.TEST_CASE` messages.
+
+It also listens for :attr:`MessageType.FEEDBACK` messages to update its
+neural network with observed rewards, so that over time it learns to generate
+test cases that are more likely to find defects.
+"""
+
+from __future__ import annotations
+
+import logging
+import random
+import string
+import uuid
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set
+
+import numpy as np
+
+from mannf.core.agents.base import BaseAgent
+from mannf.core.messaging.bus import MessageBus
+from mannf.core.messaging.messages import Message, MessageType
+from mannf.core.neural import NeuralNetwork
+from mannf.core.testing.models import TestCase
+
+if TYPE_CHECKING:
+    from mannf.llm.base import LLMProvider
+
+logger = logging.getLogger(__name__)
+
+
+class TestAgent(BaseAgent):
+    """Generates adaptive test cases for distributed service endpoints.
+
+    Parameters
+    ----------
+    agent_id:
+        Unique agent identifier.
+    bus:
+        Shared message bus.
+    service_names:
+        Names of the services this agent can target.
+    feature_dim:
+        Dimensionality of the input feature vector fed to the NN.
+    """
+
+    def __init__(
+        self,
+        agent_id: str,
+        bus: MessageBus,
+        service_names: List[str],
+        feature_dim: int = 8,
+        llm_provider: Optional["LLMProvider"] = None,
+    ) -> None:
+        network = NeuralNetwork(
+            layer_sizes=[feature_dim, 32, 16, len(service_names)],
+            hidden_activation="relu",
+            learning_rate=0.01,
+        )
+        super().__init__(agent_id, bus, network)
+        self.service_names = service_names
+        self.feature_dim = feature_dim
+        self._generated_count = 0
+        self.llm_provider = llm_provider
+
+    @property
+    def subscribed_types(self) -> Set[MessageType]:
+        return {MessageType.TEST_REQUEST, MessageType.FEEDBACK}
+
+    # ------------------------------------------------------------------
+    # Message handling
+    # ------------------------------------------------------------------
+
+    async def _handle_message(self, message: Message) -> None:
+        if message.type == MessageType.TEST_REQUEST:
+            test_case = self._generate_test_case(context=message.payload)
+            await self._publish(
+                message.reply(MessageType.TEST_CASE, test_case, sender_id=self.agent_id)
+            )
+        elif message.type == MessageType.FEEDBACK:
+            self._apply_feedback(message.payload)
+
+    async def _on_idle(self) -> None:
+        # Proactively generate a test case every idle cycle
+        test_case = self._generate_test_case()
+        await self._publish(Message(MessageType.TEST_CASE, self.agent_id, test_case))
+
+    # ------------------------------------------------------------------
+    # Test generation
+    # ------------------------------------------------------------------
+
+    def _generate_test_case(
+        self, context: Optional[Dict[str, Any]] = None
+    ) -> TestCase:
+        """Use the neural network to select a target and build a test case."""
+        # Build a context feature vector
+        ctx_vec = self._context_to_features(context)
+
+        # Forward pass: get unnormalised scores for each service
+        assert self.network is not None
+        scores = self.network.predict(ctx_vec)
+
+        # Softmax to get a probability distribution over services
+        exp_scores = np.exp(scores - scores.max())
+        probs = exp_scores / exp_scores.sum()
+
+        # Sample from the distribution (exploration built into softmax)
+        target_idx = int(np.random.choice(len(self.service_names), p=probs))
+        target = self.service_names[target_idx]
+        priority = float(probs[target_idx])
+
+        inputs = self._generate_inputs(target, context)
+        self._generated_count += 1
+
+        return TestCase(
+            target=target,
+            inputs=inputs,
+            expected_behavior=f"Service {target!r} should return a successful response",
+            priority=priority,
+            metadata={"generator": self.agent_id, "generation": self._generated_count},
+        )
+
+    async def _generate_test_case_with_llm(
+        self, context: Optional[Dict[str, Any]] = None
+    ) -> TestCase:
+        """Generate a test case, using LLM for inputs 20% of the time when available."""
+        # Build a context feature vector
+        ctx_vec = self._context_to_features(context)
+
+        # Forward pass: get unnormalised scores for each service
+        assert self.network is not None
+        scores = self.network.predict(ctx_vec)
+
+        # Softmax to get a probability distribution over services
+        exp_scores = np.exp(scores - scores.max())
+        probs = exp_scores / exp_scores.sum()
+
+        # Sample from the distribution (exploration built into softmax)
+        target_idx = int(np.random.choice(len(self.service_names), p=probs))
+        target = self.service_names[target_idx]
+        priority = float(probs[target_idx])
+
+        # Use LLM 20% of the time when available
+        use_llm = (
+            self.llm_provider is not None
+            and self.llm_provider.is_available()
+            and random.random() < 0.20
+        )
+
+        if use_llm:
+            assert self.llm_provider is not None
+            try:
+                endpoint_info = {
+                    "method": "POST",
+                    "path": target,
+                    "description": f"Service endpoint: {target}",
+                    "parameters": [],
+                    "schema": {},
+                }
+                raw_cases = await self.llm_provider.generate_test_cases(
+                    endpoint_info, {}, n=1
+                )
+                if raw_cases and isinstance(raw_cases[0].get("inputs"), dict):
+                    inputs = raw_cases[0]["inputs"]
+                    self._generated_count += 1
+                    return TestCase(
+                        target=target,
+                        inputs=inputs,
+                        expected_behavior=raw_cases[0].get(
+                            "expected_behavior",
+                            f"Service {target!r} should return a successful response",
+                        ),
+                        priority=priority,
+                        metadata={
+                            "generator": "llm",
+                            "generation": self._generated_count,
+                            "agent_id": self.agent_id,
+                        },
+                    )
+            except Exception as exc:  # noqa: BLE001
+                logger.warning("TestAgent %s: LLM input generation failed: %s", self.agent_id, exc)
+
+        inputs = self._generate_inputs(target, context)
+        self._generated_count += 1
+
+        return TestCase(
+            target=target,
+            inputs=inputs,
+            expected_behavior=f"Service {target!r} should return a successful response",
+            priority=priority,
+            metadata={"generator": self.agent_id, "generation": self._generated_count},
+        )
+
+    def _context_to_features(self, context: Optional[Dict[str, Any]]) -> np.ndarray:
+        """Convert an optional context dictionary into a feature vector."""
+        vec = np.zeros(self.feature_dim)
+        if context is None:
+            return vec
+        # Encode pass_rate if available
+        vec[0] = float(context.get("pass_rate", 0.5))
+        # Encode defect_detection_rate
+        vec[1] = float(context.get("defect_detection_rate", 0.0))
+        # Encode step count (normalised)
+        vec[2] = min(float(context.get("step_count", 0)) / 1000.0, 1.0)
+        # Hash target hint if present
+        target_hint = context.get("target_hint", "")
+        if target_hint and target_hint in self.service_names:
+            idx = self.service_names.index(target_hint)
+            vec[3] = idx / max(len(self.service_names) - 1, 1)
+        return vec
+
+    @staticmethod
+    def _generate_inputs(target: str, context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+        """Produce plausible random inputs for a service invocation."""
+        return {
+            "request_id": str(uuid.uuid4()),
+            "user_id": random.randint(1, 10_000),
+            "payload_size": random.randint(1, 1024),
+            "action": random.choice(["create", "read", "update", "delete", "list"]),
+            "auth_token": "".join(random.choices(string.ascii_letters, k=16)),
+        }
+
+    # ------------------------------------------------------------------
+    # Feedback / online learning
+    # ------------------------------------------------------------------
+
+    def _apply_feedback(self, feedback: Optional[Dict[str, Any]]) -> None:
+        """Update the network weights given a feedback payload."""
+        if not feedback:
+            return
+        try:
+            x = np.array(feedback["features"], dtype=float)
+            y = np.array(feedback["target"], dtype=float)
+            assert self.network is not None
+            self.network.train_step(x, y)
+        except (KeyError, TypeError, ValueError) as exc:
+            logger.warning("TestAgent %s: invalid feedback payload – %s", self.agent_id, exc)
+
+    @property
+    def generated_count(self) -> int:
+        return self._generated_count

mannf/core/agents/visual_regression_agent.py

@@ -0,0 +1,311 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""VisualRegressionAgent – detects visual regressions in functional test runs.
+
+Listens for :attr:`~mannf.core.messaging.messages.MessageType.SCREENSHOT_CAPTURED`
+events published by :class:`~mannf.core.agents.browser_executor_agent.BrowserExecutorAgent`,
+compares each screenshot against the stored baseline, and publishes:
+
+* :attr:`~MessageType.VISUAL_BASELINE_SAVED` when a new baseline is stored
+  (first run or ``update_baselines=True``).
+* :attr:`~MessageType.VISUAL_DIFF_DETECTED` when a screenshot exceeds the
+  configured diff threshold.
+
+Screenshot bytes must be deposited via :meth:`register_screenshot` before
+(or immediately after) the :attr:`~MessageType.SCREENSHOT_CAPTURED` message
+arrives so the agent can retrieve them for comparison.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+from typing import Any
+
+from mannf.core.agents.bdi_agent import BDIAgent
+from mannf.core.browser.visual_comparer import (
+    VisualComparisonResult,
+    compare_screenshots,
+    load_baseline,
+    save_baseline,
+)
+from mannf.core.messaging.bus import MessageBus
+from mannf.core.messaging.messages import Message, MessageType
+
+logger = logging.getLogger(__name__)
+
+
+class VisualRegressionAgent(BDIAgent):
+    """BDI agent that performs visual regression testing.
+
+    Parameters
+    ----------
+    agent_id:
+        Unique identifier for this agent.
+    bus:
+        Shared message bus.
+    service_names:
+        Services in the system under test (forwarded to :class:`BDIAgent`).
+    baseline_dir:
+        Directory where baseline PNG files are stored.
+    diff_threshold:
+        Maximum allowed diff percentage (0–100) before a visual regression is
+        flagged.
+    update_baselines:
+        When ``True``, every screenshot is saved as the new baseline instead
+        of being compared.
+    """
+
+    def __init__(
+        self,
+        agent_id: str,
+        bus: MessageBus,
+        service_names: list[str],
+        baseline_dir: str = ".nat/visual_baselines",
+        diff_threshold: float = 0.1,
+        update_baselines: bool = False,
+    ) -> None:
+        super().__init__(agent_id=agent_id, bus=bus, service_names=service_names)
+
+        self._baseline_dir = baseline_dir
+        self._diff_threshold = diff_threshold
+        self._update_baselines = update_baselines
+
+        # url -> latest PNG bytes deposited by browser agents / orchestrator
+        self._screenshot_registry: dict[str, bytes] = {}
+
+        # Tracking for the visual report
+        self._results: list[dict[str, Any]] = []
+        self._baselines_created = 0
+        self._tested_urls: set[str] = set()
+
+    # ------------------------------------------------------------------
+    # BDIAgent contract
+    # ------------------------------------------------------------------
+
+    @property
+    def subscribed_types(self) -> set[MessageType]:
+        return {
+            MessageType.SCREENSHOT_CAPTURED,
+            MessageType.INTERACTION_RESULT,
+            MessageType.BELIEF_UPDATE,
+        }
+
+    async def _handle_message(self, message: Message) -> None:
+        if message.type == MessageType.SCREENSHOT_CAPTURED:
+            await self._on_screenshot_captured(message)
+        elif message.type == MessageType.INTERACTION_RESULT:
+            self._on_interaction_result(message)
+        elif message.type == MessageType.BELIEF_UPDATE:
+            payload = message.payload or {}
+            peer_beliefs = payload.get("beliefs", {})
+            if peer_beliefs:
+                self._absorb_peer_beliefs(message.sender_id, peer_beliefs)
+
+    # ------------------------------------------------------------------
+    # Screenshot registry
+    # ------------------------------------------------------------------
+
+    def register_screenshot(self, url: str, png_bytes: bytes) -> None:
+        """Deposit screenshot bytes so they can be retrieved on the next
+        :attr:`~MessageType.SCREENSHOT_CAPTURED` message for *url*.
+
+        Parameters
+        ----------
+        url:
+            The page URL the screenshot was taken from.
+        png_bytes:
+            Raw PNG image bytes.
+        """
+        self._screenshot_registry[url] = png_bytes
+
+    # ------------------------------------------------------------------
+    # Message handlers
+    # ------------------------------------------------------------------
+
+    async def _on_screenshot_captured(self, message: Message) -> None:
+        payload = message.payload or {}
+        url: str = payload.get("url", "")
+
+        png_bytes = self._screenshot_registry.get(url)
+        if png_bytes is None:
+            logger.debug(
+                "VisualRegressionAgent %s: no screenshot bytes registered for %s — skipping",
+                self.agent_id, url,
+            )
+            return
+
+        await self.compare_with_baseline(url, png_bytes)
+
+    def _on_interaction_result(self, message: Message) -> None:
+        payload = message.payload or {}
+        url: str = payload.get("url", "")
+        if url:
+            self._tested_urls.add(url)
+
+    # ------------------------------------------------------------------
+    # Core comparison logic
+    # ------------------------------------------------------------------
+
+    async def compare_with_baseline(
+        self, url: str, screenshot: bytes
+    ) -> VisualComparisonResult:
+        """Compare *screenshot* against the stored baseline for *url*.
+
+        If no baseline exists, or :attr:`update_baselines` is ``True``, the
+        screenshot is saved as the new baseline and
+        :attr:`~MessageType.VISUAL_BASELINE_SAVED` is published.
+
+        Otherwise the screenshot is compared pixel-by-pixel; if the diff
+        exceeds :attr:`_diff_threshold`,
+        :attr:`~MessageType.VISUAL_DIFF_DETECTED` is published.
+
+        Parameters
+        ----------
+        url:
+            The page URL used to derive the baseline filename.
+        screenshot:
+            Raw PNG bytes of the current screenshot.
+
+        Returns
+        -------
+        VisualComparisonResult
+            The comparison outcome.
+        """
+        filename = self._url_to_filename(url)
+        baseline_path = os.path.join(self._baseline_dir, f"{filename}.png")
+
+        existing_baseline = load_baseline(baseline_path)
+
+        if existing_baseline is None or self._update_baselines:
+            # Save as new baseline
+            save_baseline(baseline_path, screenshot)
+            self._baselines_created += 1
+            logger.debug(
+                "VisualRegressionAgent %s: saved baseline for %s → %s",
+                self.agent_id, url, baseline_path,
+            )
+            await self._publish(
+                Message(
+                    MessageType.VISUAL_BASELINE_SAVED,
+                    sender_id=self.agent_id,
+                    payload={"url": url, "baseline_path": baseline_path},
+                )
+            )
+            result = VisualComparisonResult(
+                matched=True,
+                diff_percentage=0.0,
+                diff_image=None,
+                baseline_size=(0, 0),
+                actual_size=(0, 0),
+                threshold=self._diff_threshold,
+            )
+            self._results.append(
+                {
+                    "url": url,
+                    "matched": True,
+                    "diff_percentage": 0.0,
+                    "baseline_path": baseline_path,
+                    "baseline_created": True,
+                }
+            )
+            return result
+
+        # Compare against existing baseline
+        result = await compare_screenshots(
+            existing_baseline, screenshot, self._diff_threshold
+        )
+
+        if not result.matched:
+            logger.info(
+                "VisualRegressionAgent %s: visual diff detected for %s "
+                "(diff=%.2f%% > threshold=%.2f%%)",
+                self.agent_id, url, result.diff_percentage, self._diff_threshold,
+            )
+            await self._publish(
+                Message(
+                    MessageType.VISUAL_DIFF_DETECTED,
+                    sender_id=self.agent_id,
+                    payload={
+                        "url": url,
+                        "diff_percentage": result.diff_percentage,
+                        "threshold": self._diff_threshold,
+                        "baseline_path": baseline_path,
+                    },
+                )
+            )
+
+            # Persist diff image alongside the baseline
+            if result.diff_image is not None:
+                diff_path = os.path.join(self._baseline_dir, f"{filename}_diff.png")
+                save_baseline(diff_path, result.diff_image)
+                logger.debug(
+                    "VisualRegressionAgent %s: diff image saved to %s",
+                    self.agent_id, diff_path,
+                )
+
+        self._results.append(
+            {
+                "url": url,
+                "matched": result.matched,
+                "diff_percentage": result.diff_percentage,
+                "baseline_path": baseline_path,
+                "baseline_created": False,
+            }
+        )
+        return result
+
+    # ------------------------------------------------------------------
+    # Reporting
+    # ------------------------------------------------------------------
+
+    def get_visual_report(self) -> dict[str, Any]:
+        """Return a summary of all visual comparisons performed.
+
+        Returns
+        -------
+        dict
+            ``{total_comparisons, passed, failed, baselines_created,
+               results: [{url, matched, diff_percentage, baseline_path}...]}``
+        """
+        passed = sum(1 for r in self._results if r["matched"])
+        failed = len(self._results) - passed
+        return {
+            "total_comparisons": len(self._results),
+            "passed": passed,
+            "failed": failed,
+            "baselines_created": self._baselines_created,
+            "results": [
+                {
+                    "url": r["url"],
+                    "matched": r["matched"],
+                    "diff_percentage": r["diff_percentage"],
+                    "baseline_path": r["baseline_path"],
+                }
+                for r in self._results
+            ],
+        }
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _url_to_filename(url: str) -> str:
+        """Derive a safe filesystem filename from *url* using a SHA-256 prefix.
+
+        Parameters
+        ----------
+        url:
+            Any URL string.
+
+        Returns
+        -------
+        str
+            A 16-character hexadecimal string safe for use as a filename.
+        """
+        return hashlib.sha256(url.encode()).hexdigest()[:16]