nat-engine 1__py3-none-any.whl

mannf/core/functional_orchestrator.py

@@ -0,0 +1,428 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""FunctionalTestOrchestrator – high-level lifecycle manager for browser-based
+functional testing.
+
+Analogous to :class:`~mannf.core.nat_orchestrator.NATOrchestrator` but
+purpose-built for Playwright-driven UI tests:
+
+  * Spins up N :class:`~mannf.core.agents.browser_executor_agent.BrowserExecutorAgent`
+    instances and one :class:`~mannf.core.agents.browser_coordinator_agent.BrowserCoordinatorAgent`.
+  * Dispatches a list of ``{url, steps}`` task dicts through the coordinator.
+  * Monitors ``INTERACTION_RESULT`` messages for completion.
+  * Returns a structured report with pass/fail counts, per-task outcomes,
+    timing, screenshots and DOM snapshots.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from typing import Any
+
+from mannf.core.agents.browser_coordinator_agent import BrowserCoordinatorAgent
+from mannf.core.agents.browser_executor_agent import BrowserExecutorAgent
+from mannf.core.agents.visual_regression_agent import VisualRegressionAgent
+from mannf.core.agents.accessibility_scanner_agent import AccessibilityScannerAgent
+from mannf.core.agents.performance_testing_agent import PerformanceTestingAgent
+from mannf.core.messaging.bus import MessageBus
+from mannf.core.messaging.messages import Message, MessageType
+from mannf.core.reporting.unified_report import UnifiedReportGenerator
+
+logger = logging.getLogger(__name__)
+
+
+class FunctionalTestOrchestrator:
+    """Manages the full lifecycle for browser-based functional testing.
+
+    Parameters
+    ----------
+    num_browser_agents:
+        Number of :class:`BrowserExecutorAgent` instances to create.
+    browser_type:
+        Playwright browser family: ``"chromium"``, ``"firefox"``, or
+        ``"webkit"``.
+    headless:
+        Run browsers in headless mode (no visible window).
+    max_tasks:
+        Maximum number of functional tests to execute before stopping.
+    bid_timeout_s:
+        ECNP bid collection window (seconds).
+    enable_visual_regression:
+        When ``True``, create a :class:`VisualRegressionAgent` and include
+        visual regression results in the :meth:`report` output.
+    baseline_dir:
+        Directory to store baseline PNG screenshots (used when
+        *enable_visual_regression* is ``True``).
+    diff_threshold:
+        Maximum allowed diff percentage before a visual regression is flagged.
+    update_baselines:
+        When ``True``, always save new screenshots as baselines instead of
+        comparing them.
+    enable_accessibility_scan:
+        When ``True``, create an :class:`AccessibilityScannerAgent` and include
+        accessibility results in the :meth:`report` output.
+    min_compliance_score:
+        Minimum compliance percentage (0–100) used by
+        :class:`AccessibilityScannerAgent` when *enable_accessibility_scan*
+        is ``True``.
+    enable_performance_testing:
+        When ``True``, create a :class:`PerformanceTestingAgent` and include
+        performance results in the :meth:`report` output.
+    min_performance_score:
+        Minimum performance score (0–100) used by
+        :class:`PerformanceTestingAgent` when *enable_performance_testing*
+        is ``True``.
+    custom_perf_thresholds:
+        Optional threshold overrides for :class:`PerformanceTestingAgent`.
+    """
+
+    def __init__(
+        self,
+        num_browser_agents: int = 2,
+        browser_type: str = "chromium",
+        headless: bool = True,
+        max_tasks: int = 50,
+        bid_timeout_s: float = 0.05,
+        enable_visual_regression: bool = False,
+        baseline_dir: str = ".nat/visual_baselines",
+        diff_threshold: float = 0.1,
+        update_baselines: bool = False,
+        enable_accessibility_scan: bool = False,
+        min_compliance_score: float = 80.0,
+        enable_performance_testing: bool = False,
+        min_performance_score: float = 70.0,
+        custom_perf_thresholds: dict[str, tuple[float, float, str]] | None = None,
+        llm_provider: Any | None = None,
+        root_cause_quota: int = -1,
+    ) -> None:
+        self._num_browser_agents = num_browser_agents
+        self._browser_type = browser_type
+        self._headless = headless
+        self._max_tasks = max_tasks
+        self._enable_visual_regression = enable_visual_regression
+        self._enable_accessibility_scan = enable_accessibility_scan
+        self._enable_performance_testing = enable_performance_testing
+        self._llm_provider = llm_provider
+        self._root_cause_quota = root_cause_quota
+
+        self._bus = MessageBus()
+
+        # Browser executor agents
+        self._browser_agents: list[BrowserExecutorAgent] = [
+            BrowserExecutorAgent(
+                agent_id=f"browser-executor-{i}",
+                bus=self._bus,
+                service_names=["browser"],
+                browser_type=browser_type,
+                headless=headless,
+            )
+            for i in range(num_browser_agents)
+        ]
+
+        # Browser coordinator
+        self._coordinator = BrowserCoordinatorAgent(
+            agent_id="browser-coordinator-0",
+            bus=self._bus,
+            service_names=["browser"],
+            bid_timeout_s=bid_timeout_s,
+        )
+
+        # Optional visual regression agent
+        self._visual_agent: VisualRegressionAgent | None = None
+        if enable_visual_regression:
+            self._visual_agent = VisualRegressionAgent(
+                agent_id="visual-regression-0",
+                bus=self._bus,
+                service_names=["browser"],
+                baseline_dir=baseline_dir,
+                diff_threshold=diff_threshold,
+                update_baselines=update_baselines,
+            )
+
+        # Optional accessibility scanner agent
+        self._accessibility_agent: AccessibilityScannerAgent | None = None
+        if enable_accessibility_scan:
+            self._accessibility_agent = AccessibilityScannerAgent(
+                agent_id="accessibility-scanner-0",
+                bus=self._bus,
+                service_names=["browser"],
+                min_compliance_score=min_compliance_score,
+            )
+
+        # Optional performance testing agent
+        self._performance_agent: PerformanceTestingAgent | None = None
+        if enable_performance_testing:
+            self._performance_agent = PerformanceTestingAgent(
+                agent_id="performance-testing-0",
+                bus=self._bus,
+                service_names=["browser"],
+                custom_thresholds=custom_perf_thresholds,
+                min_performance_score=min_performance_score,
+            )
+
+        # Subscribe orchestrator to INTERACTION_RESULT for completion tracking
+        subscribed_types: set[MessageType] = {MessageType.INTERACTION_RESULT}
+        if enable_accessibility_scan:
+            subscribed_types.add(MessageType.ACCESSIBILITY_SCAN_RESULT)
+
+        self._result_queue: asyncio.Queue[Message] = self._bus.subscribe(
+            "functional-orchestrator",
+            subscribed_types,
+        )
+
+        self._completed = 0
+        self._start_time: float | None = None
+
+    # ------------------------------------------------------------------
+    # Main entry point
+    # ------------------------------------------------------------------
+
+    async def run(self, tasks: list[dict[str, Any]]) -> dict[str, Any]:
+        """Dispatch browser tasks and wait for all to complete.
+
+        Parameters
+        ----------
+        tasks:
+            List of task dicts, each with the shape::
+
+                {
+                    "url": "https://example.com",
+                    "steps": [
+                        {"action": "fill", "selector": "#email", "value": "..."},
+                        {"action": "click", "selector": "#submit"},
+                    ],
+                }
+
+        Returns
+        -------
+        dict
+            Summary with ``passed``, ``failed``, ``total``, ``tasks``,
+            ``elapsed_seconds``.
+        """
+        num_tasks = min(len(tasks), self._max_tasks)
+        self._start_time = time.monotonic()
+
+        all_agents = self._browser_agents + [self._coordinator]  # type: ignore[list-item]
+        if self._visual_agent is not None:
+            all_agents = all_agents + [self._visual_agent]  # type: ignore[list-item]
+        if self._accessibility_agent is not None:
+            all_agents = all_agents + [self._accessibility_agent]  # type: ignore[list-item]
+        if self._performance_agent is not None:
+            all_agents = all_agents + [self._performance_agent]  # type: ignore[list-item]
+
+        # Start agent run loops (each calls start() internally)
+        agent_tasks = [asyncio.create_task(a.run()) for a in all_agents]
+
+        # Yield so all run() tasks execute their start() and register subscriptions
+        await asyncio.sleep(0)
+
+        # Start browsers on all executor agents
+        for browser_agent in self._browser_agents:
+            await browser_agent.start_browser()
+
+        # Allow subscriptions to propagate
+        await asyncio.sleep(0)
+
+        try:
+            # Dispatch tasks through the coordinator
+            dispatched_ids: list[str] = []
+            for task in tasks[:num_tasks]:
+                task_id = await self._coordinator.dispatch_browser_task(
+                    url=task["url"],
+                    steps=task.get("steps", []),
+                )
+                dispatched_ids.append(task_id)
+
+            # Wait for all dispatched tasks to produce results
+            await self._monitor_completions(num_tasks)
+
+            # Yield briefly so coordinator can drain its inbox (it processes
+            # INTERACTION_RESULT / DOM_SNAPSHOT / SCREENSHOT_CAPTURED that
+            # were queued while the orchestrator was handling completions)
+            await asyncio.sleep(0.05)
+
+        finally:
+            # Stop browsers
+            for browser_agent in self._browser_agents:
+                await browser_agent.stop_browser()
+
+            # Stop all agents
+            for agent in all_agents:
+                agent._running = False
+            await asyncio.gather(*agent_tasks, return_exceptions=True)
+
+        # Phase 10.3: best-effort root cause analysis for failed tasks
+        await self._schedule_root_cause_analysis()
+
+        return self.report()
+
+    # ------------------------------------------------------------------
+    # Completion monitoring
+    # ------------------------------------------------------------------
+
+    async def _monitor_completions(self, expected: int) -> None:
+        """Read INTERACTION_RESULT messages until *expected* tasks complete."""
+        while self._completed < expected:
+            try:
+                await asyncio.wait_for(self._result_queue.get(), timeout=10.0)
+                self._completed += 1
+            except asyncio.TimeoutError:
+                logger.warning(
+                    "FunctionalTestOrchestrator: no result after 10s "
+                    "(completed=%d / expected=%d)",
+                    self._completed, expected,
+                )
+                break
+
+    # ------------------------------------------------------------------
+    # Report
+    # ------------------------------------------------------------------
+
+    def report(self) -> dict[str, Any]:
+        """Return structured results for the completed functional test run.
+
+        Returns
+        -------
+        dict
+            ``{passed, failed, total, tasks, elapsed_seconds,
+               dom_snapshots, screenshots, agent_count}``
+            When visual regression is enabled, a ``visual_regression`` key is
+            also included.
+            When accessibility scanning is enabled, an ``accessibility`` key is
+            also included.
+        """
+        elapsed = time.monotonic() - (self._start_time or 0.0)
+        functional = self._coordinator.get_functional_results()
+
+        result: dict[str, Any] = {
+            "passed": functional["passed"],
+            "failed": functional["failed"],
+            "total": functional["total"],
+            "tasks": functional["tasks"],
+            "dom_snapshots": functional["dom_snapshots"],
+            "screenshots": functional["screenshots"],
+            "elapsed_seconds": round(elapsed, 2),
+            "agent_count": len(self._browser_agents) + 1,
+        }
+
+        if self._visual_agent is not None:
+            result["visual_regression"] = self._visual_agent.get_visual_report()
+
+        if self._accessibility_agent is not None:
+            result["accessibility"] = self._accessibility_agent.get_accessibility_report()
+
+        if self._performance_agent is not None:
+            result["performance"] = self._performance_agent.get_performance_report()
+
+        # Phase 10.3: include any previously computed root cause suggestions
+        if hasattr(self, "_root_cause_suggestions"):
+            result["root_cause_analysis"] = self._root_cause_suggestions
+
+        return result
+
+    # ------------------------------------------------------------------
+    # Properties
+    # ------------------------------------------------------------------
+
+    @property
+    def coordinator(self) -> BrowserCoordinatorAgent:
+        return self._coordinator
+
+    @property
+    def browser_agents(self) -> list[BrowserExecutorAgent]:
+        return list(self._browser_agents)
+
+    @property
+    def visual_agent(self) -> VisualRegressionAgent | None:
+        """The :class:`VisualRegressionAgent` instance, or ``None`` if not enabled."""
+        return self._visual_agent
+
+    @property
+    def accessibility_agent(self) -> AccessibilityScannerAgent | None:
+        """The :class:`AccessibilityScannerAgent` instance, or ``None`` if not enabled."""
+        return self._accessibility_agent
+
+    @property
+    def performance_agent(self) -> PerformanceTestingAgent | None:
+        """The :class:`PerformanceTestingAgent` instance, or ``None`` if not enabled."""
+        return self._performance_agent
+
+    # ------------------------------------------------------------------
+    # Unified report generation
+    # ------------------------------------------------------------------
+
+    async def _schedule_root_cause_analysis(self) -> None:
+        """Best-effort root cause analysis for failed functional test tasks."""
+        if self._llm_provider is None:
+            return
+
+        try:
+            from mannf.product.llm.root_cause_service import ScanRootCauseAnalyzer  # noqa: PLC0415
+
+            functional = self._coordinator.get_functional_results()
+            failed_tasks = [
+                t for t in functional.get("tasks", [])
+                if not t.get("passed", True)
+            ]
+            if not failed_tasks:
+                self._root_cause_suggestions: list[dict[str, Any]] = []
+                return
+
+            failures = [
+                {
+                    "id": t.get("task_id", ""),
+                    "type": "functional",
+                    "url": t.get("url", ""),
+                    "error": t.get("error") or t.get("outcome", "Task failed"),
+                    "severity": "medium",
+                }
+                for t in failed_tasks
+            ]
+
+            analyzer = ScanRootCauseAnalyzer(
+                llm_provider=self._llm_provider,
+                quota=self._root_cause_quota,
+            )
+            suggestions = await analyzer.analyze_failures(failures, {"scan_type": "functional"})
+            self._root_cause_suggestions = [s.model_dump() for s in suggestions]
+            logger.info(
+                "FunctionalTestOrchestrator: root cause analysis produced %d suggestion(s)",
+                len(suggestions),
+            )
+        except Exception as exc:  # noqa: BLE001
+            logger.debug("FunctionalTestOrchestrator: root cause analysis skipped — %s", exc)
+            self._root_cause_suggestions = []
+
+    def generate_report(
+        self,
+        output_dir: str = "nat_reports",
+        formats: list[str] | None = None,
+    ) -> list[str]:
+        """Generate a unified HTML/JSON report from the last completed run.
+
+        This method calls :meth:`report` internally to obtain the latest
+        results, then passes them to
+        :class:`~mannf.core.reporting.unified_report.UnifiedReportGenerator`
+        to produce the output files.
+
+        Parameters
+        ----------
+        output_dir:
+            Directory to write report files into.  Created if it does not
+            exist.  Defaults to ``"nat_reports"``.
+        formats:
+            List of ``"html"`` and/or ``"json"``.  Defaults to both.
+
+        Returns
+        -------
+        list[str]
+            Absolute paths of the files that were written.
+        """
+        report_dict = self.report()
+        generator = UnifiedReportGenerator()
+        return generator.save_report(report_dict, output_dir=output_dir, formats=formats)

mannf/core/messaging/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Messaging sub-package for the Multi-Agent Neural Network Framework."""
+
+from mannf.core.messaging.messages import Message, MessageType
+from mannf.core.messaging.bus import MessageBus
+
+__all__ = ["Message", "MessageType", "MessageBus"]

mannf/core/messaging/bus.py

@@ -0,0 +1,113 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Async publish-subscribe message bus for inter-agent communication.
+
+Agents subscribe to one or more :class:`MessageType` values and receive
+matching :class:`Message` objects through an ``asyncio.Queue``.  The bus is
+the *only* shared state between agents, keeping coupling low.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Dict, List, Optional, Set
+
+from mannf.core.messaging.messages import Message, MessageType
+
+logger = logging.getLogger(__name__)
+
+
+class MessageBus:
+    """Lightweight, in-process publish/subscribe broker.
+
+    Usage::
+
+        bus = MessageBus()
+        queue = bus.subscribe("agent-1", {MessageType.TEST_RESULT})
+        await bus.publish(Message(MessageType.TEST_RESULT, sender_id="agent-2", payload=result))
+        msg = await queue.get()
+    """
+
+    def __init__(self) -> None:
+        # subscriber_id -> (set of subscribed types, asyncio Queue)
+        self._subscribers: Dict[str, tuple[Set[MessageType], asyncio.Queue[Message]]] = {}
+        self._lock = asyncio.Lock()
+
+    # ------------------------------------------------------------------
+    # Subscription management
+    # ------------------------------------------------------------------
+
+    def subscribe(
+        self,
+        subscriber_id: str,
+        message_types: Set[MessageType],
+        maxsize: int = 4096,
+    ) -> asyncio.Queue[Message]:
+        """Register *subscriber_id* to receive messages of *message_types*.
+
+        Returns an ``asyncio.Queue`` the subscriber should read from.
+        """
+        q: asyncio.Queue[Message] = asyncio.Queue(maxsize=maxsize)
+        self._subscribers[subscriber_id] = (message_types, q)
+        logger.debug("Subscribed %s to %s", subscriber_id, message_types)
+        return q
+
+    def unsubscribe(self, subscriber_id: str) -> None:
+        self._subscribers.pop(subscriber_id, None)
+        logger.debug("Unsubscribed %s", subscriber_id)
+
+    # ------------------------------------------------------------------
+    # Publishing
+    # ------------------------------------------------------------------
+
+    async def publish(self, message: Message) -> int:
+        """Deliver *message* to all matching subscribers.
+
+        Returns the number of queues the message was delivered to.
+        """
+        delivered = 0
+        for sub_id, (types, queue) in list(self._subscribers.items()):
+            if message.type in types:
+                try:
+                    queue.put_nowait(message)
+                    delivered += 1
+                except asyncio.QueueFull:
+                    logger.warning(
+                        "Queue full for subscriber %s; dropping message %s",
+                        sub_id,
+                        message.id,
+                    )
+        logger.debug(
+            "Published %s from %s to %d subscriber(s)",
+            message.type.name,
+            message.sender_id,
+            delivered,
+        )
+        return delivered
+
+    def publish_sync(self, message: Message) -> int:
+        """Synchronous variant (does not await); safe from non-async contexts."""
+        delivered = 0
+        for sub_id, (types, queue) in list(self._subscribers.items()):
+            if message.type in types:
+                try:
+                    queue.put_nowait(message)
+                    delivered += 1
+                except asyncio.QueueFull:
+                    logger.warning(
+                        "Queue full for subscriber %s; dropping message %s",
+                        sub_id,
+                        message.id,
+                    )
+        return delivered
+
+    @property
+    def subscriber_count(self) -> int:
+        return len(self._subscribers)
+
+    def subscriber_ids(self) -> List[str]:
+        return list(self._subscribers.keys())

mannf/core/messaging/messages.py

@@ -0,0 +1,89 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Message types used by agents to communicate over the message bus."""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from typing import Any, Optional
+
+
+class MessageType(Enum):
+    # Lifecycle
+    AGENT_STARTED = auto()
+    AGENT_STOPPED = auto()
+
+    # Test coordination (basic orchestrator)
+    TEST_REQUEST = auto()       # ask TestAgent to generate a test case
+    TEST_CASE = auto()          # a generated TestCase ready for execution
+    TEST_RESULT = auto()        # result after execution
+    TEST_VERDICT = auto()       # OracleAgent's pass/fail decision
+
+    # Monitoring
+    METRICS_UPDATE = auto()     # MonitorAgent reports system metrics
+    ANOMALY_DETECTED = auto()   # MonitorAgent detected an anomaly
+
+    # Adaptation
+    STRATEGY_UPDATE = auto()    # AdaptiveController broadcasts new weights/strategy
+    FEEDBACK = auto()           # generic feedback signal for NN training
+
+    # ── NAT / ECNP messages (Extended Contract Net Protocol) ────────────
+    TASK_ANNOUNCEMENT = auto()  # CoordinatorAgent broadcasts a testing task
+    TASK_BID = auto()           # ExecutorAgent submits a bid for a task
+    CONTRACT_AWARD = auto()     # CoordinatorAgent awards the contract to a bidder
+    CONTRACT_RESULT = auto()    # ExecutorAgent reports execution outcome to Coordinator
+
+    # BDI belief propagation
+    BELIEF_UPDATE = auto()      # AnalyzerAgent broadcasts updated fault-likelihood beliefs
+    BELIEF_REQUEST = auto()     # Any agent requests the current belief snapshot
+
+    # ── Browser / Functional Testing messages ────────────────────────────
+    INTERACTION_REQUEST = auto()    # Request a browser interaction (click, fill, navigate)
+    INTERACTION_RESULT = auto()     # Result of a browser interaction
+    DOM_SNAPSHOT = auto()           # DOM snapshot captured by a browser agent
+    SCREENSHOT_CAPTURED = auto()    # Screenshot taken by a browser agent
+
+    # ── Visual Regression messages ────────────────────────────────────────
+    VISUAL_DIFF_DETECTED = auto()   # Screenshot differs from baseline beyond threshold
+    VISUAL_BASELINE_SAVED = auto()  # New baseline screenshot stored
+
+    # ── Accessibility Scanning messages ──────────────────────────────────
+    ACCESSIBILITY_SCAN_REQUEST = auto()   # Request an accessibility scan on a page
+    ACCESSIBILITY_SCAN_RESULT = auto()    # Result of an accessibility scan (violations found)
+
+    # ── Performance Testing messages ──────────────────────────────────────
+    PERFORMANCE_METRICS_CAPTURED = auto()   # Performance metrics collected from a page
+    PERFORMANCE_THRESHOLD_VIOLATED = auto() # A performance metric exceeded its threshold
+
+    # ── Web Crawler messages ──────────────────────────────────────────────
+    CRAWL_PROGRESS = auto()     # WebCrawlerAgent reports discovery progress (live updates)
+    CRAWL_RESULT = auto()       # WebCrawlerAgent publishes completed discovery model
+
+    # ── Autonomous Test Loop messages ─────────────────────────────────────
+    AUTONOMOUS_ITERATION = auto()   # AutonomousTestLoopAgent completed one iteration
+    AUTONOMOUS_COMPLETE = auto()    # AutonomousTestLoopAgent finished the full run
+
+
+@dataclass
+class Message:
+    """Envelope that carries a payload between agents on the message bus."""
+
+    type: MessageType
+    sender_id: str
+    payload: Any = None
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    correlation_id: Optional[str] = None  # links replies to requests
+
+    def reply(self, msg_type: MessageType, payload: Any, sender_id: str) -> "Message":
+        """Convenience constructor for a reply to this message."""
+        return Message(
+            type=msg_type,
+            sender_id=sender_id,
+            payload=payload,
+            correlation_id=self.id,
+        )