nat-engine 1__py3-none-any.whl

mannf/core/orchestrator.py

@@ -0,0 +1,272 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Orchestrator – the central coordinator of the MANNF framework.
+
+The :class:`Orchestrator` manages:
+  1. Startup / shutdown of all agents.
+  2. The test-execution loop: request test cases → execute → record results.
+  3. The adaptive feedback loop: update the :class:`AdaptiveController` after
+     each test and broadcast strategy updates to agents.
+  4. Progress logging and final report generation.
+
+Usage::
+
+    from mannf import Orchestrator
+    from mannf.core.distributed import MockDistributedSystem
+
+    sut = MockDistributedSystem(seed=42)
+    orch = Orchestrator(sut, num_test_agents=3, max_steps=200)
+    await orch.run()
+    report = orch.report()
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from typing import Any, Dict, List, Optional
+
+from mannf.core.agents.monitor_agent import MonitorAgent
+from mannf.core.agents.oracle_agent import OracleAgent
+from mannf.core.agents.test_agent import TestAgent
+from mannf.core.distributed.system_under_test import SystemUnderTest
+from mannf.core.messaging.bus import MessageBus
+from mannf.core.messaging.messages import Message, MessageType
+from mannf.core.testing.adaptive_controller import AdaptiveController
+from mannf.core.testing.models import TestCase, TestResult, TestSuite
+
+logger = logging.getLogger(__name__)
+
+
+class Orchestrator:
+    """Coordinates all agents and drives the adaptive testing loop.
+
+    Parameters
+    ----------
+    sut:
+        The system under test (real or mock).
+    num_test_agents:
+        How many parallel :class:`TestAgent` instances to run.
+    max_steps:
+        Total number of test-execution steps before stopping.
+    suite_name:
+        Name used in the test suite and final report.
+    log_interval:
+        Log a progress summary every *log_interval* steps.
+    """
+
+    def __init__(
+        self,
+        sut: SystemUnderTest,
+        num_test_agents: int = 2,
+        max_steps: int = 100,
+        suite_name: str = "Adaptive Test Run",
+        log_interval: int = 10,
+    ) -> None:
+        self.sut = sut
+        self.max_steps = max_steps
+        self.log_interval = log_interval
+
+        self._bus = MessageBus()
+        self._suite = TestSuite(name=suite_name)
+        self._controller = AdaptiveController()
+
+        service_names = sut.registry.all()
+        svc_names = [e.name for e in service_names]
+
+        # Create agents
+        self._test_agents: List[TestAgent] = [
+            TestAgent(f"test-agent-{i}", self._bus, svc_names)
+            for i in range(num_test_agents)
+        ]
+        self._monitor = MonitorAgent("monitor-agent", self._bus)
+        self._oracle = OracleAgent("oracle-agent", self._bus)
+
+        self._step = 0
+        self._start_time: Optional[float] = None
+
+    # ------------------------------------------------------------------
+    # Main entry point
+    # ------------------------------------------------------------------
+
+    async def run(self) -> "TestSuite":
+        """Execute the full adaptive testing loop.
+
+        Agents run as concurrent asyncio tasks so they process messages from
+        the bus while the orchestrator drives the test-execution loop.
+
+        Returns the completed :class:`TestSuite`.
+        """
+        self._start_time = time.monotonic()
+        logger.info("Orchestrator: starting with max_steps=%d", self.max_steps)
+
+        all_agents = [self._monitor, self._oracle] + self._test_agents
+
+        # Launch each agent as an independent asyncio task (their run() loops
+        # process bus messages concurrently with the test-execution loop).
+        agent_tasks = [asyncio.create_task(agent.run()) for agent in all_agents]
+
+        # Give agents a moment to complete their start() subscriptions
+        await asyncio.sleep(0)
+
+        try:
+            await self._run_loop()
+        finally:
+            # Signal all agents to stop and wait for clean shutdown
+            for agent in all_agents:
+                agent._running = False
+            await asyncio.gather(*agent_tasks, return_exceptions=True)
+
+        elapsed = time.monotonic() - (self._start_time or 0)
+        logger.info(
+            "Orchestrator: finished %d steps in %.1f s", self._step, elapsed
+        )
+        return self._suite
+
+    # ------------------------------------------------------------------
+    # Test-execution loop
+    # ------------------------------------------------------------------
+
+    async def _run_loop(self) -> None:
+        while self._step < self.max_steps:
+            # 1. Pick a test agent (round-robin)
+            agent = self._test_agents[self._step % len(self._test_agents)]
+
+            # 2. Request a test case
+            test_case = await self._request_test_case(agent)
+            if test_case is None:
+                await asyncio.sleep(0.01)
+                continue
+
+            # 3. Apply adaptive selection from a small candidate pool
+            candidates = [test_case]
+            for _ in range(2):
+                extra = await self._request_test_case(
+                    self._test_agents[(self._step + 1) % len(self._test_agents)]
+                )
+                if extra:
+                    candidates.append(extra)
+
+            selected = self._controller.select(candidates) or test_case
+
+            # 4. Execute against the SUT
+            result = await self.sut.execute(selected)
+            result.agent_id = agent.agent_id
+
+            # 5. Record result
+            self._suite.add_case(selected)
+            self._suite.record_result(result)
+
+            # 6. Broadcast result to monitor & oracle (single unified message)
+            await self._bus.publish(
+                Message(
+                    MessageType.TEST_RESULT,
+                    "orchestrator",
+                    {"result": result, "test_case": selected},
+                )
+            )
+
+            # 7. Update the adaptive controller
+            self._controller.update(selected, result)
+
+            # 8. Send feedback to the test agents
+            feedback = self._build_feedback(selected, result)
+            await self._bus.publish(
+                Message(MessageType.FEEDBACK, "orchestrator", feedback)
+            )
+
+            self._step += 1
+
+            if self._step % self.log_interval == 0:
+                self._log_progress()
+
+            # Yield control to allow agent background tasks to process messages
+            await asyncio.sleep(0)
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    async def _request_test_case(self, agent: TestAgent) -> Optional[TestCase]:
+        """Ask an agent to generate a test case synchronously (in-process)."""
+        context = {
+            "pass_rate": self._suite.pass_rate,
+            "defect_detection_rate": self._suite.defect_detection_rate,
+            "step_count": self._step,
+        }
+        req = Message(MessageType.TEST_REQUEST, "orchestrator", context)
+        # Call the agent's generation logic directly for synchronous behaviour
+        test_case = agent._generate_test_case(context=context)
+        return test_case
+
+    def _build_feedback(
+        self, test_case: TestCase, result: TestResult
+    ) -> Dict[str, Any]:
+        """Build a feedback payload for the test agents."""
+        reward = result.reward_signal()
+        features = test_case.feature_vector().tolist()
+        # Target: bias towards the service that produced the reward
+        target = [0.0] * max(len(self._test_agents), 1)
+        return {
+            "features": features,
+            "target": [reward],
+            "reward": reward,
+            "test_case_id": test_case.id,
+        }
+
+    def _log_progress(self) -> None:
+        summary = self._suite.summary()
+        elapsed = time.monotonic() - (self._start_time or time.monotonic())
+        logger.info(
+            "Step %4d/%d | passed=%d failed=%d errored=%d "
+            "| defect_rate=%.1f%% | ε=%.3f | elapsed=%.1fs",
+            self._step,
+            self.max_steps,
+            summary["passed"],
+            summary["failed"],
+            summary["errored"],
+            summary["defect_detection_rate"] * 100,
+            self._controller.epsilon,
+            elapsed,
+        )
+
+    # ------------------------------------------------------------------
+    # Reporting
+    # ------------------------------------------------------------------
+
+    def report(self) -> Dict[str, Any]:
+        """Return a structured summary of the completed test run."""
+        elapsed = (
+            time.monotonic() - self._start_time if self._start_time else 0.0
+        )
+        suite_summary = self._suite.summary()
+        monitor_metrics = self._monitor.metrics
+
+        return {
+            "suite": suite_summary,
+            "monitor": monitor_metrics,
+            "oracle": {
+                "verdict_count": self._oracle.verdict_count,
+                "defect_count": self._oracle.defect_count,
+                "defect_rate": round(self._oracle.defect_rate, 4),
+            },
+            "adaptive_controller": {
+                "step_count": self._controller.step_count,
+                "epsilon": round(self._controller.epsilon, 4),
+                "average_reward": round(self._controller.average_reward, 4),
+            },
+            "agent_count": len(self._test_agents) + 2,  # +monitor +oracle
+            "elapsed_seconds": round(elapsed, 2),
+        }
+
+    @property
+    def suite(self) -> TestSuite:
+        return self._suite
+
+    @property
+    def controller(self) -> AdaptiveController:
+        return self._controller

mannf/core/prioritization/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Prioritization sub-package for belief-driven test ordering and risk scoring."""
+
+from mannf.core.prioritization.adaptive_controller import AdaptiveController
+from mannf.core.prioritization.belief_prioritizer import BeliefPrioritizer, PrioritizationStrategy
+from mannf.core.prioritization.risk_scorer import RiskScorer
+
+__all__ = [
+    "BeliefPrioritizer",
+    "PrioritizationStrategy",
+    "RiskScorer",
+    "AdaptiveController",
+]