nat-engine 1__py3-none-any.whl

mannf/core/nat_orchestrator.py

@@ -0,0 +1,342 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""NAT Orchestrator – full NeuroAgentTest BDI/ECNP architecture.
+
+This orchestrator implements the complete NAT framework as described in
+Chapter 4 of the thesis, using:
+
+  * **PlannerAgents**    – formulate test objectives from belief state
+  * **ExecutorAgents**   – bid on and execute contracts via ECNP
+  * **AnalyzerAgents**   – process results, update beliefs, detect anomalies
+  * **CoordinatorAgent** – allocates tasks via Extended Contract Net Protocol
+
+All agents run concurrently as ``asyncio`` tasks and communicate through the
+shared :class:`MessageBus`.  The orchestrator drives agent lifecycle and
+produces the final report.
+
+Emergence: unlike the simple round-robin :class:`Orchestrator`, this
+implementation produces emergent specialisation through ECNP negotiation –
+services with higher fault likelihood attract more testing attention without
+any explicit assignment.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from typing import Any, Dict, List, Optional
+
+from mannf.core.agents.analyzer_agent import AnalyzerAgent
+from mannf.core.agents.coordinator_agent import CoordinatorAgent
+from mannf.core.agents.executor_agent import ExecutorAgent
+from mannf.core.agents.planner_agent import PlannerAgent
+from mannf.core.anomaly.enhanced_detector import EnhancedAnomalyDetector
+from mannf.core.distributed.system_under_test import SystemUnderTest
+from mannf.core.messaging.bus import MessageBus
+from mannf.core.messaging.messages import Message, MessageType
+from mannf.core.prioritization.belief_prioritizer import BeliefPrioritizer
+from mannf.core.prioritization.risk_scorer import RiskScorer
+from mannf.core.testing.models import TestResult, TestSuite
+
+logger = logging.getLogger(__name__)
+
+
+def _infer_status_code(result: TestResult, actual: Any) -> int:
+    """Infer an HTTP status code from a test result for anomaly detection."""
+    if result.error:
+        return 500
+    if isinstance(actual, dict) and "status_code" in actual:
+        try:
+            return int(actual["status_code"])
+        except (ValueError, TypeError):
+            pass
+    return 200 if result.passed else 400
+
+
+class NATOrchestrator:
+    """Full NeuroAgentTest orchestrator using BDI agents and ECNP.
+
+    Parameters
+    ----------
+    sut:
+        The system under test.
+    num_planners:
+        Number of PlannerAgent instances.
+    num_executors:
+        Number of ExecutorAgent instances.
+    num_analyzers:
+        Number of AnalyzerAgent instances.
+    max_contracts:
+        Stop after this many contracts have been completed.
+    suite_name:
+        Label for the test suite in the final report.
+    log_interval:
+        Log a progress summary every *log_interval* completions.
+    bid_timeout_s:
+        ECNP bid collection window (seconds).
+    """
+
+    def __init__(
+        self,
+        sut: SystemUnderTest,
+        num_planners: int = 2,
+        num_executors: int = 3,
+        num_analyzers: int = 1,
+        max_contracts: int = 100,
+        suite_name: str = "NAT Run",
+        log_interval: int = 10,
+        bid_timeout_s: float = 0.05,
+        belief_prioritizer: Optional[BeliefPrioritizer] = None,
+        anomaly_detector: Optional[EnhancedAnomalyDetector] = None,
+    ) -> None:
+        self.sut = sut
+        self.max_contracts = max_contracts
+        self.log_interval = log_interval
+        self.belief_prioritizer = belief_prioritizer
+        self.anomaly_detector = anomaly_detector
+
+        self._bus = MessageBus()
+        self._suite = TestSuite(name=suite_name)
+
+        service_names = [e.name for e in sut.registry.all()]
+
+        # Risk scorer is always active; it feeds get_risk_report()
+        self._risk_scorer = RiskScorer()
+
+        # Create agents
+        self._planners: List[PlannerAgent] = [
+            PlannerAgent(f"planner-{i}", self._bus, service_names)
+            for i in range(num_planners)
+        ]
+        self._executors: List[ExecutorAgent] = [
+            ExecutorAgent(f"executor-{i}", self._bus, service_names, sut)
+            for i in range(num_executors)
+        ]
+        self._analyzers: List[AnalyzerAgent] = [
+            AnalyzerAgent(f"analyzer-{i}", self._bus, service_names)
+            for i in range(num_analyzers)
+        ]
+        self._coordinator = CoordinatorAgent(
+            "coordinator-0", self._bus, service_names, bid_timeout_s=bid_timeout_s
+        )
+
+        # Subscribe orchestrator to monitor completion
+        # TEST_RESULT is published once per execution by ExecutorAgents
+        self._result_queue = self._bus.subscribe(
+            "nat-orchestrator",
+            {MessageType.TEST_RESULT},
+        )
+
+        self._completed = 0
+        self._start_time: Optional[float] = None
+
+    # ------------------------------------------------------------------
+    # Main entry point
+    # ------------------------------------------------------------------
+
+    async def run(self) -> TestSuite:
+        """Launch all agents and wait for max_contracts completions."""
+        self._start_time = time.monotonic()
+        logger.info("NATOrchestrator: starting (max_contracts=%d)", self.max_contracts)
+
+        all_agents = (
+            self._planners + self._executors + self._analyzers + [self._coordinator]
+        )
+        tasks = [asyncio.create_task(a.run()) for a in all_agents]
+
+        # Give agents time to subscribe and start their run() loops
+        await asyncio.sleep(0)
+
+        # Kickstart the pipeline: trigger initial task announcements from all
+        # planners so executors have something to bid on immediately
+        for planner in self._planners:
+            asyncio.get_running_loop().create_task(planner._announce_tasks())
+
+        # Allow announcements to propagate before monitoring begins
+        await asyncio.sleep(0.01)
+
+        try:
+            await self._monitor_completions()
+        finally:
+            for a in all_agents:
+                a._running = False
+            await asyncio.gather(*tasks, return_exceptions=True)
+
+        elapsed = time.monotonic() - (self._start_time or 0)
+        logger.info(
+            "NATOrchestrator: finished %d contracts in %.1fs",
+            self._completed, elapsed,
+        )
+        return self._suite
+
+    # ------------------------------------------------------------------
+    # Completion monitoring
+    # ------------------------------------------------------------------
+
+    async def _monitor_completions(self) -> None:
+        """Read CONTRACT_RESULT / TEST_RESULT messages and build the suite."""
+        while self._completed < self.max_contracts:
+            try:
+                msg = await asyncio.wait_for(self._result_queue.get(), timeout=5.0)
+            except asyncio.TimeoutError:
+                logger.warning(
+                    "NATOrchestrator: no results after 5s (completed=%d)",
+                    self._completed,
+                )
+                break
+
+            payload = msg.payload
+            result: Optional[TestResult] = None
+            test_case = None
+
+            if isinstance(payload, dict):
+                result = payload.get("result")
+                test_case = payload.get("test_case")
+            elif isinstance(payload, TestResult):
+                result = payload
+
+            if result is None:
+                continue
+
+            if test_case is not None:
+                self._suite.add_case(test_case)
+            self._suite.record_result(result)
+            self._completed += 1
+
+            # Update belief prioritizer and risk scorer with the result
+            target = test_case.target if test_case is not None else "unknown"
+            if self.belief_prioritizer is not None:
+                self.belief_prioritizer.record_result(result, target)
+
+            belief_states = [a.beliefs for a in self._analyzers]
+            self._risk_scorer.record(target, result, belief_states)
+
+            # Run anomaly detection on the result
+            if self.anomaly_detector is not None:
+                actual = result.actual_output
+                self.anomaly_detector.observe(
+                    endpoint=target,
+                    latency_ms=result.execution_time_ms,
+                    status_code=_infer_status_code(result, actual),
+                    payload=actual,
+                    is_error=bool(result.error),
+                )
+
+            if self._completed % self.log_interval == 0:
+                self._log_progress()
+
+    def _log_progress(self) -> None:
+        s = self._suite.summary()
+        elapsed = time.monotonic() - (self._start_time or time.monotonic())
+        # Aggregate belief state from all analyzers
+        avg_entropy = (
+            sum(a.beliefs.entropy() for a in self._analyzers) / len(self._analyzers)
+            if self._analyzers else 0.0
+        )
+        logger.info(
+            "NAT %4d/%d | passed=%d failed=%d errored=%d "
+            "| defect_rate=%.1f%% | avg_belief_entropy=%.3f "
+            "| contracts_awarded=%d | elapsed=%.1fs",
+            self._completed, self.max_contracts,
+            s["passed"], s["failed"], s["errored"],
+            s["defect_detection_rate"] * 100,
+            avg_entropy,
+            self._coordinator.contracts_awarded,
+            elapsed,
+        )
+
+    # ------------------------------------------------------------------
+    # Report
+    # ------------------------------------------------------------------
+
+    def report(self) -> Dict[str, Any]:
+        """Return a structured summary of the completed NAT run."""
+        elapsed = time.monotonic() - (self._start_time or 0)
+        suite_summary = self._suite.summary()
+
+        # Aggregate belief state across analyzers
+        avg_entropy = (
+            sum(a.beliefs.entropy() for a in self._analyzers) / len(self._analyzers)
+            if self._analyzers else 0.0
+        )
+
+        # Per-service failure counts from each executor
+        failure_counts: Dict[str, int] = {}
+        for exc in self._executors:
+            for svc, count in exc.beliefs.fault_likelihood.items():
+                pass  # fault_likelihood is beliefs, not counts
+        # Use SUT call log for ground truth
+        failure_counts = self.sut.failure_counts() if hasattr(self.sut, "failure_counts") else {}
+
+        # Belief state from analyzers
+        combined_beliefs: Dict[str, float] = {}
+        for analyzer in self._analyzers:
+            for svc, val in analyzer.beliefs.fault_likelihood.items():
+                combined_beliefs[svc] = (combined_beliefs.get(svc, 0.0) + val)
+        if self._analyzers:
+            combined_beliefs = {k: v / len(self._analyzers) for k, v in combined_beliefs.items()}
+
+        return {
+            "suite": suite_summary,
+            "coordinator": {
+                "contracts_awarded": self._coordinator.contracts_awarded,
+                "contracts_completed": self._coordinator.contracts_completed,
+            },
+            "executors": [
+                {
+                    "id": e.agent_id,
+                    "executions": e.execution_count,
+                    "avg_latency_ms": round(e.avg_latency_ms, 2),
+                }
+                for e in self._executors
+            ],
+            "analyzers": [
+                {
+                    "id": a.agent_id,
+                    "total_results": a.total_results,
+                    "belief_entropy": round(a.beliefs.entropy(), 4),
+                    "top_risk_services": a.beliefs.top_k(3),
+                }
+                for a in self._analyzers
+            ],
+            "beliefs": {
+                k: round(v, 4)
+                for k, v in sorted(
+                    combined_beliefs.items(), key=lambda x: x[1], reverse=True
+                )
+            },
+            "avg_belief_entropy": round(avg_entropy, 4),
+            "actual_failure_counts": failure_counts,
+            "agent_count": (
+                len(self._planners) + len(self._executors)
+                + len(self._analyzers) + 1
+            ),
+            "elapsed_seconds": round(elapsed, 2),
+            "risk_report": self.get_risk_report(),
+            "anomaly_alerts": (
+                self.anomaly_detector.get_alerts_dict()
+                if self.anomaly_detector is not None
+                else []
+            ),
+        }
+
+    def get_risk_report(self) -> List[Dict[str, Any]]:
+        """Return the current per-endpoint risk report from the risk scorer."""
+        belief_states = [a.beliefs for a in self._analyzers]
+        return self._risk_scorer.get_risk_report(belief_states)
+
+    @property
+    def suite(self) -> TestSuite:
+        return self._suite
+
+    @property
+    def coordinator(self) -> CoordinatorAgent:
+        return self._coordinator
+
+    @property
+    def risk_scorer(self) -> RiskScorer:
+        return self._risk_scorer

mannf/core/neural/__init__.py

@@ -0,0 +1,183 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Neural network primitives built on NumPy.
+
+Provides a lightweight, dependency-free implementation of a fully-connected
+feedforward network with backpropagation.  Each agent in the framework owns
+one or more of these networks to guide its decision-making.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from typing import List, Sequence
+
+
+# ---------------------------------------------------------------------------
+# Activation functions
+# ---------------------------------------------------------------------------
+
+def relu(x: np.ndarray) -> np.ndarray:
+    return np.maximum(0.0, x)
+
+
+def relu_deriv(x: np.ndarray) -> np.ndarray:
+    return (x > 0.0).astype(float)
+
+
+def sigmoid(x: np.ndarray) -> np.ndarray:
+    return 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))
+
+
+def sigmoid_deriv(x: np.ndarray) -> np.ndarray:
+    s = sigmoid(x)
+    return s * (1.0 - s)
+
+
+def softmax(x: np.ndarray) -> np.ndarray:
+    e = np.exp(x - np.max(x))
+    return e / e.sum()
+
+
+# ---------------------------------------------------------------------------
+# Network layer
+# ---------------------------------------------------------------------------
+
+class Layer:
+    """Single fully-connected layer with weight matrix and bias vector."""
+
+    def __init__(self, n_in: int, n_out: int, activation: str = "relu") -> None:
+        self.activation = activation
+        # Xavier / Glorot initialisation
+        scale = np.sqrt(2.0 / n_in)
+        self.W: np.ndarray = np.random.randn(n_in, n_out) * scale
+        self.b: np.ndarray = np.zeros(n_out)
+
+        # Cache for backprop
+        self._x: np.ndarray | None = None
+        self._z: np.ndarray | None = None
+
+    # -- forward ----------------------------------------------------------
+
+    def forward(self, x: np.ndarray) -> np.ndarray:
+        self._x = x
+        self._z = x @ self.W + self.b
+        return self._activate(self._z)
+
+    def _activate(self, z: np.ndarray) -> np.ndarray:
+        if self.activation == "relu":
+            return relu(z)
+        if self.activation == "sigmoid":
+            return sigmoid(z)
+        if self.activation == "tanh":
+            return np.tanh(z)
+        if self.activation == "linear":
+            return z
+        raise ValueError(f"Unknown activation: {self.activation}")
+
+    def _activate_deriv(self, z: np.ndarray) -> np.ndarray:
+        if self.activation == "relu":
+            return relu_deriv(z)
+        if self.activation == "sigmoid":
+            return sigmoid_deriv(z)
+        if self.activation == "tanh":
+            return 1.0 - np.tanh(z) ** 2
+        if self.activation == "linear":
+            return np.ones_like(z)
+        raise ValueError(f"Unknown activation: {self.activation}")
+
+    # -- backward ---------------------------------------------------------
+
+    def backward(self, grad_out: np.ndarray) -> np.ndarray:
+        """Return gradient w.r.t. input; cache weight/bias gradients."""
+        assert self._z is not None and self._x is not None
+        grad_z = grad_out * self._activate_deriv(self._z)
+        self._grad_W = self._x[:, None] * grad_z[None, :]  # outer product
+        self._grad_b = grad_z
+        return grad_z @ self.W.T  # propagate to previous layer
+
+    # -- parameter update -------------------------------------------------
+
+    def apply_gradients(self, lr: float) -> None:
+        assert hasattr(self, "_grad_W"), "backward() must be called first"
+        self.W -= lr * self._grad_W
+        self.b -= lr * self._grad_b
+
+
+# ---------------------------------------------------------------------------
+# Fully-connected network
+# ---------------------------------------------------------------------------
+
+class NeuralNetwork:
+    """Multi-layer perceptron with MSE loss and online gradient descent.
+
+    Parameters
+    ----------
+    layer_sizes:
+        Sequence of integers specifying the width of each layer,
+        e.g. ``[8, 16, 8, 4]`` creates three layers.
+    hidden_activation:
+        Activation used for all hidden layers.  Output layer uses *linear*
+        activation so the network can regress arbitrary real values.
+    learning_rate:
+        Step size for stochastic gradient descent updates.
+    """
+
+    def __init__(
+        self,
+        layer_sizes: Sequence[int],
+        hidden_activation: str = "relu",
+        learning_rate: float = 0.01,
+    ) -> None:
+        if len(layer_sizes) < 2:
+            raise ValueError("layer_sizes must contain at least two elements (in, out)")
+        self.lr = learning_rate
+        self.layers: List[Layer] = []
+        for i in range(len(layer_sizes) - 1):
+            act = hidden_activation if i < len(layer_sizes) - 2 else "linear"
+            self.layers.append(Layer(layer_sizes[i], layer_sizes[i + 1], activation=act))
+
+    # -- forward pass -----------------------------------------------------
+
+    def predict(self, x: np.ndarray) -> np.ndarray:
+        """Return network output for input vector *x*."""
+        out = x.astype(float)
+        for layer in self.layers:
+            out = layer.forward(out)
+        return out
+
+    # -- training step ----------------------------------------------------
+
+    def train_step(self, x: np.ndarray, target: np.ndarray) -> float:
+        """Forward pass, MSE loss, backprop, and SGD update.
+
+        Returns the scalar MSE loss value.
+        """
+        output = self.predict(x)
+        loss = float(np.mean((output - target) ** 2))
+
+        # Gradient of MSE loss w.r.t. output
+        grad = 2.0 * (output - target) / len(output)
+
+        for layer in reversed(self.layers):
+            grad = layer.backward(grad)
+
+        for layer in self.layers:
+            layer.apply_gradients(self.lr)
+
+        return loss
+
+    # -- persistence helpers ----------------------------------------------
+
+    def get_weights(self) -> List[dict]:
+        """Return a snapshot of all weight matrices and bias vectors."""
+        return [{"W": layer.W.copy(), "b": layer.b.copy()} for layer in self.layers]
+
+    def set_weights(self, weights: List[dict]) -> None:
+        """Restore weights from a previous snapshot."""
+        for layer, wb in zip(self.layers, weights):
+            layer.W = wb["W"].copy()
+            layer.b = wb["b"].copy()