nat-engine 1__py3-none-any.whl

mannf/core/testing/adaptive_controller.py

@@ -0,0 +1,149 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Adaptive testing controller.
+
+Maintains a neural-network-based policy that scores candidate test cases
+and decides *which* test to run next (exploitation) vs *generating* a new
+exploratory test (exploration).
+
+The exploration–exploitation trade-off is managed with an ε-greedy strategy
+where ε decays over time as the agents gain experience.
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+from typing import List, Optional
+
+import numpy as np
+
+from mannf.core.neural import NeuralNetwork
+from mannf.core.testing.models import TestCase, TestResult
+
+logger = logging.getLogger(__name__)
+
+# Feature dimensionality must match TestCase.feature_vector()
+_FEATURE_DIM = 8
+
+
+class AdaptiveController:
+    """Selects and prioritises test cases using a learned value function.
+
+    The controller maintains a *value network* V(s) that predicts the
+    expected reward (defect-finding potential) of a test case.  After each
+    test execution, the network is updated with the observed reward via a
+    single-sample gradient step (online learning).
+
+    Parameters
+    ----------
+    initial_epsilon:
+        Initial exploration probability (0–1).  1 = fully random.
+    min_epsilon:
+        Minimum exploration probability after decay.
+    epsilon_decay:
+        Multiplicative decay applied after each test execution.
+    learning_rate:
+        Step size for the value-network update.
+    """
+
+    def __init__(
+        self,
+        initial_epsilon: float = 0.9,
+        min_epsilon: float = 0.05,
+        epsilon_decay: float = 0.995,
+        learning_rate: float = 0.005,
+    ) -> None:
+        self.epsilon = initial_epsilon
+        self.min_epsilon = min_epsilon
+        self.epsilon_decay = epsilon_decay
+
+        # Value network: maps a test-case feature vector to a scalar reward estimate
+        self.value_net = NeuralNetwork(
+            layer_sizes=[_FEATURE_DIM, 16, 8, 1],
+            hidden_activation="relu",
+            learning_rate=learning_rate,
+        )
+
+        self._step_count = 0
+        self._cumulative_reward = 0.0
+
+    # ------------------------------------------------------------------
+    # Selection
+    # ------------------------------------------------------------------
+
+    def select(self, candidates: List[TestCase]) -> Optional[TestCase]:
+        """Pick the next test case to execute.
+
+        With probability *ε* a random candidate is selected (exploration);
+        otherwise the candidate with the highest predicted reward is chosen
+        (exploitation).
+        """
+        if not candidates:
+            return None
+
+        if np.random.random() < self.epsilon:
+            choice = candidates[int(np.random.randint(len(candidates)))]
+            logger.debug("AdaptiveController: exploring → %s", choice.id[:8])
+        else:
+            scores = [
+                float(self.value_net.predict(c.feature_vector(_FEATURE_DIM))[0])
+                for c in candidates
+            ]
+            best_idx = int(np.argmax(scores))
+            choice = candidates[best_idx]
+            logger.debug(
+                "AdaptiveController: exploiting → %s (score=%.4f)",
+                choice.id[:8],
+                scores[best_idx],
+            )
+        return choice
+
+    # ------------------------------------------------------------------
+    # Learning
+    # ------------------------------------------------------------------
+
+    def update(self, test_case: TestCase, result: TestResult) -> float:
+        """Update the value network with the observed reward signal.
+
+        Returns the MSE training loss for the current step.
+        """
+        reward = result.reward_signal()
+        self._cumulative_reward += reward
+
+        x = test_case.feature_vector(_FEATURE_DIM)
+        target = np.array([reward])
+        loss = self.value_net.train_step(x, target)
+
+        # Decay exploration rate
+        self.epsilon = max(self.min_epsilon, self.epsilon * self.epsilon_decay)
+        self._step_count += 1
+
+        logger.debug(
+            "AdaptiveController update: reward=%.3f loss=%.6f ε=%.4f",
+            reward,
+            loss,
+            self.epsilon,
+        )
+        return loss
+
+    # ------------------------------------------------------------------
+    # Diagnostics
+    # ------------------------------------------------------------------
+
+    @property
+    def step_count(self) -> int:
+        return self._step_count
+
+    @property
+    def average_reward(self) -> float:
+        if self._step_count == 0:
+            return 0.0
+        return self._cumulative_reward / self._step_count
+
+    def score(self, test_case: TestCase) -> float:
+        """Return the current predicted value for a test case."""
+        return float(self.value_net.predict(test_case.feature_vector(_FEATURE_DIM))[0])

mannf/core/testing/models.py

@@ -0,0 +1,179 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Core data models for the adaptive testing layer."""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+if TYPE_CHECKING:
+    import numpy as np
+
+
+class TestStatus(Enum):
+    PENDING = auto()
+    RUNNING = auto()
+    PASSED = auto()
+    FAILED = auto()
+    ERROR = auto()
+    SKIPPED = auto()
+
+
+@dataclass
+class TestCase:
+    """Represents a single test to be executed against a distributed endpoint.
+
+    Parameters
+    ----------
+    target:
+        Logical name of the service/endpoint being tested.
+    inputs:
+        Dictionary of input parameters to send.
+    expected_behavior:
+        Human-readable description of the expected outcome.
+    priority:
+        Float in [0, 1]; higher values mean the test should run sooner.
+    metadata:
+        Arbitrary key-value pairs (e.g. tags, author, generation strategy).
+    """
+
+    target: str
+    inputs: Dict[str, Any]
+    expected_behavior: str
+    priority: float = 0.5
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    status: TestStatus = TestStatus.PENDING
+
+    def feature_vector(self, feature_dim: int = 8) -> "np.ndarray":
+        """Return a fixed-length numeric feature vector used as NN input.
+
+        The vector encodes: priority, input complexity, id hash projection.
+        """
+        import numpy as np
+
+        vec = np.zeros(feature_dim)
+        vec[0] = self.priority
+        vec[1] = len(self.inputs) / 20.0          # normalised input arity
+        vec[2] = len(self.target) / 50.0           # normalised target length
+        # Hash the id into the remaining dimensions for uniqueness
+        h = hash(self.id) & 0xFFFFFF
+        for i in range(3, feature_dim):
+            vec[i] = ((h >> (i * 3)) & 0x7) / 7.0
+        return vec
+
+
+@dataclass
+class TestResult:
+    """The outcome of executing a :class:`TestCase`.
+
+    Parameters
+    ----------
+    test_case_id:
+        The id of the test case that was executed.
+    passed:
+        Whether the test produced the expected behaviour.
+    actual_output:
+        Raw output returned by the system under test.
+    execution_time_ms:
+        Wall-clock time taken to execute the test.
+    error:
+        Exception message if an error occurred (``None`` otherwise).
+    agent_id:
+        ID of the agent that executed/validated the test.
+    """
+
+    test_case_id: str
+    passed: bool
+    actual_output: Any = None
+    execution_time_ms: float = 0.0
+    error: Optional[str] = None
+    agent_id: Optional[str] = None
+
+    @property
+    def status(self) -> TestStatus:
+        if self.error:
+            return TestStatus.ERROR
+        return TestStatus.PASSED if self.passed else TestStatus.FAILED
+
+    def reward_signal(self) -> float:
+        """Scalar reward used to update agent neural networks.
+
+        * +1.0  – test failed (found a real defect; useful information)
+        * +0.2  – test errored (interesting, but possibly noisy)
+        * -0.1  – test passed (confirms existing behaviour; less informative)
+
+        The asymmetric reward encourages agents to generate test cases that
+        find failures (the primary goal of adaptive testing).
+        """
+        if self.error:
+            return 0.2
+        return 1.0 if not self.passed else -0.1
+
+
+@dataclass
+class TestSuite:
+    """An ordered collection of :class:`TestCase` objects with run statistics."""
+
+    name: str
+    cases: List[TestCase] = field(default_factory=list)
+    results: List[TestResult] = field(default_factory=list)
+
+    # -- mutators ---------------------------------------------------------
+
+    def add_case(self, case: TestCase) -> None:
+        self.cases.append(case)
+
+    def record_result(self, result: TestResult) -> None:
+        self.results.append(result)
+
+    # -- statistics -------------------------------------------------------
+
+    @property
+    def total(self) -> int:
+        return len(self.results)
+
+    @property
+    def passed(self) -> int:
+        return sum(1 for r in self.results if r.passed and not r.error)
+
+    @property
+    def failed(self) -> int:
+        return sum(1 for r in self.results if not r.passed and not r.error)
+
+    @property
+    def errored(self) -> int:
+        return sum(1 for r in self.results if r.error)
+
+    @property
+    def pass_rate(self) -> float:
+        return self.passed / self.total if self.total > 0 else 0.0
+
+    @property
+    def defect_detection_rate(self) -> float:
+        """Fraction of executions that revealed a defect (failed or errored)."""
+        return (self.failed + self.errored) / self.total if self.total > 0 else 0.0
+
+    @property
+    def average_execution_time_ms(self) -> float:
+        if not self.results:
+            return 0.0
+        return sum(r.execution_time_ms for r in self.results) / len(self.results)
+
+    def summary(self) -> Dict[str, Any]:
+        return {
+            "name": self.name,
+            "total": self.total,
+            "passed": self.passed,
+            "failed": self.failed,
+            "errored": self.errored,
+            "pass_rate": round(self.pass_rate, 4),
+            "defect_detection_rate": round(self.defect_detection_rate, 4),
+            "avg_execution_time_ms": round(self.average_execution_time_ms, 2),
+        }

mannf/core/validation/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Validation sub-package for self-validation and canary testing (Phase 7.4)."""
+
+from mannf.core.validation.self_validation_runner import SelfValidationRunner
+
+__all__ = ["SelfValidationRunner"]

mannf/core/validation/self_validation_runner.py

@@ -0,0 +1,180 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Self-validation canary runner (Phase 7.4).
+
+:class:`SelfValidationRunner` runs a set of *known-good* and *known-bad*
+canary endpoints against the predictor and compares predicted vs. actual
+outcomes.  The accuracy, precision, recall, and F1 metrics are returned in a
+structured report so leadership and auditors can verify detection fidelity.
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+from typing import Any, Dict, List, Optional, Sequence
+
+logger = logging.getLogger(__name__)
+
+
+class SelfValidationRunner:
+    """Compare predicted fault likelihood against known-ground-truth outcomes.
+
+    Parameters
+    ----------
+    known_good:
+        Sequence of endpoint keys (e.g. ``"/api/health"``) that are expected
+        to *pass* (ground truth = non-faulty).
+    known_bad:
+        Sequence of endpoint keys expected to *fail* (ground truth = faulty).
+    threshold:
+        Predicted fault-likelihood above which an endpoint is classified as
+        *faulty*.  Defaults to ``0.5``.
+    """
+
+    def __init__(
+        self,
+        known_good: Sequence[str],
+        known_bad: Sequence[str],
+        threshold: float = 0.5,
+    ) -> None:
+        if not 0.0 < threshold < 1.0:
+            raise ValueError(f"threshold must be in (0.0, 1.0), got {threshold!r}")
+        self.known_good: List[str] = list(known_good)
+        self.known_bad: List[str] = list(known_bad)
+        self.threshold = threshold
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def run(
+        self,
+        predicted: Dict[str, float],
+    ) -> Dict[str, Any]:
+        """Evaluate predicted fault likelihoods against ground-truth labels.
+
+        Parameters
+        ----------
+        predicted:
+            Mapping of ``{endpoint: fault_likelihood}`` (values in [0, 1]).
+            Endpoints absent from *predicted* are treated as ``0.0`` (assumed
+            safe).
+
+        Returns
+        -------
+        dict
+            Structured validation report containing:
+
+            ``accuracy``, ``precision``, ``recall``, ``f1`` — overall
+            classification metrics (float in [0, 1]).
+
+            ``tp``, ``fp``, ``tn``, ``fn`` — raw confusion-matrix counts.
+
+            ``per_endpoint`` — list of per-endpoint dicts with
+            ``endpoint``, ``ground_truth``, ``predicted_score``,
+            ``predicted_label``, ``correct``.
+
+            ``threshold`` — the decision threshold used.
+
+            ``total_canaries`` — total number of canary endpoints evaluated.
+        """
+        per_endpoint: List[Dict[str, Any]] = []
+        tp = fp = tn = fn = 0
+
+        all_canaries = [
+            (ep, True) for ep in self.known_bad
+        ] + [
+            (ep, False) for ep in self.known_good
+        ]
+
+        for ep, ground_truth_faulty in all_canaries:
+            score = float(predicted.get(ep, 0.0))
+            predicted_faulty = score >= self.threshold
+            correct = predicted_faulty == ground_truth_faulty
+
+            if ground_truth_faulty and predicted_faulty:
+                tp += 1
+            elif not ground_truth_faulty and predicted_faulty:
+                fp += 1
+            elif not ground_truth_faulty and not predicted_faulty:
+                tn += 1
+            else:
+                fn += 1
+
+            per_endpoint.append(
+                {
+                    "endpoint": ep,
+                    "ground_truth": "faulty" if ground_truth_faulty else "healthy",
+                    "predicted_score": round(score, 4),
+                    "predicted_label": "faulty" if predicted_faulty else "healthy",
+                    "correct": correct,
+                }
+            )
+
+        total = tp + fp + tn + fn
+        accuracy = (tp + tn) / total if total > 0 else 0.0
+        precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+        recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+        f1 = (
+            2 * precision * recall / (precision + recall)
+            if (precision + recall) > 0
+            else 0.0
+        )
+
+        logger.info(
+            "SelfValidationRunner: accuracy=%.3f precision=%.3f recall=%.3f f1=%.3f "
+            "(tp=%d fp=%d tn=%d fn=%d threshold=%.2f)",
+            accuracy, precision, recall, f1, tp, fp, tn, fn, self.threshold,
+        )
+
+        return {
+            "accuracy": round(accuracy, 4),
+            "precision": round(precision, 4),
+            "recall": round(recall, 4),
+            "f1": round(f1, 4),
+            "tp": tp,
+            "fp": fp,
+            "tn": tn,
+            "fn": fn,
+            "threshold": self.threshold,
+            "total_canaries": total,
+            "per_endpoint": per_endpoint,
+        }
+
+    # ------------------------------------------------------------------
+    # Convenience helper
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def from_risk_report(
+        risk_report: List[Dict[str, Any]],
+        known_good: Sequence[str],
+        known_bad: Sequence[str],
+        threshold: float = 0.5,
+    ) -> "SelfValidationRunner":
+        """Run self-validation directly from a :meth:`RiskScorer.get_risk_report` result.
+
+        Parameters
+        ----------
+        risk_report:
+            Output of ``RiskScorer.get_risk_report()``.
+        known_good:
+            Endpoint keys expected to be healthy.
+        known_bad:
+            Endpoint keys expected to be faulty.
+        threshold:
+            Decision threshold.
+
+        Returns
+        -------
+        SelfValidationRunner
+            A runner pre-configured with the supplied canary lists.
+        """
+        runner = SelfValidationRunner(known_good, known_bad, threshold)
+        predicted = {entry["endpoint"]: entry["risk_score"] for entry in risk_report}
+        runner._last_report = runner.run(predicted)  # type: ignore[attr-defined]
+        return runner

mannf/dashboard/__init__.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+# Backward-compatible shim — source of truth has moved to mannf.product.dashboard
+from mannf.product.dashboard import *  # noqa: F401, F403

mannf/dashboard/app.py

@@ -0,0 +1,9 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+# Backward-compatible shim — source of truth has moved to mannf.product.dashboard.app
+import sys
+import mannf.product.dashboard.app as _real
+sys.modules[__name__] = _real

mannf/dashboard/models.py

@@ -0,0 +1,9 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+# Backward-compatible shim — source of truth has moved to mannf.product.dashboard.models
+import sys
+import mannf.product.dashboard.models as _real
+sys.modules[__name__] = _real