nat-engine 1__py3-none-any.whl

mannf/core/diagnostics/root_cause_analyzer.py

@@ -0,0 +1,273 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""LLM-powered root cause analysis for autonomous loop failures (Phase 6.5).
+
+Calls the configured LLM provider with :data:`ROOT_CAUSE_ANALYSIS_PROMPT`
+(defined in :mod:`mannf.product.llm.prompts`) to suggest likely root causes
+for new failures.
+
+Output schema (JSON returned by LLM, validated here)::
+
+    {
+        "likely_cause": "...",
+        "suggested_fix": "...",
+        "confidence": 0.8,
+        "files_to_check": ["path/to/file.py", ...]
+    }
+
+Missing or invalid fields are filled with safe defaults so the caller always
+gets a usable dict.
+
+The analyzer is gated by:
+- LLM provider availability (returns ``None`` immediately when not configured)
+- Severity threshold (only called for failures whose ``fault_likelihood``
+  exceeds *min_fault_likelihood*)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+# Minimum fault_likelihood to trigger root cause analysis (avoids burning LLM
+# tokens on low-probability noise).
+_DEFAULT_MIN_FAULT_LIKELIHOOD = 0.6
+
+
+class RootCauseResult:
+    """Structured result from the LLM root cause analysis call.
+
+    Attributes
+    ----------
+    likely_cause : str
+    suggested_fix : str
+    confidence : float
+    files_to_check : list[str]
+    raw_response : str
+        The raw string returned by the LLM (for debugging / auditing).
+    """
+
+    def __init__(
+        self,
+        likely_cause: str,
+        suggested_fix: str,
+        confidence: float,
+        files_to_check: List[str],
+        raw_response: str = "",
+    ) -> None:
+        self.likely_cause = likely_cause
+        self.suggested_fix = suggested_fix
+        self.confidence = max(0.0, min(1.0, float(confidence)))
+        self.files_to_check = files_to_check
+        self.raw_response = raw_response
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "likely_cause": self.likely_cause,
+            "suggested_fix": self.suggested_fix,
+            "confidence": self.confidence,
+            "files_to_check": self.files_to_check,
+        }
+
+
+def _parse_llm_response(raw: str) -> Dict[str, Any]:
+    """Extract the JSON object from the LLM response string."""
+    # Strip markdown fences if present
+    text = raw.strip()
+    if text.startswith("```"):
+        lines = text.splitlines()
+        # Drop first line (```json or ```) and last line (```)
+        inner = [l for l in lines[1:] if l.strip() != "```"]
+        text = "\n".join(inner).strip()
+    return json.loads(text)
+
+
+class RootCauseAnalyzer:
+    """Analyse failures with an LLM to suggest root causes.
+
+    Parameters
+    ----------
+    llm_provider : object or None
+        An initialised LLM provider instance that exposes an
+        ``acomplete(prompt: str) -> str`` (async) or
+        ``complete(prompt: str) -> str`` (sync) coroutine / method.
+        When ``None``, :meth:`analyze` returns ``None`` immediately.
+    min_fault_likelihood : float
+        Minimum fault likelihood required to trigger analysis.
+    """
+
+    def __init__(
+        self,
+        llm_provider: Optional[Any] = None,
+        min_fault_likelihood: float = _DEFAULT_MIN_FAULT_LIKELIHOOD,
+    ) -> None:
+        self._provider = llm_provider
+        self._min_fault_likelihood = min_fault_likelihood
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    async def analyze(
+        self,
+        failure: Dict[str, Any],
+        fault_likelihood: float = 1.0,
+        belief_history: Optional[List[Dict[str, Any]]] = None,
+        dom_snapshot: str = "",
+        scenario_steps: Optional[List[str]] = None,
+        page_metadata: Optional[Dict[str, Any]] = None,
+    ) -> Optional[RootCauseResult]:
+        """Analyse a single failure and return a :class:`RootCauseResult`.
+
+        Returns ``None`` when LLM is unavailable or fault_likelihood is below
+        threshold.
+
+        Parameters
+        ----------
+        failure:
+            Failure dict with at least ``url``, ``error``, ``task_id`` keys.
+        fault_likelihood:
+            Current belief state fault_likelihood for the page (0–1).
+        belief_history:
+            Optional list of prior belief snapshots for this page, each a
+            dict with ``iteration``, ``fault_likelihood``, ``timestamp``.
+        dom_snapshot:
+            Optional DOM/HTML snapshot string for context.
+        scenario_steps:
+            Optional list of step descriptions from the failing scenario.
+        page_metadata:
+            Optional metadata dict about the page.
+        """
+        if self._provider is None:
+            return None
+
+        if fault_likelihood < self._min_fault_likelihood:
+            return None
+
+        prompt = self._build_prompt(
+            failure=failure,
+            fault_likelihood=fault_likelihood,
+            belief_history=belief_history or [],
+            dom_snapshot=dom_snapshot,
+            scenario_steps=scenario_steps or [],
+            page_metadata=page_metadata or {},
+        )
+
+        try:
+            raw = await self._call_llm(prompt)
+            return self._parse_result(raw)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("RootCauseAnalyzer: LLM call failed — %s", exc)
+            return None
+
+    def analyze_sync(
+        self,
+        failure: Dict[str, Any],
+        fault_likelihood: float = 1.0,
+        belief_history: Optional[List[Dict[str, Any]]] = None,
+        dom_snapshot: str = "",
+        scenario_steps: Optional[List[str]] = None,
+        page_metadata: Optional[Dict[str, Any]] = None,
+    ) -> Optional[RootCauseResult]:
+        """Synchronous variant of :meth:`analyze`."""
+        if self._provider is None:
+            return None
+        if fault_likelihood < self._min_fault_likelihood:
+            return None
+
+        prompt = self._build_prompt(
+            failure=failure,
+            fault_likelihood=fault_likelihood,
+            belief_history=belief_history or [],
+            dom_snapshot=dom_snapshot,
+            scenario_steps=scenario_steps or [],
+            page_metadata=page_metadata or {},
+        )
+
+        try:
+            raw = self._call_llm_sync(prompt)
+            return self._parse_result(raw)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("RootCauseAnalyzer: sync LLM call failed — %s", exc)
+            return None
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _build_prompt(
+        self,
+        failure: Dict[str, Any],
+        fault_likelihood: float,
+        belief_history: List[Dict[str, Any]],
+        dom_snapshot: str,
+        scenario_steps: List[str],
+        page_metadata: Dict[str, Any],
+    ) -> str:
+        from mannf.product.llm.prompts import ROOT_CAUSE_ANALYSIS_PROMPT  # noqa: PLC0415
+
+        belief_history_str = (
+            json.dumps(belief_history, indent=2) if belief_history else "No prior history available"
+        )
+        scenario_steps_str = (
+            "\n".join(f"  {i + 1}. {s}" for i, s in enumerate(scenario_steps))
+            if scenario_steps
+            else "  (no steps recorded)"
+        )
+        dom_snippet = (dom_snapshot or "")[:2000]  # cap DOM context length
+        page_meta_str = json.dumps(page_metadata) if page_metadata else "{}"
+
+        return ROOT_CAUSE_ANALYSIS_PROMPT.format(
+            url=failure.get("url", ""),
+            page=failure.get("page", ""),
+            error=failure.get("error", ""),
+            task_id=failure.get("task_id", ""),
+            fault_likelihood=round(fault_likelihood, 3),
+            belief_history=belief_history_str,
+            dom_snapshot=dom_snippet,
+            scenario_steps=scenario_steps_str,
+            page_metadata=page_meta_str,
+        )
+
+    async def _call_llm(self, prompt: str) -> str:
+        """Invoke the LLM provider (async)."""
+        provider = self._provider
+        if hasattr(provider, "acomplete"):
+            return await provider.acomplete(prompt)
+        if hasattr(provider, "complete"):
+            result = provider.complete(prompt)
+            # Handle both sync and coroutine returns
+            if hasattr(result, "__await__"):
+                return await result
+            return result
+        raise AttributeError("LLM provider has no 'acomplete' or 'complete' method")
+
+    def _call_llm_sync(self, prompt: str) -> str:
+        """Invoke the LLM provider (sync)."""
+        provider = self._provider
+        if hasattr(provider, "complete"):
+            return provider.complete(prompt)
+        raise AttributeError("LLM provider has no 'complete' method for sync use")
+
+    @staticmethod
+    def _parse_result(raw: str) -> RootCauseResult:
+        """Parse the LLM JSON response into a :class:`RootCauseResult`."""
+        try:
+            data = _parse_llm_response(raw)
+        except (json.JSONDecodeError, ValueError):
+            logger.warning("RootCauseAnalyzer: could not parse LLM response as JSON")
+            data = {}
+
+        return RootCauseResult(
+            likely_cause=str(data.get("likely_cause", "Unable to determine root cause")),
+            suggested_fix=str(data.get("suggested_fix", "Investigate the error manually")),
+            confidence=float(data.get("confidence", 0.0)),
+            files_to_check=list(data.get("files_to_check", [])),
+            raw_response=raw,
+        )

mannf/core/distributed/__init__.py

@@ -0,0 +1,16 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Distributed sub-package for the Multi-Agent Neural Network Framework."""
+
+from mannf.core.distributed.endpoint import Endpoint, EndpointRegistry
+from mannf.core.distributed.system_under_test import SystemUnderTest, MockDistributedSystem
+
+__all__ = [
+    "Endpoint",
+    "EndpointRegistry",
+    "SystemUnderTest",
+    "MockDistributedSystem",
+]

mannf/core/distributed/endpoint.py

@@ -0,0 +1,139 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Distributed endpoint model.
+
+An :class:`Endpoint` represents one service/component in the system under test.
+It stores connection metadata and exposes a uniform *invoke* interface.
+
+Supported protocol values include:
+
+* ``"http"`` / ``"https"`` — REST HTTP APIs
+* ``"grpc"`` — gRPC services (plaintext)
+* ``"grpcs"`` — gRPC services (TLS)
+* ``"ws"`` — WebSocket connections (plaintext)
+* ``"wss"`` — WebSocket connections (TLS)
+* ``"kafka"`` — Kafka broker
+* ``"tcp"`` — generic TCP
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+# Protocol families for quick categorisation
+_HTTP_PROTOCOLS = frozenset({"http", "https"})
+_GRPC_PROTOCOLS = frozenset({"grpc", "grpcs"})
+_WS_PROTOCOLS = frozenset({"ws", "wss", "websocket", "websockets"})
+_ASYNC_PROTOCOLS = frozenset({"kafka", "amqp", "amqps", "mqtt"})
+
+
+@dataclass
+class Endpoint:
+    """Metadata about one distributed service endpoint.
+
+    Parameters
+    ----------
+    name:
+        Logical name of the service (e.g. ``"auth-service"``).
+    host:
+        Hostname or IP address.
+    port:
+        TCP port.
+    protocol:
+        Application protocol — one of ``"http"``, ``"https"``, ``"grpc"``,
+        ``"grpcs"``, ``"ws"``, ``"wss"``, ``"kafka"``, ``"tcp"``.
+    health_check_path:
+        Path used to verify the service is alive (HTTP/WebSocket only).
+    tags:
+        Arbitrary labels for filtering/grouping.
+    """
+
+    name: str
+    host: str
+    port: int
+    protocol: str = "http"
+    health_check_path: str = "/health"
+    tags: Dict[str, str] = field(default_factory=dict)
+
+    # ------------------------------------------------------------------
+    # Protocol family helpers
+    # ------------------------------------------------------------------
+
+    @property
+    def is_http(self) -> bool:
+        """Return True for HTTP/HTTPS endpoints."""
+        return self.protocol in _HTTP_PROTOCOLS
+
+    @property
+    def is_grpc(self) -> bool:
+        """Return True for gRPC endpoints (plain or TLS)."""
+        return self.protocol in _GRPC_PROTOCOLS
+
+    @property
+    def is_websocket(self) -> bool:
+        """Return True for WebSocket endpoints (plain or TLS)."""
+        return self.protocol in _WS_PROTOCOLS
+
+    @property
+    def is_async(self) -> bool:
+        """Return True for async/event-driven endpoints (Kafka, AMQP, MQTT)."""
+        return self.protocol in _ASYNC_PROTOCOLS
+
+    # ------------------------------------------------------------------
+    # URL builders
+    # ------------------------------------------------------------------
+
+    @property
+    def base_url(self) -> str:
+        if self.protocol == "http":
+            return f"http://{self.host}:{self.port}"
+        if self.protocol == "https":
+            return f"https://{self.host}:{self.port}"
+        if self.protocol in ("grpc", "grpcs"):
+            return f"{self.host}:{self.port}"
+        if self.protocol == "ws":
+            return f"ws://{self.host}:{self.port}"
+        if self.protocol in ("wss", "websocket", "websockets"):
+            return f"wss://{self.host}:{self.port}"
+        return f"{self.protocol}://{self.host}:{self.port}"
+
+    def __str__(self) -> str:
+        return f"{self.name}@{self.base_url}"
+
+
+@dataclass
+class EndpointRegistry:
+    """Holds all known endpoints for the system under test."""
+
+    _endpoints: Dict[str, Endpoint] = field(default_factory=dict)
+
+    def register(self, endpoint: Endpoint) -> None:
+        self._endpoints[endpoint.name] = endpoint
+
+    def get(self, name: str) -> Optional[Endpoint]:
+        return self._endpoints.get(name)
+
+    def all(self) -> list[Endpoint]:
+        return list(self._endpoints.values())
+
+    def by_protocol(self, protocol: str) -> list[Endpoint]:
+        """Return all endpoints with the given *protocol*."""
+        return [e for e in self._endpoints.values() if e.protocol == protocol]
+
+    def grpc_endpoints(self) -> list[Endpoint]:
+        """Return all gRPC endpoints."""
+        return [e for e in self._endpoints.values() if e.is_grpc]
+
+    def websocket_endpoints(self) -> list[Endpoint]:
+        """Return all WebSocket endpoints."""
+        return [e for e in self._endpoints.values() if e.is_websocket]
+
+    def by_tag(self, key: str, value: str) -> list[Endpoint]:
+        return [e for e in self._endpoints.values() if e.tags.get(key) == value]
+
+    def __len__(self) -> int:
+        return len(self._endpoints)

mannf/core/distributed/system_under_test.py

@@ -0,0 +1,207 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""System-Under-Test (SUT) abstraction.
+
+:class:`SystemUnderTest` is the interface the agents use to interact with the
+distributed system being tested.  The concrete class provided here is
+:class:`MockDistributedSystem`, a fully in-process simulation of a small
+microservice landscape that is ready to use without any network infrastructure.
+
+The mock deliberately injects configurable faults so the framework can
+demonstrate its ability to find them adaptively.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import random
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+from mannf.core.distributed.endpoint import Endpoint, EndpointRegistry
+from mannf.core.testing.models import TestCase, TestResult
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Abstract interface
+# ---------------------------------------------------------------------------
+
+class SystemUnderTest(ABC):
+    """Abstract base class for any system the framework will test."""
+
+    @abstractmethod
+    async def execute(self, test_case: TestCase) -> TestResult:
+        """Execute *test_case* and return its result."""
+
+    @property
+    @abstractmethod
+    def registry(self) -> EndpointRegistry:
+        """Return the endpoint registry for this system."""
+
+
+# ---------------------------------------------------------------------------
+# Mock distributed system
+# ---------------------------------------------------------------------------
+
+@dataclass
+class _ServiceConfig:
+    """Configuration for one simulated microservice."""
+
+    name: str
+    failure_rate: float = 0.0         # probability of returning a failure response
+    error_rate: float = 0.0           # probability of raising an exception
+    latency_ms: float = 10.0          # base latency
+    latency_jitter_ms: float = 5.0    # latency jitter (uniform)
+
+
+class MockDistributedSystem(SystemUnderTest):
+    """An in-process simulation of a multi-service distributed system.
+
+    Services can be added with configurable failure/error rates so that the
+    adaptive testing agents can demonstrate their ability to converge on
+    high-failure areas.
+
+    Parameters
+    ----------
+    seed:
+        Optional random seed for reproducible simulations.
+    """
+
+    # Default service topology
+    _DEFAULT_SERVICES: List[_ServiceConfig] = [
+        _ServiceConfig("api-gateway",      failure_rate=0.05, error_rate=0.01, latency_ms=15),
+        _ServiceConfig("auth-service",     failure_rate=0.10, error_rate=0.02, latency_ms=20),
+        _ServiceConfig("user-service",     failure_rate=0.03, error_rate=0.01, latency_ms=10),
+        _ServiceConfig("order-service",    failure_rate=0.15, error_rate=0.03, latency_ms=25),
+        _ServiceConfig("payment-service",  failure_rate=0.20, error_rate=0.05, latency_ms=30),
+        _ServiceConfig("inventory-service",failure_rate=0.08, error_rate=0.02, latency_ms=12),
+        _ServiceConfig("notification-svc", failure_rate=0.04, error_rate=0.01, latency_ms=8),
+        _ServiceConfig("analytics-service",failure_rate=0.06, error_rate=0.01, latency_ms=50),
+    ]
+
+    def __init__(
+        self,
+        services: Optional[List[_ServiceConfig]] = None,
+        seed: Optional[int] = None,
+    ) -> None:
+        if seed is not None:
+            random.seed(seed)
+
+        self._services: Dict[str, _ServiceConfig] = {}
+        self._registry = EndpointRegistry()
+
+        for svc in services or self._DEFAULT_SERVICES:
+            self._services[svc.name] = svc
+            self._registry.register(
+                Endpoint(
+                    name=svc.name,
+                    host="localhost",
+                    port=8000 + len(self._services),
+                    protocol="http",
+                )
+            )
+
+        self._call_log: List[Dict[str, Any]] = []
+
+    # -- SystemUnderTest interface ----------------------------------------
+
+    @property
+    def registry(self) -> EndpointRegistry:
+        return self._registry
+
+    async def execute(self, test_case: TestCase) -> TestResult:
+        """Simulate executing *test_case* against the target service."""
+        start = time.monotonic()
+
+        service = self._services.get(test_case.target)
+        if service is None:
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=False,
+                error=f"Unknown service: {test_case.target}",
+                execution_time_ms=(time.monotonic() - start) * 1000,
+            )
+
+        # Simulate network latency
+        jitter = random.uniform(0, service.latency_jitter_ms)
+        await asyncio.sleep((service.latency_ms + jitter) / 1000.0)
+
+        execution_time_ms = (time.monotonic() - start) * 1000
+
+        # Simulate errors (exceptions)
+        if random.random() < service.error_rate:
+            error_msg = f"Simulated error in {service.name}: connection timeout"
+            self._log(test_case, passed=False, error=error_msg)
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=False,
+                error=error_msg,
+                execution_time_ms=execution_time_ms,
+            )
+
+        # Simulate functional failures
+        passed = random.random() >= service.failure_rate
+
+        output = {
+            "service": service.name,
+            "status": "ok" if passed else "error",
+            "inputs_received": test_case.inputs,
+        }
+        self._log(test_case, passed=passed)
+        return TestResult(
+            test_case_id=test_case.id,
+            passed=passed,
+            actual_output=output,
+            execution_time_ms=execution_time_ms,
+        )
+
+    # -- Introspection helpers --------------------------------------------
+
+    def _log(
+        self,
+        test_case: TestCase,
+        passed: bool,
+        error: Optional[str] = None,
+    ) -> None:
+        self._call_log.append(
+            {
+                "target": test_case.target,
+                "passed": passed,
+                "error": error,
+            }
+        )
+
+    def call_log(self) -> List[Dict[str, Any]]:
+        """Return a copy of the call log (for testing / observability)."""
+        return list(self._call_log)
+
+    def failure_counts(self) -> Dict[str, int]:
+        """Return per-service failure+error counts."""
+        counts: Dict[str, int] = {}
+        for entry in self._call_log:
+            if not entry["passed"] or entry["error"]:
+                counts[entry["target"]] = counts.get(entry["target"], 0) + 1
+        return counts
+
+    def service_names(self) -> List[str]:
+        return list(self._services.keys())
+
+    def add_service(self, config: _ServiceConfig) -> None:
+        """Dynamically register an additional simulated service."""
+        self._services[config.name] = config
+        self._registry.register(
+            Endpoint(
+                name=config.name,
+                host="localhost",
+                port=8000 + len(self._services),
+                protocol="http",
+            )
+        )