nat-engine 1__py3-none-any.whl

mannf/product/integrations/websocket_sut.py

@@ -0,0 +1,434 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""WebSocket System-Under-Test implementation.
+
+:class:`WebSocketSUT` connects to WebSocket endpoints, sends/receives messages,
+and detects anomalies (unexpected disconnects, malformed frames, high latency).
+It implements the same ``execute()`` / ``health_check()`` interface as
+:class:`~mannf.product.integrations.http_sut.HttpApiSUT`.
+
+The adapter uses the ``websockets`` library when available.  When ``websockets``
+is not installed, execute() returns a skip result with an informative error
+so the rest of NAT continues to function.
+
+Anomaly detection includes:
+
+* **Disconnect rate** — fraction of test runs that ended with an unexpected
+  close from the server.
+* **Message latency** — time between send and first received response.
+* **Frame integrity** — responses that cannot be decoded as JSON are flagged.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import time
+from typing import Any, Dict, List, Optional
+
+from mannf.core.distributed.endpoint import Endpoint, EndpointRegistry
+from mannf.core.distributed.system_under_test import SystemUnderTest
+from mannf.core.testing.models import TestCase, TestResult
+
+logger = logging.getLogger(__name__)
+
+
+def _websockets_available() -> bool:
+    """Return True if the ``websockets`` package is installed."""
+    try:
+        import websockets  # noqa: F401
+        return True
+    except ImportError:
+        return False
+
+
+def _parse_ws_url(url: str) -> tuple[str, int, str]:
+    """Parse a WebSocket URL into (host, port, path)."""
+    scheme_stripped = url
+    for prefix in ("wss://", "ws://"):
+        if url.startswith(prefix):
+            scheme_stripped = url[len(prefix):]
+            break
+    if "/" in scheme_stripped:
+        hostport, _, path = scheme_stripped.partition("/")
+        path = "/" + path
+    else:
+        hostport = scheme_stripped
+        path = "/"
+    if ":" in hostport:
+        host, _, port_str = hostport.partition(":")
+        try:
+            port = int(port_str)
+        except ValueError:
+            port = 443 if url.startswith("wss://") else 80
+    else:
+        host = hostport
+        port = 443 if url.startswith("wss://") else 80
+    return host, port, path
+
+
+class WebSocketSUT(SystemUnderTest):
+    """Execute WebSocket test cases against a live WebSocket server.
+
+    Parameters
+    ----------
+    ws_url:
+        Base WebSocket URL (e.g. ``"ws://localhost:8765"``).
+    timeout:
+        Connect and send/receive timeout in seconds (default: 10).
+    headers:
+        Additional HTTP headers sent during the WebSocket handshake.
+    max_message_size:
+        Maximum size (bytes) of a single received message (default: 1 MB).
+    """
+
+    def __init__(
+        self,
+        ws_url: str,
+        timeout: float = 10.0,
+        headers: Optional[Dict[str, str]] = None,
+        max_message_size: int = 1024 * 1024,
+    ) -> None:
+        self.ws_url = ws_url.rstrip("/")
+        self.timeout = timeout
+        self.headers: Dict[str, str] = headers or {}
+        self.max_message_size = max_message_size
+
+        # Connection state tracking for anomaly detection / risk scoring
+        self._disconnect_count: int = 0
+        self._total_runs: int = 0
+        self._latency_history: List[float] = []
+
+        host, port, _path = _parse_ws_url(ws_url)
+        self._registry = EndpointRegistry()
+        self._registry.register(
+            Endpoint(
+                name="default",
+                host=host,
+                port=port,
+                protocol="wss" if ws_url.startswith("wss://") else "ws",
+            )
+        )
+
+    # ------------------------------------------------------------------
+    # SystemUnderTest interface
+    # ------------------------------------------------------------------
+
+    @property
+    def registry(self) -> EndpointRegistry:
+        return self._registry
+
+    @property
+    def disconnect_rate(self) -> float:
+        """Fraction of runs that ended with an unexpected server disconnect."""
+        if self._total_runs == 0:
+            return 0.0
+        return self._disconnect_count / self._total_runs
+
+    @property
+    def average_latency_ms(self) -> float:
+        """Average send→receive latency in milliseconds across all runs."""
+        if not self._latency_history:
+            return 0.0
+        return sum(self._latency_history) / len(self._latency_history)
+
+    async def execute(self, test_case: TestCase) -> TestResult:
+        """Connect, send a message (if specified), and receive a response.
+
+        ``test_case.target`` is treated as the channel path to append to
+        ``ws_url`` (e.g. ``"/chat"`` → ``"ws://localhost:8765/chat"``).
+
+        ``test_case.inputs`` may contain:
+
+        * ``action`` — one of ``"send_receive"`` (default), ``"connect_close"``,
+          ``"receive_only"``, ``"reconnect"`` (disconnect then reconnect).
+        * ``payload`` — dict or string to send as a message.
+        * ``expected_response_keys`` — list of JSON keys expected in the first
+          received message.
+        * ``receive_timeout`` — per-receive timeout in seconds (default: ``timeout``).
+        """
+        if not _websockets_available():
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=False,
+                actual_output=None,
+                execution_time_ms=0.0,
+                error=(
+                    "websockets package is not installed. "
+                    "Install it with: pip install websockets"
+                ),
+            )
+
+        action: str = str(test_case.inputs.get("action", "send_receive")).lower()
+        channel_path: str = test_case.target or ""
+        if channel_path and not channel_path.startswith("/"):
+            channel_path = "/" + channel_path
+
+        full_url = self.ws_url + channel_path
+
+        self._total_runs += 1
+
+        if action == "connect_close":
+            return await self._action_connect_close(test_case, full_url)
+        elif action == "receive_only":
+            return await self._action_receive_only(test_case, full_url)
+        elif action == "reconnect":
+            return await self._action_reconnect(test_case, full_url)
+        else:
+            return await self._action_send_receive(test_case, full_url)
+
+    async def health_check(self) -> bool:
+        """Check that the WebSocket server is reachable by opening and closing a connection."""
+        if not _websockets_available():
+            return False
+        try:
+            import websockets
+
+            async def _health_check_connect_close() -> None:
+                ws = await websockets.connect(
+                    self.ws_url,
+                    additional_headers=self.headers,
+                    max_size=self.max_message_size,
+                )
+                await ws.close()
+
+            await asyncio.wait_for(_health_check_connect_close(), timeout=min(self.timeout, 5.0))
+            return True
+        except Exception:  # noqa: BLE001
+            return False
+
+    # ------------------------------------------------------------------
+    # Action helpers
+    # ------------------------------------------------------------------
+
+    async def _action_connect_close(self, test_case: TestCase, url: str) -> TestResult:
+        """Simply connect and close — verify the handshake succeeds."""
+        start = time.monotonic()
+        try:
+            import websockets
+
+            async def _test_connection() -> None:
+                _ws = await websockets.connect(
+                    url,
+                    additional_headers=self.headers,
+                    max_size=self.max_message_size,
+                )
+                await _ws.close()
+
+            await asyncio.wait_for(_test_connection(), timeout=self.timeout)
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=True,
+                actual_output={"connected": True},
+                execution_time_ms=elapsed_ms,
+                error=None,
+            )
+        except Exception as exc:  # noqa: BLE001
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            self._disconnect_count += 1
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=False,
+                actual_output=None,
+                execution_time_ms=elapsed_ms,
+                error=f"WebSocket connection failed: {exc}",
+            )
+
+    async def _action_send_receive(self, test_case: TestCase, url: str) -> TestResult:
+        """Connect, send a payload, and receive one response message."""
+        import websockets
+
+        payload = test_case.inputs.get("payload")
+        expected_keys: list = test_case.inputs.get("expected_response_keys") or []
+        receive_timeout = float(test_case.inputs.get("receive_timeout") or self.timeout)
+
+        if isinstance(payload, dict):
+            message_str = json.dumps(payload)
+        else:
+            message_str = str(payload) if payload is not None else ""
+
+        start = time.monotonic()
+        try:
+            ws = await asyncio.wait_for(
+                websockets.connect(
+                    url,
+                    additional_headers=self.headers,
+                    max_size=self.max_message_size,
+                ),
+                timeout=self.timeout,
+            )
+
+            send_time = time.monotonic()
+            if message_str:
+                await asyncio.wait_for(ws.send(message_str), timeout=self.timeout)
+
+            # Receive
+            raw_response: Any = None
+            try:
+                raw_response = await asyncio.wait_for(
+                    ws.recv(), timeout=receive_timeout
+                )
+            except asyncio.TimeoutError:
+                await ws.close()
+                elapsed_ms = (time.monotonic() - start) * 1000.0
+                self._disconnect_count += 1
+                return TestResult(
+                    test_case_id=test_case.id,
+                    passed=False,
+                    actual_output=None,
+                    execution_time_ms=elapsed_ms,
+                    error="Timed out waiting for WebSocket response",
+                )
+
+            recv_latency_ms = (time.monotonic() - send_time) * 1000.0
+            self._latency_history.append(recv_latency_ms)
+
+            # Try to parse as JSON
+            parsed_response: Any = raw_response
+            try:
+                parsed_response = json.loads(raw_response)
+            except (json.JSONDecodeError, TypeError):
+                pass
+
+            await ws.close()
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+
+            # Validate expected keys
+            passed = True
+            error_msg: Optional[str] = None
+            if expected_keys and isinstance(parsed_response, dict):
+                missing = [k for k in expected_keys if k not in parsed_response]
+                if missing:
+                    passed = False
+                    error_msg = f"Response missing expected keys: {missing}"
+
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=passed,
+                actual_output=parsed_response,
+                execution_time_ms=elapsed_ms,
+                error=error_msg,
+            )
+
+        except Exception as exc:  # noqa: BLE001
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            self._disconnect_count += 1
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=False,
+                actual_output=None,
+                execution_time_ms=elapsed_ms,
+                error=f"WebSocket error: {exc}",
+            )
+
+    async def _action_receive_only(self, test_case: TestCase, url: str) -> TestResult:
+        """Connect and wait for an unsolicited server message."""
+        import websockets
+
+        receive_timeout = float(test_case.inputs.get("receive_timeout") or self.timeout)
+        start = time.monotonic()
+        try:
+            ws = await asyncio.wait_for(
+                websockets.connect(
+                    url,
+                    additional_headers=self.headers,
+                    max_size=self.max_message_size,
+                ),
+                timeout=self.timeout,
+            )
+
+            recv_start = time.monotonic()
+            try:
+                raw = await asyncio.wait_for(ws.recv(), timeout=receive_timeout)
+            except asyncio.TimeoutError:
+                await ws.close()
+                elapsed_ms = (time.monotonic() - start) * 1000.0
+                return TestResult(
+                    test_case_id=test_case.id,
+                    passed=False,
+                    actual_output=None,
+                    execution_time_ms=elapsed_ms,
+                    error="Timed out waiting for server-push message",
+                )
+
+            recv_latency_ms = (time.monotonic() - recv_start) * 1000.0
+            self._latency_history.append(recv_latency_ms)
+            await ws.close()
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+
+            try:
+                parsed = json.loads(raw)
+            except (json.JSONDecodeError, TypeError):
+                parsed = raw
+
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=True,
+                actual_output=parsed,
+                execution_time_ms=elapsed_ms,
+                error=None,
+            )
+
+        except Exception as exc:  # noqa: BLE001
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            self._disconnect_count += 1
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=False,
+                actual_output=None,
+                execution_time_ms=elapsed_ms,
+                error=f"WebSocket error: {exc}",
+            )
+
+    async def _action_reconnect(self, test_case: TestCase, url: str) -> TestResult:
+        """Disconnect abruptly then reconnect — verify server handles reconnection."""
+        import websockets
+
+        start = time.monotonic()
+        try:
+            # First connection — close without handshake
+            ws = await asyncio.wait_for(
+                websockets.connect(
+                    url,
+                    additional_headers=self.headers,
+                    max_size=self.max_message_size,
+                ),
+                timeout=self.timeout,
+            )
+            await ws.close(code=1001)  # going away
+
+            # Second connection — should succeed
+            ws2 = await asyncio.wait_for(
+                websockets.connect(
+                    url,
+                    additional_headers=self.headers,
+                    max_size=self.max_message_size,
+                ),
+                timeout=self.timeout,
+            )
+            await ws2.close()
+
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=True,
+                actual_output={"reconnected": True},
+                execution_time_ms=elapsed_ms,
+                error=None,
+            )
+
+        except Exception as exc:  # noqa: BLE001
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            self._disconnect_count += 1
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=False,
+                actual_output=None,
+                execution_time_ms=elapsed_ms,
+                error=f"WebSocket reconnection failed: {exc}",
+            )

mannf/product/llm/__init__.py

@@ -0,0 +1,25 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""LLM provider package for NAT.
+
+Provides a pluggable LLM abstraction layer for augmenting test case generation
+with AI-generated edge-case, adversarial, and semantic inputs.
+
+Example usage::
+
+    from mannf.product.llm import LLMProvider, get_provider
+    from mannf.product.llm.config import LLMConfig
+
+    config = LLMConfig(provider="openai", api_key="sk-...")
+    provider = get_provider(config)
+    if provider.is_available():
+        cases = await provider.generate_test_cases(endpoint_info, spec_context, n=5)
+"""
+
+from mannf.product.llm.base import LLMProvider
+from mannf.product.llm.factory import get_provider
+
+__all__ = ["LLMProvider", "get_provider"]

mannf/product/llm/anthropic_provider.py

@@ -0,0 +1,94 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Anthropic LLM provider (httpx-based, no anthropic SDK dependency)."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+from typing import Any, Optional
+
+import httpx
+
+from mannf.product.llm.base import LLMProvider
+from mannf.product.llm.config import LLMConfig
+
+logger = logging.getLogger(__name__)
+
+_ANTHROPIC_API_URL = "https://api.anthropic.com/v1/messages"
+_ANTHROPIC_VERSION = "2023-06-01"
+
+
+class AnthropicProvider(LLMProvider):
+    """LLM provider that calls the Anthropic Messages API.
+
+    Uses ``httpx`` directly — no ``anthropic`` SDK dependency required.
+
+    Parameters
+    ----------
+    config:
+        :class:`~mannf.product.llm.config.LLMConfig` instance.  When ``None``, a
+        default config is created (reads ``ANTHROPIC_API_KEY`` from env).
+    """
+
+    def __init__(self, config: Optional[LLMConfig] = None) -> None:
+        if config is None:
+            config = LLMConfig(provider="anthropic")
+        self._config = config
+        self._api_key: str = config.api_key or os.environ.get("ANTHROPIC_API_KEY", "")
+
+    @property
+    def provider_name(self) -> str:
+        return "anthropic"
+
+    def is_available(self) -> bool:
+        return bool(self._api_key)
+
+    async def generate(self, prompt: str, **kwargs: Any) -> str:
+        """Send *prompt* to the Anthropic Messages API and return the response text."""
+        if not self.is_available():
+            logger.warning("AnthropicProvider: no API key configured")
+            return ""
+
+        headers = {
+            "x-api-key": self._api_key,
+            "anthropic-version": _ANTHROPIC_VERSION,
+            "Content-Type": "application/json",
+        }
+        payload = {
+            "model": self._config.model,
+            "max_tokens": self._config.max_tokens,
+            "temperature": self._config.temperature,
+            "messages": [{"role": "user", "content": prompt}],
+        }
+
+        last_exc: Exception = RuntimeError("No attempts made")
+        for attempt in range(self._config.max_retries):
+            try:
+                async with httpx.AsyncClient(timeout=self._config.timeout) as client:
+                    resp = await client.post(_ANTHROPIC_API_URL, headers=headers, json=payload)
+                    resp.raise_for_status()
+                    data = resp.json()
+                    return data["content"][0]["text"]
+            except (httpx.HTTPStatusError, httpx.RequestError, KeyError, json.JSONDecodeError) as exc:
+                last_exc = exc
+                if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code in (400, 401, 403):
+                    logger.error("AnthropicProvider: non-retryable error %s", exc)
+                    return ""
+                wait = 2 ** attempt
+                logger.warning(
+                    "AnthropicProvider: attempt %d/%d failed (%s), retrying in %ds",
+                    attempt + 1,
+                    self._config.max_retries,
+                    exc,
+                    wait,
+                )
+                await asyncio.sleep(wait)
+
+        logger.error("AnthropicProvider: all retries exhausted: %s", last_exc)
+        return ""