nat-engine 1__py3-none-any.whl

mannf/product/integrations/grpc_sut.py

@@ -0,0 +1,469 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""gRPC System-Under-Test implementation.
+
+:class:`GrpcSUT` invokes unary and streaming RPC methods against a live gRPC
+service.  It implements the same ``execute()`` / ``health_check()`` interface
+as :class:`~mannf.product.integrations.http_sut.HttpApiSUT` so the
+:class:`~mannf.nat_orchestrator.NATOrchestrator` and BDI agent pipeline work
+unchanged.
+
+The adapter uses ``grpcio`` when available.  When ``grpcio`` is *not*
+installed it falls back to a lightweight HTTP/2 hint-based mode that records a
+skip result, allowing the rest of NAT to function in environments without the
+``grpcio`` package.
+
+Supported RPC types:
+
+* **Unary** — single request / single response.
+* **Server-streaming** — single request / multiple responses consumed.
+* **Client-streaming** — multiple requests / single response (sent sequentially).
+* **Bidirectional streaming** — multiple requests / multiple responses.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any, Dict, Iterator, List, Optional
+
+from mannf.core.distributed.endpoint import Endpoint, EndpointRegistry
+from mannf.core.distributed.system_under_test import SystemUnderTest
+from mannf.core.testing.models import TestCase, TestResult
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# gRPC status code constants (mirrors grpc.StatusCode)
+# ---------------------------------------------------------------------------
+
+_OK = 0
+_STATUS_NAMES: dict[int, str] = {
+    0: "OK",
+    1: "CANCELLED",
+    2: "UNKNOWN",
+    3: "INVALID_ARGUMENT",
+    4: "DEADLINE_EXCEEDED",
+    5: "NOT_FOUND",
+    6: "ALREADY_EXISTS",
+    7: "PERMISSION_DENIED",
+    8: "RESOURCE_EXHAUSTED",
+    9: "FAILED_PRECONDITION",
+    10: "ABORTED",
+    11: "OUT_OF_RANGE",
+    12: "UNIMPLEMENTED",
+    13: "INTERNAL",
+    14: "UNAVAILABLE",
+    15: "DATA_LOSS",
+    16: "UNAUTHENTICATED",
+}
+
+
+def _grpc_available() -> bool:
+    """Return True if the ``grpcio`` package is installed."""
+    try:
+        import grpc  # noqa: F401
+        return True
+    except ImportError:
+        return False
+
+
+def _passthrough(x: Any) -> Any:
+    """Identity serializer/deserializer — pass raw bytes through unchanged."""
+    return x
+
+
+def _make_stub(channel: Any, service_name: str, method_name: str) -> Any:
+    """Create a dynamic gRPC stub method using the generic unary–unary approach."""
+    method_path = f"/{service_name}/{method_name}"
+    return channel.unary_unary(
+        method_path,
+        request_serializer=_passthrough,
+        response_deserializer=_passthrough,
+    )
+
+
+class GrpcSUT(SystemUnderTest):
+    """Execute gRPC test cases against a live gRPC service.
+
+    Parameters
+    ----------
+    endpoint:
+        gRPC server address in ``host:port`` format
+        (e.g. ``"localhost:50051"``).
+    timeout:
+        Per-call timeout in seconds (default: 10).
+    use_tls:
+        Use TLS credentials (default: ``False`` — insecure channel).
+    metadata:
+        Sequence of ``(key, value)`` pairs sent as gRPC call metadata
+        (e.g. for authentication tokens).
+    max_streaming_responses:
+        Maximum number of responses to collect from server-streaming or
+        bidi-streaming RPCs before closing the stream (default: 100).
+    """
+
+    def __init__(
+        self,
+        endpoint: str,
+        timeout: float = 10.0,
+        use_tls: bool = False,
+        metadata: Optional[List[tuple]] = None,
+        max_streaming_responses: int = 100,
+    ) -> None:
+        self.endpoint = endpoint
+        self.timeout = timeout
+        self.use_tls = use_tls
+        self.call_metadata: List[tuple] = metadata or []
+        self.max_streaming_responses = max_streaming_responses
+
+        self._registry = EndpointRegistry()
+        host, _, port_str = endpoint.partition(":")
+        try:
+            port = int(port_str) if port_str else 50051
+        except ValueError:
+            port = 50051
+        self._registry.register(
+            Endpoint(
+                name="default",
+                host=host or "localhost",
+                port=port,
+                protocol="grpc",
+            )
+        )
+
+    # ------------------------------------------------------------------
+    # SystemUnderTest interface
+    # ------------------------------------------------------------------
+
+    @property
+    def registry(self) -> EndpointRegistry:
+        return self._registry
+
+    async def execute(self, test_case: TestCase) -> TestResult:
+        """Invoke the gRPC method described by *test_case* and return the result.
+
+        ``test_case.target`` is treated as the full gRPC method path
+        (e.g. ``"/helloworld.Greeter/SayHello"``).
+
+        ``test_case.inputs`` may contain:
+
+        * ``request`` — dict payload serialised to JSON-encoded bytes for the
+          request message.
+        * ``rpc_type`` — one of ``"unary"``, ``"server_streaming"``,
+          ``"client_streaming"``, ``"bidi_streaming"`` (default: ``"unary"``).
+        * ``requests`` — list of request dicts for streaming RPCs.
+        * ``expected_status_codes`` — list of acceptable gRPC status codes
+          (ints).  Defaults to ``[0]`` (OK).
+        """
+        if not _grpc_available():
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=False,
+                actual_output=None,
+                execution_time_ms=0.0,
+                error=(
+                    "grpcio package is not installed. "
+                    "Install it with: pip install grpcio"
+                ),
+            )
+
+        rpc_type: str = str(test_case.inputs.get("rpc_type", "unary")).lower()
+        method_path: str = test_case.target
+        request_data: dict = test_case.inputs.get("request") or {}
+        expected_codes = frozenset(
+            test_case.inputs.get("expected_status_codes") or [_OK]
+        )
+
+        if rpc_type == "unary":
+            return await self._execute_unary(
+                test_case, method_path, request_data, expected_codes
+            )
+        elif rpc_type == "server_streaming":
+            return await self._execute_server_streaming(
+                test_case, method_path, request_data, expected_codes
+            )
+        elif rpc_type == "client_streaming":
+            requests = test_case.inputs.get("requests") or [request_data]
+            return await self._execute_client_streaming(
+                test_case, method_path, requests, expected_codes
+            )
+        elif rpc_type == "bidi_streaming":
+            requests = test_case.inputs.get("requests") or [request_data]
+            return await self._execute_bidi_streaming(
+                test_case, method_path, requests, expected_codes
+            )
+        else:
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=False,
+                actual_output=None,
+                execution_time_ms=0.0,
+                error=f"Unknown rpc_type: {rpc_type!r}",
+            )
+
+    async def health_check(self) -> bool:
+        """Check that the gRPC server is reachable via a channel connectivity check."""
+        if not _grpc_available():
+            return False
+        try:
+            import grpc
+
+            if self.use_tls:
+                credentials = grpc.ssl_channel_credentials()
+                channel = grpc.secure_channel(self.endpoint, credentials)
+            else:
+                channel = grpc.insecure_channel(self.endpoint)
+
+            grpc.channel_ready_future(channel).result(timeout=min(self.timeout, 5.0))
+            channel.close()
+            return True
+        except Exception:  # noqa: BLE001
+            return False
+
+    # ------------------------------------------------------------------
+    # RPC-type-specific execution helpers
+    # ------------------------------------------------------------------
+
+    def _make_channel(self) -> Any:
+        """Create an insecure or TLS gRPC channel."""
+        import grpc
+
+        if self.use_tls:
+            credentials = grpc.ssl_channel_credentials()
+            return grpc.secure_channel(self.endpoint, credentials)
+        return grpc.insecure_channel(self.endpoint)
+
+    def _encode_request(self, request_data: dict) -> bytes:
+        """Encode *request_data* as raw bytes (JSON encoding for dynamic stub)."""
+        import json as _json
+        return _json.dumps(request_data).encode("utf-8")
+
+    def _parse_rpc_path(self, method_path: str) -> tuple[str, str]:
+        """Return (service_path, method_name) from a full method path."""
+        # e.g. "/helloworld.Greeter/SayHello" → ("/helloworld.Greeter", "SayHello")
+        path = method_path.lstrip("/")
+        if "/" in path:
+            svc, _, method = path.rpartition("/")
+            return svc, method
+        return path, ""
+
+    async def _execute_unary(
+        self,
+        test_case: TestCase,
+        method_path: str,
+        request_data: dict,
+        expected_codes: frozenset,
+    ) -> TestResult:
+        """Execute a unary RPC call."""
+        import grpc
+
+        start = time.monotonic()
+        try:
+            channel = self._make_channel()
+            request_bytes = self._encode_request(request_data)
+
+            svc_path, method_name = self._parse_rpc_path(method_path)
+            stub = channel.unary_unary(
+                method_path,
+                request_serializer=_passthrough,
+                response_deserializer=_passthrough,
+            )
+
+            response = stub(request_bytes, timeout=self.timeout, metadata=self.call_metadata)
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+
+            # Dynamic stub returns raw bytes on success
+            passed = _OK in expected_codes
+            channel.close()
+
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=passed,
+                actual_output={"raw_bytes": len(response) if response else 0},
+                execution_time_ms=elapsed_ms,
+                error=None if passed else f"Unexpected gRPC status; expected one of {expected_codes}",
+            )
+
+        except Exception as exc:  # noqa: BLE001
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            status_code = _extract_grpc_status_code(exc)
+            passed = status_code in expected_codes if status_code is not None else False
+
+            error_msg: Optional[str] = None
+            if not passed:
+                status_name = _STATUS_NAMES.get(status_code or 2, "UNKNOWN")
+                error_msg = f"gRPC {status_name} ({status_code}): {exc}"
+
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=passed,
+                actual_output=None,
+                execution_time_ms=elapsed_ms,
+                error=error_msg,
+            )
+
+    async def _execute_server_streaming(
+        self,
+        test_case: TestCase,
+        method_path: str,
+        request_data: dict,
+        expected_codes: frozenset,
+    ) -> TestResult:
+        """Execute a server-streaming RPC call, collecting up to N responses."""
+        start = time.monotonic()
+        try:
+            channel = self._make_channel()
+            request_bytes = self._encode_request(request_data)
+
+            stub = channel.unary_stream(
+                method_path,
+                request_serializer=_passthrough,
+                response_deserializer=_passthrough,
+            )
+
+            responses: list[Any] = []
+            for resp in stub(request_bytes, timeout=self.timeout, metadata=self.call_metadata):
+                responses.append(resp)
+                if len(responses) >= self.max_streaming_responses:
+                    break
+
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            passed = _OK in expected_codes
+            channel.close()
+
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=passed,
+                actual_output={"response_count": len(responses)},
+                execution_time_ms=elapsed_ms,
+                error=None,
+            )
+
+        except Exception as exc:  # noqa: BLE001
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            status_code = _extract_grpc_status_code(exc)
+            passed = status_code in expected_codes if status_code is not None else False
+            status_name = _STATUS_NAMES.get(status_code or 2, "UNKNOWN")
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=passed,
+                actual_output=None,
+                execution_time_ms=elapsed_ms,
+                error=f"gRPC {status_name}: {exc}" if not passed else None,
+            )
+
+    async def _execute_client_streaming(
+        self,
+        test_case: TestCase,
+        method_path: str,
+        requests: list[dict],
+        expected_codes: frozenset,
+    ) -> TestResult:
+        """Execute a client-streaming RPC call."""
+        start = time.monotonic()
+        try:
+            channel = self._make_channel()
+
+            def _request_iter() -> Iterator[bytes]:
+                for req in requests:
+                    yield self._encode_request(req)
+
+            stub = channel.stream_unary(
+                method_path,
+                request_serializer=_passthrough,
+                response_deserializer=_passthrough,
+            )
+
+            response = stub(_request_iter(), timeout=self.timeout, metadata=self.call_metadata)
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            passed = _OK in expected_codes
+            channel.close()
+
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=passed,
+                actual_output={"raw_bytes": len(response) if response else 0},
+                execution_time_ms=elapsed_ms,
+                error=None if passed else "Unexpected gRPC status",
+            )
+
+        except Exception as exc:  # noqa: BLE001
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            status_code = _extract_grpc_status_code(exc)
+            passed = status_code in expected_codes if status_code is not None else False
+            status_name = _STATUS_NAMES.get(status_code or 2, "UNKNOWN")
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=passed,
+                actual_output=None,
+                execution_time_ms=elapsed_ms,
+                error=f"gRPC {status_name}: {exc}" if not passed else None,
+            )
+
+    async def _execute_bidi_streaming(
+        self,
+        test_case: TestCase,
+        method_path: str,
+        requests: list[dict],
+        expected_codes: frozenset,
+    ) -> TestResult:
+        """Execute a bidirectional streaming RPC call."""
+        start = time.monotonic()
+        try:
+            channel = self._make_channel()
+
+            def _request_iter() -> Iterator[bytes]:
+                for req in requests:
+                    yield self._encode_request(req)
+
+            stub = channel.stream_stream(
+                method_path,
+                request_serializer=_passthrough,
+                response_deserializer=_passthrough,
+            )
+
+            responses: list[Any] = []
+            for resp in stub(_request_iter(), timeout=self.timeout, metadata=self.call_metadata):
+                responses.append(resp)
+                if len(responses) >= self.max_streaming_responses:
+                    break
+
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            passed = _OK in expected_codes
+            channel.close()
+
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=passed,
+                actual_output={"response_count": len(responses)},
+                execution_time_ms=elapsed_ms,
+                error=None,
+            )
+
+        except Exception as exc:  # noqa: BLE001
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            status_code = _extract_grpc_status_code(exc)
+            passed = status_code in expected_codes if status_code is not None else False
+            status_name = _STATUS_NAMES.get(status_code or 2, "UNKNOWN")
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=passed,
+                actual_output=None,
+                execution_time_ms=elapsed_ms,
+                error=f"gRPC {status_name}: {exc}" if not passed else None,
+            )
+
+
+def _extract_grpc_status_code(exc: Exception) -> Optional[int]:
+    """Extract a numeric gRPC status code from an RpcError exception."""
+    try:
+        # grpc.RpcError has a .code() method
+        code_obj = exc.code()  # type: ignore[attr-defined]
+        # grpc.StatusCode enum has .value[0] (the numeric code)
+        return int(code_obj.value[0])
+    except Exception:  # noqa: BLE001
+        return None

mannf/product/integrations/http_sut.py

@@ -0,0 +1,237 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""HTTP System-Under-Test implementation.
+
+:class:`HttpApiSUT` is a concrete :class:`~mannf.core.distributed.system_under_test.SystemUnderTest`
+that executes test cases against real HTTP APIs using ``httpx``.
+"""
+
+from __future__ import annotations
+
+import time
+import logging
+from typing import Any, Dict, List, Optional
+
+import httpx
+
+from mannf.core.distributed.endpoint import Endpoint, EndpointRegistry
+from mannf.core.distributed.system_under_test import SystemUnderTest
+from mannf.product.integrations.auth import AuthStrategy, NoAuth
+from mannf.core.testing.models import TestCase, TestResult
+
+# Imported lazily to avoid a hard circular dependency
+_RegressionRecorder = None
+
+
+def _get_recorder_type():
+    global _RegressionRecorder
+    if _RegressionRecorder is None:
+        from mannf.product.regression.recorder import RegressionRecorder  # noqa: PLC0415
+        _RegressionRecorder = RegressionRecorder
+    return _RegressionRecorder
+
+logger = logging.getLogger(__name__)
+
+# HTTP methods that send a request body
+_BODY_METHODS = {"POST", "PUT", "PATCH"}
+
+
+class HttpApiSUT(SystemUnderTest):
+    """Execute test cases against a real HTTP API.
+
+    Parameters
+    ----------
+    base_url:
+        Root URL of the API under test (e.g. ``"http://localhost:8080"``).
+    default_headers:
+        Headers added to every request (e.g. ``{"Accept": "application/json"}``).
+    timeout:
+        Default request timeout in seconds.
+    auth:
+        :class:`~mannf.product.integrations.auth.AuthStrategy` applied before each
+        request.  Defaults to :class:`~mannf.product.integrations.auth.NoAuth`.
+    expected_status_codes:
+        Iterable of HTTP status codes considered passing.  Defaults to all
+        2xx codes (200–299).
+    latency_sla_ms:
+        Maximum acceptable response time in milliseconds.  A response that
+        exceeds this limit is treated as a failure even if the status code is
+        otherwise acceptable.  ``None`` means no SLA check.
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        default_headers: Optional[Dict[str, str]] = None,
+        timeout: float = 10.0,
+        auth: Optional[AuthStrategy] = None,
+        expected_status_codes: Optional[List[int]] = None,
+        latency_sla_ms: Optional[float] = None,
+        recorder: Optional[Any] = None,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.default_headers: Dict[str, str] = default_headers or {}
+        self.timeout = timeout
+        self.auth: AuthStrategy = auth or NoAuth()
+        self.latency_sla_ms = latency_sla_ms
+        self.recorder = recorder
+
+        if expected_status_codes is not None:
+            self._expected_status_codes: Optional[frozenset] = frozenset(expected_status_codes)
+        else:
+            self._expected_status_codes = None  # 2xx check applied dynamically
+
+        self._registry = EndpointRegistry()
+        # Register a single "default" endpoint representing the base URL
+        from urllib.parse import urlparse
+        parsed = urlparse(self.base_url)
+        host = parsed.hostname or "localhost"
+        port = parsed.port or (443 if parsed.scheme == "https" else 80)
+        self._registry.register(
+            Endpoint(
+                name="default",
+                host=host,
+                port=port,
+                protocol=parsed.scheme or "http",
+            )
+        )
+
+    # ------------------------------------------------------------------
+    # SystemUnderTest interface
+    # ------------------------------------------------------------------
+
+    @property
+    def registry(self) -> EndpointRegistry:
+        return self._registry
+
+    async def execute(self, test_case: TestCase) -> TestResult:
+        """Make an HTTP request described by *test_case* and return the result.
+
+        ``test_case.target`` is treated as the endpoint path (e.g. ``"/users"``).
+        ``test_case.inputs`` may contain:
+
+        * ``method`` – HTTP verb (default ``"GET"``).
+        * ``headers`` – per-request headers merged on top of *default_headers*.
+        * ``body`` – raw request body bytes or string.
+        * ``json`` – JSON-serialisable dict sent as the request body.
+        * ``query_params`` – dict of URL query parameters.
+        * ``expected_status_codes`` – per-test override of acceptable status
+          codes (list of ints).
+        """
+        method: str = str(test_case.inputs.get("method", "GET")).upper()
+        path: str = test_case.target
+        if not path.startswith("/"):
+            path = "/" + path
+        url = self.base_url + path
+
+        # Merge headers
+        merged_headers: Dict[str, str] = {**self.default_headers}
+        extra_headers = test_case.inputs.get("headers") or {}
+        merged_headers.update(extra_headers)
+
+        request_kwargs: Dict[str, Any] = {
+            "method": method,
+            "url": url,
+            "headers": merged_headers,
+            "timeout": self.timeout,
+        }
+
+        query_params = test_case.inputs.get("query_params")
+        if query_params:
+            request_kwargs["params"] = query_params
+
+        if method in _BODY_METHODS:
+            if "json" in test_case.inputs:
+                request_kwargs["json"] = test_case.inputs["json"]
+            elif "body" in test_case.inputs:
+                request_kwargs["content"] = test_case.inputs["body"]
+
+        # Apply auth strategy
+        request_kwargs = await self.auth.apply(request_kwargs)
+
+        # Determine per-test expected status codes
+        per_test_codes = test_case.inputs.get("expected_status_codes")
+        if per_test_codes is not None:
+            expected_codes: Optional[frozenset] = frozenset(per_test_codes)
+        else:
+            expected_codes = self._expected_status_codes
+
+        # Execute the request
+        start = time.monotonic()
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.request(**request_kwargs)
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+
+            # Determine pass/fail
+            if expected_codes is not None:
+                status_ok = response.status_code in expected_codes
+            else:
+                status_ok = 200 <= response.status_code < 300
+
+            latency_ok = (
+                self.latency_sla_ms is None or elapsed_ms <= self.latency_sla_ms
+            )
+            passed = status_ok and latency_ok
+
+            error_msg: Optional[str] = None
+            if not status_ok:
+                error_msg = f"Unexpected status {response.status_code}"
+            elif not latency_ok:
+                error_msg = (
+                    f"Latency {elapsed_ms:.1f}ms exceeds SLA {self.latency_sla_ms}ms"
+                )
+
+            # Try to parse response body as JSON, fall back to text
+            try:
+                actual_output: Any = response.json()
+            except Exception:
+                actual_output = response.text
+
+            if self.recorder is not None:
+                self.recorder.record(
+                    test_case_id=test_case.id,
+                    method=method,
+                    url=url,
+                    request_headers=merged_headers,
+                    request_body=request_kwargs.get("json") or request_kwargs.get("content"),
+                    query_params=query_params,
+                    status_code=response.status_code,
+                    response_headers=dict(response.headers),
+                    response_body=actual_output,
+                    latency_ms=elapsed_ms,
+                    passed=passed,
+                    error=error_msg,
+                )
+
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=passed,
+                actual_output=actual_output,
+                execution_time_ms=elapsed_ms,
+                error=error_msg,
+            )
+
+        except httpx.TimeoutException as exc:
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            logger.warning("HttpApiSUT: timeout for %s %s — %s", method, url, exc)
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=False,
+                actual_output=None,
+                execution_time_ms=elapsed_ms,
+                error=f"Request timed out: {exc}",
+            )
+        except httpx.RequestError as exc:
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            logger.warning("HttpApiSUT: connection error for %s %s — %s", method, url, exc)
+            return TestResult(
+                test_case_id=test_case.id,
+                passed=False,
+                actual_output=None,
+                execution_time_ms=elapsed_ms,
+                error=f"Connection error: {exc}",
+            )