nat-engine 1__py3-none-any.whl

mannf/product/llm/base.py

@@ -0,0 +1,267 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Abstract base class for LLM providers."""
+
+from __future__ import annotations
+
+import json
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+from mannf.product.llm.prompts import EDGE_CASE_PROMPT, NL_TEST_AUTHORING_PROMPT, TEST_PLAN_PROMPT
+
+logger = logging.getLogger(__name__)
+
+
+class LLMProvider(ABC):
+    """Abstract base class for LLM providers.
+
+    All concrete implementations must be safe to use in an async context and
+    must never raise exceptions that would abort a scan — errors are logged and
+    an empty list is returned instead.
+    """
+
+    @property
+    @abstractmethod
+    def provider_name(self) -> str:
+        """Human-readable provider name (e.g. ``"openai"``)."""
+
+    @abstractmethod
+    def is_available(self) -> bool:
+        """Return ``True`` if an API key is configured and the provider can be used."""
+
+    @abstractmethod
+    async def generate(self, prompt: str, **kwargs: Any) -> str:
+        """Send *prompt* to the LLM and return the raw response text."""
+
+    async def generate_test_cases(
+        self,
+        endpoint_info: Dict[str, Any],
+        spec_context: Dict[str, Any],
+        n: int = 5,
+    ) -> List[Dict[str, Any]]:
+        """Generate *n* structured test cases for the given endpoint.
+
+        Parameters
+        ----------
+        endpoint_info:
+            Dict with keys ``method``, ``path``, ``description``,
+            ``parameters``, ``schema``.
+        spec_context:
+            Additional context from the spec (servers, info, etc.).
+        n:
+            Number of test cases to request.
+
+        Returns
+        -------
+        list[dict]
+            Each dict has at least ``inputs`` and ``expected_behavior`` keys.
+            Returns an empty list on any error.
+        """
+        if not self.is_available():
+            return []
+
+        prompt = EDGE_CASE_PROMPT.format(
+            n=n,
+            method=endpoint_info.get("method", "GET"),
+            path=endpoint_info.get("path", "/"),
+            description=endpoint_info.get("description", ""),
+            parameters=json.dumps(endpoint_info.get("parameters", []), indent=2),
+            schema=json.dumps(endpoint_info.get("schema", {}), indent=2),
+        )
+
+        try:
+            raw = await self.generate(prompt)
+            return self._parse_test_cases(raw)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("%s: generate_test_cases failed: %s", self.provider_name, exc)
+            return []
+
+    async def generate_test_plan(
+        self,
+        api_summary: Dict[str, Any],
+    ) -> Optional["TestPlan"]:
+        """Generate a structured test plan for an entire API specification.
+
+        Parameters
+        ----------
+        api_summary:
+            Structured dict produced by the ingestor pipeline containing
+            endpoint groups, parameter descriptions, auth requirements, and
+            schema information.
+
+        Returns
+        -------
+        TestPlan | None
+            A parsed :class:`~mannf.product.llm.test_plan_models.TestPlan`
+            model on success, or ``None`` on error / unavailable provider.
+        """
+        from mannf.product.llm.test_plan_models import EndpointGroup, Priority, TestPlan  # noqa: PLC0415
+
+        if not self.is_available():
+            return None
+
+        prompt = TEST_PLAN_PROMPT.format(
+            api_summary=json.dumps(api_summary, indent=2),
+        )
+
+        try:
+            raw = await self.generate(prompt)
+            plan_dict = self._parse_json_object(raw)
+            if not isinstance(plan_dict, dict):
+                logger.warning("%s: generate_test_plan returned non-dict JSON", self.provider_name)
+                return None
+
+            groups: List[EndpointGroup] = []
+            for g in plan_dict.get("endpoint_groups", []):
+                if not isinstance(g, dict):
+                    continue
+                try:
+                    groups.append(
+                        EndpointGroup(
+                            name=g.get("name", "Unnamed Group"),
+                            endpoints=g.get("endpoints", []),
+                            priority=Priority(g.get("priority", "medium")),
+                            risk_assessment=g.get("risk_assessment", ""),
+                            suggested_strategies=g.get("suggested_strategies", []),
+                            edge_cases=g.get("edge_cases", []),
+                        )
+                    )
+                except Exception as exc:  # noqa: BLE001
+                    logger.debug("%s: skipping malformed endpoint group: %s", self.provider_name, exc)
+
+            import uuid  # noqa: PLC0415
+            from datetime import datetime, timezone  # noqa: PLC0415
+            return TestPlan(
+                id=str(uuid.uuid4()),
+                created_at=datetime.now(timezone.utc).isoformat(),
+                spec_source=api_summary.get("spec_source", ""),
+                endpoint_groups=groups,
+                estimated_coverage=plan_dict.get("estimated_coverage"),
+            )
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("%s: generate_test_plan failed: %s", self.provider_name, exc)
+            return None
+
+    async def generate_from_natural_language(
+        self,
+        description: str,
+        context: Optional[Dict[str, Any]] = None,
+    ) -> List[Dict[str, Any]]:
+        """Translate a plain-English test description into executable scenarios.
+
+        Parameters
+        ----------
+        description:
+            Free-form natural-language description of what to test.
+        context:
+            Optional dict with keys ``base_url``, ``auth_type``, and
+            ``known_endpoints`` (list of ``"METHOD /path"`` strings).
+
+        Returns
+        -------
+        list[dict]
+            Each dict is an executable test scenario compatible with
+            :class:`~mannf.core.functional_orchestrator.FunctionalTestOrchestrator`
+            (browser type) or
+            :class:`~mannf.product.security.scanner.SecurityScanner` (api type).
+            Returns an empty list on any error or if the provider is unavailable.
+        """
+        if not self.is_available():
+            return []
+
+        ctx = context or {}
+        known_endpoints = ctx.get("known_endpoints") or []
+        if isinstance(known_endpoints, list):
+            known_endpoints_str = ", ".join(known_endpoints) if known_endpoints else "none"
+        else:
+            known_endpoints_str = str(known_endpoints)
+
+        prompt = NL_TEST_AUTHORING_PROMPT.format(
+            description=description,
+            base_url=ctx.get("base_url", ""),
+            auth_type=ctx.get("auth_type", "none"),
+            known_endpoints=known_endpoints_str,
+        )
+
+        try:
+            raw = await self.generate(prompt)
+            return self._parse_test_cases(raw)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "%s: generate_from_natural_language failed: %s",
+                self.provider_name,
+                exc,
+            )
+            return []
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _parse_test_cases(raw: str) -> List[Dict[str, Any]]:
+        """Parse LLM response text into a list of test case dicts.
+
+        Attempts JSON parsing first; falls back to extracting any JSON array
+        found within the text.  Returns an empty list if nothing parseable is
+        found.
+        """
+        if not raw or not raw.strip():
+            return []
+
+        # Try direct JSON parse
+        try:
+            parsed = json.loads(raw.strip())
+            if isinstance(parsed, list):
+                return [tc for tc in parsed if isinstance(tc, dict)]
+            return []
+        except json.JSONDecodeError:
+            pass
+
+        # Fallback: find the first [...] block in the text
+        start = raw.find("[")
+        end = raw.rfind("]")
+        if start != -1 and end != -1 and end > start:
+            try:
+                parsed = json.loads(raw[start : end + 1])
+                if isinstance(parsed, list):
+                    return [tc for tc in parsed if isinstance(tc, dict)]
+            except json.JSONDecodeError:
+                pass
+
+        logger.debug("LLMProvider: could not parse response as JSON array")
+        return []
+
+    @staticmethod
+    def _parse_json_object(raw: str) -> Any:
+        """Parse LLM response text into a JSON object (dict or list).
+
+        Attempts direct JSON parsing first; falls back to extracting the first
+        ``{…}`` block found within the text.  Returns ``None`` if nothing
+        parseable is found.
+        """
+        if not raw or not raw.strip():
+            return None
+
+        # Try direct parse
+        try:
+            return json.loads(raw.strip())
+        except json.JSONDecodeError:
+            pass
+
+        # Fallback: find the first {...} block in the text
+        start = raw.find("{")
+        end = raw.rfind("}")
+        if start != -1 and end != -1 and end > start:
+            try:
+                return json.loads(raw[start : end + 1])
+            except json.JSONDecodeError:
+                pass
+
+        logger.debug("LLMProvider: could not parse response as JSON object")
+        return None

mannf/product/llm/config.py

@@ -0,0 +1,48 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""LLM configuration dataclass."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+@dataclass
+class LLMConfig:
+    """Configuration for an LLM provider.
+
+    Parameters
+    ----------
+    provider:
+        Provider name: ``"openai"`` or ``"anthropic"``.
+    model:
+        Model identifier (e.g. ``"gpt-4o-mini"`` or ``"claude-3-haiku-20240307"``).
+    api_key:
+        API key.  When ``None``, the provider reads from the appropriate
+        environment variable (``OPENAI_API_KEY`` / ``ANTHROPIC_API_KEY``).
+    max_tokens:
+        Maximum number of tokens to generate per response.
+    temperature:
+        Sampling temperature (0–2 for OpenAI, 0–1 for Anthropic).
+    timeout:
+        HTTP request timeout in seconds.
+    max_retries:
+        Number of retry attempts on transient errors.
+    """
+
+    provider: str = "openai"
+    model: str = ""
+    api_key: Optional[str] = None
+    max_tokens: int = 1024
+    temperature: float = 0.7
+    timeout: float = 30.0
+    max_retries: int = 3
+
+    def __post_init__(self) -> None:
+        if not self.model:
+            defaults = {"openai": "gpt-4o-mini", "anthropic": "claude-3-haiku-20240307"}
+            self.model = defaults.get(self.provider, "gpt-4o-mini")

mannf/product/llm/factory.py

@@ -0,0 +1,42 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Factory function for creating LLM provider instances."""
+
+from __future__ import annotations
+
+from mannf.product.llm.base import LLMProvider
+from mannf.product.llm.config import LLMConfig
+
+
+def get_provider(config: LLMConfig) -> LLMProvider:
+    """Return an :class:`~mannf.product.llm.base.LLMProvider` instance for *config*.
+
+    Parameters
+    ----------
+    config:
+        Provider configuration.
+
+    Returns
+    -------
+    LLMProvider
+        Concrete provider instance.
+
+    Raises
+    ------
+    ValueError
+        If ``config.provider`` is not a recognised provider name.
+    """
+    name = (config.provider or "").lower()
+    if name == "openai":
+        from mannf.product.llm.openai_provider import OpenAIProvider
+        return OpenAIProvider(config)
+    if name == "anthropic":
+        from mannf.product.llm.anthropic_provider import AnthropicProvider
+        return AnthropicProvider(config)
+    raise ValueError(
+        f"Unknown LLM provider {config.provider!r}. "
+        "Supported providers: 'openai', 'anthropic'."
+    )

mannf/product/llm/openai_provider.py

@@ -0,0 +1,93 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""OpenAI LLM provider (httpx-based, no openai SDK dependency)."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+from typing import Any, Optional
+
+import httpx
+
+from mannf.product.llm.base import LLMProvider
+from mannf.product.llm.config import LLMConfig
+
+logger = logging.getLogger(__name__)
+
+_OPENAI_API_URL = "https://api.openai.com/v1/chat/completions"
+
+
+class OpenAIProvider(LLMProvider):
+    """LLM provider that calls the OpenAI Chat Completions API.
+
+    Uses ``httpx`` directly — no ``openai`` SDK dependency required.
+
+    Parameters
+    ----------
+    config:
+        :class:`~mannf.product.llm.config.LLMConfig` instance.  When ``None``, a
+        default config is created (reads ``OPENAI_API_KEY`` from env).
+    """
+
+    def __init__(self, config: Optional[LLMConfig] = None) -> None:
+        if config is None:
+            config = LLMConfig(provider="openai")
+        self._config = config
+        self._api_key: str = config.api_key or os.environ.get("OPENAI_API_KEY", "")
+
+    @property
+    def provider_name(self) -> str:
+        return "openai"
+
+    def is_available(self) -> bool:
+        return bool(self._api_key)
+
+    async def generate(self, prompt: str, **kwargs: Any) -> str:
+        """Send *prompt* to the OpenAI Chat Completions API and return the response text."""
+        if not self.is_available():
+            logger.warning("OpenAIProvider: no API key configured")
+            return ""
+
+        headers = {
+            "Authorization": f"Bearer {self._api_key}",
+            "Content-Type": "application/json",
+        }
+        payload = {
+            "model": self._config.model,
+            "messages": [{"role": "user", "content": prompt}],
+            "max_tokens": self._config.max_tokens,
+            "temperature": self._config.temperature,
+        }
+
+        last_exc: Exception = RuntimeError("No attempts made")
+        for attempt in range(self._config.max_retries):
+            try:
+                async with httpx.AsyncClient(timeout=self._config.timeout) as client:
+                    resp = await client.post(_OPENAI_API_URL, headers=headers, json=payload)
+                    resp.raise_for_status()
+                    data = resp.json()
+                    return data["choices"][0]["message"]["content"]
+            except (httpx.HTTPStatusError, httpx.RequestError, KeyError, json.JSONDecodeError) as exc:
+                last_exc = exc
+                if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code in (400, 401, 403):
+                    # Non-retryable
+                    logger.error("OpenAIProvider: non-retryable error %s", exc)
+                    return ""
+                wait = 2 ** attempt
+                logger.warning(
+                    "OpenAIProvider: attempt %d/%d failed (%s), retrying in %ds",
+                    attempt + 1,
+                    self._config.max_retries,
+                    exc,
+                    wait,
+                )
+                await asyncio.sleep(wait)
+
+        logger.error("OpenAIProvider: all retries exhausted: %s", last_exc)
+        return ""