nat-engine 1__py3-none-any.whl

mannf/product/ingestors/openapi_ingestor.py

@@ -0,0 +1,606 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""OpenAPI/Swagger ingestor plugin.
+
+Parses OpenAPI 3.x and Swagger 2.0 specifications (YAML or JSON) and
+produces normalised :class:`~mannf.product.ingestors.models.IngestedEndpoint`
+and :class:`~mannf.product.ingestors.models.IngestedTestCase` objects.
+
+Parsing helpers (``_load_spec``, ``_resolve_ref``, ``_schema_example``,
+``_detect_version``, ``_base_url_from_spec``) are imported directly from the
+existing :mod:`mannf.product.integrations.openapi_parser` module so that spec
+traversal logic is not duplicated.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from mannf.product.ingestors.base import IngestorPlugin
+from mannf.product.ingestors.models import (
+    IngestedEndpoint,
+    IngestedTestCase,
+    IngestorSource,
+    IngestResult,
+)
+from mannf.product.integrations.openapi_parser import (
+    _base_url_from_spec,
+    _detect_version,
+    _resolve_ref,
+    _schema_example,
+)
+
+logger = logging.getLogger(__name__)
+
+# HTTP methods that OpenAPI specs enumerate at path-item level
+_HTTP_METHODS = ("get", "post", "put", "patch", "delete", "head", "options", "trace")
+
+# Priority mapping by HTTP method
+_METHOD_PRIORITY: dict[str, str] = {
+    "POST": "high",
+    "PUT": "high",
+    "DELETE": "high",
+    "PATCH": "high",
+    "GET": "medium",
+    "OPTIONS": "low",
+    "HEAD": "low",
+    "TRACE": "low",
+}
+
+# Maximum $ref resolution depth to prevent infinite loops
+_MAX_REF_DEPTH = 10
+
+
+def _resolve_deep(
+    spec: dict[str, Any],
+    obj: Any,
+    depth: int = 0,
+) -> Any:
+    """Recursively resolve all ``$ref`` pointers in *obj* up to *_MAX_REF_DEPTH*.
+
+    Parameters
+    ----------
+    spec:
+        The root spec document (used for ``$ref`` resolution).
+    obj:
+        The object to resolve (dict, list, or scalar).
+    depth:
+        Current recursion depth (guards against cycles).
+
+    Returns
+    -------
+    Any
+        The resolved object.
+    """
+    if depth > _MAX_REF_DEPTH:
+        return obj
+
+    if isinstance(obj, dict):
+        if "$ref" in obj:
+            resolved = _resolve_ref(spec, obj["$ref"])
+            return _resolve_deep(spec, resolved, depth + 1)
+        return {k: _resolve_deep(spec, v, depth + 1) for k, v in obj.items()}
+
+    if isinstance(obj, list):
+        return [_resolve_deep(spec, item, depth + 1) for item in obj]
+
+    return obj
+
+
+def _parse_content(content: str, path: str | None) -> dict[str, Any]:
+    """Parse *content* as YAML or JSON, inferring format from *path* extension.
+
+    Parameters
+    ----------
+    content:
+        Raw spec text.
+    path:
+        Optional file path used to detect whether content is YAML or JSON.
+
+    Returns
+    -------
+    dict[str, Any]
+        Parsed spec document.
+
+    Raises
+    ------
+    ValueError
+        If content cannot be parsed as either YAML or JSON.
+    """
+    ext = Path(path).suffix.lower() if path else ""
+    if ext in {".yaml", ".yml"}:
+        result = yaml.safe_load(content)
+    elif ext == ".json":
+        result = json.loads(content)
+    else:
+        # Try YAML first (superset of JSON), fall back to JSON
+        try:
+            result = yaml.safe_load(content)
+        except yaml.YAMLError:
+            result = json.loads(content)
+
+    if not isinstance(result, dict):
+        raise ValueError("Parsed spec is not a mapping/object.")
+    return result
+
+
+def _extract_auth_requirements(
+    spec: dict[str, Any],
+    operation: dict[str, Any],
+) -> list[str]:
+    """Return a list of security scheme names required for *operation*.
+
+    Falls back to the global ``security`` requirement when the operation does
+    not define its own.
+    """
+    security_list: list[dict[str, Any]] | None = operation.get("security")
+    if security_list is None:
+        security_list = spec.get("security")  # type: ignore[assignment]
+
+    if not security_list:
+        return []
+
+    schemes: list[str] = []
+    for entry in security_list:
+        if isinstance(entry, dict):
+            schemes.extend(entry.keys())
+    return schemes
+
+
+def _merge_parameters(
+    spec: dict[str, Any],
+    path_level: list[dict[str, Any]],
+    operation_level: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    """Merge path-level and operation-level parameters.
+
+    Operation-level parameters override path-level parameters with the same
+    ``name`` + ``in`` pair.  All ``$ref`` pointers are resolved.
+
+    Parameters
+    ----------
+    spec:
+        Root spec document.
+    path_level:
+        Parameters defined at the path-item level.
+    operation_level:
+        Parameters defined at the operation level.
+
+    Returns
+    -------
+    list[dict[str, Any]]
+        Merged, resolved parameter list.
+    """
+    def _key(param: dict[str, Any]) -> tuple[str, str]:
+        return (param.get("name", ""), param.get("in", ""))
+
+    resolved_path = [
+        _resolve_deep(spec, p) for p in path_level if isinstance(p, dict)
+    ]
+    resolved_op = [
+        _resolve_deep(spec, p) for p in operation_level if isinstance(p, dict)
+    ]
+
+    merged: dict[tuple[str, str], dict[str, Any]] = {}
+    for param in resolved_path:
+        merged[_key(param)] = param
+    for param in resolved_op:
+        merged[_key(param)] = param  # operation-level wins
+
+    return list(merged.values())
+
+
+def _extract_request_body(
+    spec: dict[str, Any],
+    operation: dict[str, Any],
+    version: str,
+) -> dict[str, Any] | None:
+    """Return the resolved request body schema (OAS3) or body parameter (Swagger 2).
+
+    Parameters
+    ----------
+    spec:
+        Root spec document.
+    operation:
+        The operation object.
+    version:
+        ``"openapi3"`` or ``"swagger2"``.
+
+    Returns
+    -------
+    dict[str, Any] | None
+        The resolved request body dict, or ``None`` if absent.
+    """
+    if version == "openapi3":
+        request_body = operation.get("requestBody")
+        if request_body:
+            return _resolve_deep(spec, request_body)
+        return None
+
+    # Swagger 2 — body parameter
+    for param in operation.get("parameters", []):
+        resolved = _resolve_deep(spec, param) if isinstance(param, dict) else param
+        if isinstance(resolved, dict) and resolved.get("in") == "body":
+            return resolved
+    return None
+
+
+def _extract_responses(
+    spec: dict[str, Any],
+    operation: dict[str, Any],
+) -> dict[str, Any]:
+    """Return a resolved responses dict keyed by HTTP status code."""
+    raw = operation.get("responses", {})
+    return {str(code): _resolve_deep(spec, resp) for code, resp in raw.items()}
+
+
+def _generate_test_cases(
+    endpoint: IngestedEndpoint,
+    spec: dict[str, Any],
+) -> list[IngestedTestCase]:
+    """Generate synthetic test cases for *endpoint*.
+
+    Produces:
+    - A **happy-path** test case (valid request, expect 2xx).
+    - A **missing required params** test case (if required params exist).
+    - A **wrong type** test case (if typed params exist).
+
+    Parameters
+    ----------
+    endpoint:
+        The endpoint to generate test cases for.
+    spec:
+        Root spec document (used to resolve schemas).
+
+    Returns
+    -------
+    list[IngestedTestCase]
+        Generated test cases.
+    """
+    cases: list[IngestedTestCase] = []
+    method = endpoint.method
+    path = endpoint.path
+    priority = _METHOD_PRIORITY.get(method, "medium")
+    source_ref = f"{method} {path}"
+
+    # --- Happy-path test case ---
+    cases.append(
+        IngestedTestCase(
+            name=f"[happy path] {method} {path}",
+            description=f"Send a valid {method} request to {path} and expect a 2xx response.",
+            steps=[
+                {
+                    "action": f"{method} {path}",
+                    "description": "Send valid request",
+                }
+            ],
+            expected_result="2xx response",
+            priority=priority,
+            tags=["happy-path", "functional"],
+            source_ref=source_ref,
+            metadata={
+                "intent": "functional",
+                "test_type": "positive",
+                "operationId": endpoint.metadata.get("operationId", ""),
+            },
+        )
+    )
+
+    # Collect required parameters
+    required_params = [
+        p for p in endpoint.parameters
+        if isinstance(p, dict) and p.get("required")
+    ]
+
+    # --- Missing required params ---
+    if required_params:
+        first_required = required_params[0]
+        param_name = first_required.get("name", "param")
+        cases.append(
+            IngestedTestCase(
+                name=f"[missing required param] {method} {path} — omit '{param_name}'",
+                description=(
+                    f"Send a {method} request to {path} without required "
+                    f"parameter '{param_name}' and expect a 4xx response."
+                ),
+                steps=[
+                    {
+                        "action": f"{method} {path}",
+                        "description": f"Omit required parameter '{param_name}'",
+                    }
+                ],
+                expected_result="4xx response (400 or 422)",
+                priority=priority,
+                tags=["negative", "functional"],
+                source_ref=source_ref,
+                metadata={
+                    "intent": "functional",
+                    "test_type": "negative",
+                    "neg_type": "missing_required_param",
+                    "missing_param": param_name,
+                },
+            )
+        )
+
+    # --- Wrong type ---
+    typed_params = [
+        p for p in endpoint.parameters
+        if isinstance(p, dict) and p.get("schema", {}).get("type")
+    ]
+    if not typed_params and endpoint.request_body:
+        # Check request body for typed schema properties
+        content = endpoint.request_body.get("content", {})
+        for media_schema in content.values():
+            schema = media_schema.get("schema", {}) if isinstance(media_schema, dict) else {}
+            if schema.get("properties"):
+                typed_params = [{"_from_body": True, "schema": schema}]
+                break
+
+    if typed_params:
+        param = typed_params[0]
+        param_name = param.get("name", "body field")
+        cases.append(
+            IngestedTestCase(
+                name=f"[wrong type] {method} {path} — wrong type for '{param_name}'",
+                description=(
+                    f"Send a {method} request to {path} with wrong type for "
+                    f"'{param_name}' and expect a 4xx response."
+                ),
+                steps=[
+                    {
+                        "action": f"{method} {path}",
+                        "description": f"Use wrong type for '{param_name}'",
+                    }
+                ],
+                expected_result="4xx response (400 or 422)",
+                priority=priority,
+                tags=["negative", "functional"],
+                source_ref=source_ref,
+                metadata={
+                    "intent": "functional",
+                    "test_type": "negative",
+                    "neg_type": "wrong_type",
+                    "param": param_name,
+                },
+            )
+        )
+
+    # --- Security test case (if auth is required) ---
+    if endpoint.auth_requirements:
+        cases.append(
+            IngestedTestCase(
+                name=f"[security] {method} {path} — missing auth",
+                description=(
+                    f"Send a {method} request to {path} without authentication "
+                    f"and expect a 401 or 403 response."
+                ),
+                steps=[
+                    {
+                        "action": f"{method} {path}",
+                        "description": "Send request without auth credentials",
+                    }
+                ],
+                expected_result="401 or 403 response",
+                priority="high",
+                tags=["security", "auth"],
+                source_ref=source_ref,
+                metadata={
+                    "intent": "security",
+                    "test_type": "negative",
+                    "neg_type": "missing_auth",
+                    "auth_schemes": endpoint.auth_requirements,
+                },
+            )
+        )
+
+    return cases
+
+
+class OpenAPIIngestor(IngestorPlugin):
+    """Ingest OpenAPI 3.x and Swagger 2.0 specifications (YAML/JSON).
+
+    Produces normalised :class:`~mannf.product.ingestors.models.IngestedEndpoint`
+    and :class:`~mannf.product.ingestors.models.IngestedTestCase` objects from
+    OpenAPI 3.x (``openapi`` key) and Swagger 2.0 (``swagger`` key) specs in
+    either YAML or JSON format.
+
+    Configuration keys
+    ------------------
+    base_url : str, optional
+        Override the server URL extracted from the spec.
+    include_deprecated : bool, optional
+        When ``True``, deprecated operations are included in the output.
+        Defaults to ``False``.
+    """
+
+    name = "openapi"
+    display_name = "OpenAPI/Swagger"
+    description = "Ingest OpenAPI 3.x and Swagger 2.0 specifications (YAML/JSON)."
+    supported_extensions = [".yaml", ".yml", ".json"]
+    supported_formats = ["openapi", "swagger"]
+
+    # ------------------------------------------------------------------
+    # validate_config
+    # ------------------------------------------------------------------
+
+    def validate_config(self, config: dict) -> list[str]:
+        """Validate the plugin configuration.
+
+        Parameters
+        ----------
+        config:
+            Supported keys: ``base_url`` (str, optional),
+            ``include_deprecated`` (bool, optional).
+
+        Returns
+        -------
+        list[str]
+            Empty list — no keys are required.
+        """
+        errors: list[str] = []
+        if "base_url" in config and not isinstance(config["base_url"], str):
+            errors.append("'base_url' must be a string.")
+        if "include_deprecated" in config and not isinstance(
+            config["include_deprecated"], bool
+        ):
+            errors.append("'include_deprecated' must be a boolean.")
+        return errors
+
+    # ------------------------------------------------------------------
+    # can_handle
+    # ------------------------------------------------------------------
+
+    def can_handle(self, source: IngestorSource) -> bool:
+        """Return ``True`` for openapi/swagger formats and YAML/JSON extensions.
+
+        Parameters
+        ----------
+        source:
+            The source to evaluate.
+
+        Returns
+        -------
+        bool
+            ``True`` if this plugin can ingest the source.
+        """
+        if source.format and source.format in self.supported_formats:
+            return True
+        if not source.format and source.path is not None:
+            ext = Path(source.path).suffix.lower()
+            return ext in self.supported_extensions
+        return False
+
+    # ------------------------------------------------------------------
+    # ingest
+    # ------------------------------------------------------------------
+
+    async def ingest(self, source: IngestorSource, config: dict) -> IngestResult:
+        """Parse an OpenAPI/Swagger spec and return a normalised :class:`IngestResult`.
+
+        Parameters
+        ----------
+        source:
+            The source to ingest.  Must provide either ``raw_content`` or a
+            valid ``path``.
+        config:
+            Plugin-specific configuration.  See :meth:`validate_config` for
+            supported keys.
+
+        Returns
+        -------
+        IngestResult
+            Populated result on success; ``success=False`` with ``errors`` on
+            failure.
+        """
+        # --- Read content ---
+        try:
+            content = self._read_source(source)
+        except (FileNotFoundError, ValueError) as exc:
+            return self._make_error_result(source, str(exc))
+
+        # --- Parse YAML/JSON ---
+        try:
+            spec: dict[str, Any] = _parse_content(content, source.path)
+        except Exception as exc:  # noqa: BLE001
+            return self._make_error_result(
+                source, f"Failed to parse spec: {exc}"
+            )
+
+        # --- Detect version ---
+        version = _detect_version(spec)
+
+        # --- Extract base URL ---
+        spec_base_url: str = _base_url_from_spec(spec) or ""
+        base_url: str = config.get("base_url", spec_base_url) or spec_base_url
+
+        include_deprecated: bool = bool(config.get("include_deprecated", False))
+
+        # --- Walk paths ---
+        endpoints: list[IngestedEndpoint] = []
+        test_cases: list[IngestedTestCase] = []
+        paths: dict[str, Any] = spec.get("paths") or {}
+
+        for url_path, path_item in paths.items():
+            if not isinstance(path_item, dict):
+                continue
+
+            path_level_params: list[dict[str, Any]] = [
+                p for p in path_item.get("parameters", []) if isinstance(p, dict)
+            ]
+
+            for method_lower in _HTTP_METHODS:
+                operation: dict[str, Any] | None = path_item.get(method_lower)
+                if not isinstance(operation, dict):
+                    continue
+
+                # Skip deprecated unless explicitly included
+                if operation.get("deprecated") and not include_deprecated:
+                    continue
+
+                method = method_lower.upper()
+
+                # Merge parameters (path-level + operation-level), resolve $ref
+                merged_params = _merge_parameters(
+                    spec,
+                    path_level_params,
+                    operation.get("parameters", []),
+                )
+
+                # Request body
+                request_body = _extract_request_body(spec, operation, version)
+
+                # Responses
+                responses = _extract_responses(spec, operation)
+
+                # Auth requirements
+                auth_requirements = _extract_auth_requirements(spec, operation)
+
+                # Tags
+                tags: list[str] = operation.get("tags") or []
+
+                # Metadata
+                metadata: dict[str, Any] = {
+                    "operationId": operation.get("operationId") or "",
+                    "deprecated": bool(operation.get("deprecated", False)),
+                    "base_url": base_url,
+                    "spec_version": version,
+                }
+
+                endpoint = IngestedEndpoint(
+                    method=method,
+                    path=url_path,
+                    summary=operation.get("summary") or "",
+                    description=operation.get("description") or "",
+                    parameters=merged_params,
+                    request_body=request_body,
+                    responses=responses,
+                    auth_requirements=auth_requirements,
+                    tags=list(tags),
+                    metadata=metadata,
+                )
+                endpoints.append(endpoint)
+                test_cases.extend(_generate_test_cases(endpoint, spec))
+
+        stats: dict[str, Any] = {
+            "endpoints_count": len(endpoints),
+            "test_cases_count": len(test_cases),
+            "spec_version": version,
+            "base_url": base_url,
+        }
+
+        return IngestResult(
+            success=True,
+            source=source,
+            endpoints=endpoints,
+            test_cases=test_cases,
+            errors=[],
+            stats=stats,
+        )