nat-engine 1__py3-none-any.whl

mannf/product/llm/root_cause_service.py

@@ -0,0 +1,311 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Reusable LLM-powered root cause analysis service (Phase 10.3).
+
+Provides :class:`RootCauseSuggestion` (Pydantic model) and
+:class:`ScanRootCauseAnalyzer` — a standalone async service that can be called
+from any scan pipeline (API security scans, browser functional tests, or the
+autonomous loop) to generate LLM root cause explanations for failures.
+
+Feature gating
+--------------
+The analyzer honours the ``llm_root_cause`` quota from
+:mod:`mannf.product.billing.feature_gates`:
+
+- ``free``: disabled (0)
+- ``pro``: top 5 findings/scan
+- ``team`` / ``enterprise``: unlimited (-1)
+
+Callers should sort failures by severity **before** calling
+:meth:`ScanRootCauseAnalyzer.analyze_failures` and pass only the subset they
+are allowed to analyze.  The service itself does not enforce the quota — that
+is the responsibility of the caller (typically the server endpoint or
+orchestrator integration).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Extended prompt template — works with API/security/functional failures
+# ---------------------------------------------------------------------------
+
+_EXTENDED_ROOT_CAUSE_PROMPT = """\
+You are an expert software reliability engineer specialising in web application \
+test automation and root cause analysis.
+
+A failure has been detected.  Using the information below, identify the most \
+likely root cause and suggest concrete debugging steps.
+
+Failure details:
+  Type: {failure_type}
+  URL / Endpoint: {url}
+  Error / Description: {error}
+  Severity: {severity}
+  HTTP Status Code: {status_code}
+  HTTP Response Body (truncated): {response_body}
+  Security Finding ID: {check_id}
+  Security Finding Title: {title}
+  CWE: {cwe_id}
+  Evidence: {evidence}
+  Remediation hint: {remediation}
+  Additional context: {context}
+
+Based on the above, provide a structured JSON response with the following fields:
+  - "likely_cause": one-sentence description of the most probable root cause
+  - "suggested_fix": concise actionable suggestion for where to start debugging
+  - "confidence": float 0.0–1.0 indicating your confidence in this root cause \
+(use lower values when limited context is available)
+  - "files_to_check": list of source file paths or component names most likely \
+to contain the defect (empty list when unknown)
+
+Return ONLY the JSON object with no surrounding markdown or explanation."""
+
+
+# ---------------------------------------------------------------------------
+# Pydantic model
+# ---------------------------------------------------------------------------
+
+
+class RootCauseSuggestion(BaseModel):
+    """Structured LLM root cause suggestion for a single failure.
+
+    Compatible with any failure format: API scan findings, browser test
+    failures, or security findings.
+    """
+
+    failure_id: str = Field("", description="Identifier of the failure (task_id, finding id, etc.).")
+    failure_type: str = Field("unknown", description="'security', 'functional', or 'autonomous'.")
+    url: str = Field("", description="URL or endpoint where the failure was detected.")
+    error: str = Field("", description="Error message or failure description.")
+    severity: str = Field("", description="Severity level: 'critical', 'high', 'medium', 'low'.")
+    likely_cause: str = Field(..., description="One-sentence description of the most probable root cause.")
+    suggested_fix: str = Field(..., description="Concise actionable debugging suggestion.")
+    confidence: float = Field(..., ge=0.0, le=1.0, description="LLM confidence in this root cause (0–1).")
+    files_to_check: List[str] = Field(default_factory=list, description="Source files or components to inspect.")
+    quota_limited: bool = Field(False, description="True when this finding was not analyzed due to plan quota.")
+
+
+# ---------------------------------------------------------------------------
+# Analyzer service
+# ---------------------------------------------------------------------------
+
+
+class ScanRootCauseAnalyzer:
+    """Reusable async service that calls the LLM to explain scan failures.
+
+    Parameters
+    ----------
+    llm_provider:
+        An initialised :class:`~mannf.product.llm.base.LLMProvider` instance.
+        When ``None``, :meth:`analyze_failures` returns an empty list.
+    quota:
+        Maximum number of failures to analyze in one call.  ``-1`` means
+        unlimited.  ``0`` means disabled.  Corresponds to the
+        ``llm_root_cause`` feature gate value.
+    """
+
+    def __init__(
+        self,
+        llm_provider: Optional[Any] = None,
+        quota: int = -1,
+    ) -> None:
+        self._provider = llm_provider
+        self._quota = quota
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    async def analyze_failures(
+        self,
+        failures: List[Dict[str, Any]],
+        context: Optional[Dict[str, Any]] = None,
+    ) -> List[RootCauseSuggestion]:
+        """Analyze a list of failures and return root cause suggestions.
+
+        Parameters
+        ----------
+        failures:
+            List of failure dicts.  Each dict should contain as many of
+            the following keys as available:
+
+            - ``id`` / ``failure_id`` / ``task_id`` — unique identifier
+            - ``type`` / ``failure_type`` — ``"security"``, ``"functional"``, etc.
+            - ``url`` — URL or endpoint path
+            - ``error`` / ``description`` — error message
+            - ``severity`` — ``"critical"`` / ``"high"`` / ``"medium"`` / ``"low"``
+            - ``status_code`` — HTTP status code (for API failures)
+            - ``response_body`` — truncated HTTP response body
+            - ``check_id`` — security check identifier
+            - ``title`` — finding title
+            - ``cwe_id`` — CWE identifier
+            - ``evidence`` — security evidence string
+            - ``remediation`` — remediation hint
+        context:
+            Optional dict with additional context shared across all failures
+            (e.g., ``{"api_base_url": "...", "scan_type": "security"}``).
+
+        Returns
+        -------
+        list[RootCauseSuggestion]
+            One suggestion per analyzed failure.  Failures that hit the quota
+            wall are returned with ``quota_limited=True`` and placeholder text.
+            Returns an empty list when LLM is unavailable or quota is 0.
+        """
+        if self._provider is None or self._quota == 0:
+            return []
+
+        if not failures:
+            return []
+
+        ctx = context or {}
+        results: List[RootCauseSuggestion] = []
+
+        for idx, failure in enumerate(failures):
+            failure_id = (
+                failure.get("id")
+                or failure.get("failure_id")
+                or failure.get("task_id")
+                or str(idx)
+            )
+            severity = failure.get("severity", "")
+            url = failure.get("url", failure.get("endpoint", ""))
+            failure_type = failure.get("type", failure.get("failure_type", "unknown"))
+
+            # Quota enforcement: beyond the allowed N, return placeholder entries
+            if self._quota != -1 and idx >= self._quota:
+                results.append(
+                    RootCauseSuggestion(
+                        failure_id=str(failure_id),
+                        failure_type=failure_type,
+                        url=url,
+                        error=failure.get("error", failure.get("description", "")),
+                        severity=severity,
+                        likely_cause="Analysis skipped — plan quota reached.",
+                        suggested_fix="Upgrade your plan to analyze all findings.",
+                        confidence=0.0,
+                        files_to_check=[],
+                        quota_limited=True,
+                    )
+                )
+                continue
+
+            suggestion = await self._analyze_one(failure, ctx)
+            if suggestion is not None:
+                results.append(suggestion)
+
+        return results
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    async def _analyze_one(
+        self,
+        failure: Dict[str, Any],
+        context: Dict[str, Any],
+    ) -> Optional[RootCauseSuggestion]:
+        """Analyze a single failure dict; returns ``None`` on LLM error."""
+        failure_id = (
+            failure.get("id")
+            or failure.get("failure_id")
+            or failure.get("task_id")
+            or ""
+        )
+        failure_type = failure.get("type", failure.get("failure_type", "unknown"))
+        url = failure.get("url", failure.get("endpoint", ""))
+        error = failure.get("error", failure.get("description", ""))
+        severity = failure.get("severity", "")
+        status_code = str(failure.get("status_code", ""))
+        response_body = (failure.get("response_body") or "")[:500]
+        check_id = failure.get("check_id", "")
+        title = failure.get("title", "")
+        cwe_id = failure.get("cwe_id", "")
+        evidence = (failure.get("evidence") or "")[:500]
+        remediation = failure.get("remediation", "")
+
+        extra_ctx = dict(context)
+        extra_ctx.update(failure.get("context", {}))
+        context_str = json.dumps(extra_ctx) if extra_ctx else "{}"
+
+        prompt = _EXTENDED_ROOT_CAUSE_PROMPT.format(
+            failure_type=failure_type,
+            url=url,
+            error=error,
+            severity=severity,
+            status_code=status_code,
+            response_body=response_body,
+            check_id=check_id,
+            title=title,
+            cwe_id=cwe_id,
+            evidence=evidence,
+            remediation=remediation,
+            context=context_str,
+        )
+
+        try:
+            raw = await self._call_llm(prompt)
+            parsed = self._parse_llm_json(raw)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("ScanRootCauseAnalyzer: LLM call failed for %s — %s", failure_id, exc)
+            return None
+
+        return RootCauseSuggestion(
+            failure_id=str(failure_id),
+            failure_type=failure_type,
+            url=url,
+            error=error,
+            severity=severity,
+            likely_cause=str(parsed.get("likely_cause", "Unable to determine root cause")),
+            suggested_fix=str(parsed.get("suggested_fix", "Investigate the error manually")),
+            confidence=float(max(0.0, min(1.0, parsed.get("confidence", 0.0)))),
+            files_to_check=list(parsed.get("files_to_check", [])),
+            quota_limited=False,
+        )
+
+    async def _call_llm(self, prompt: str) -> str:
+        """Invoke the LLM provider's generate method."""
+        provider = self._provider
+        if hasattr(provider, "generate"):
+            return await provider.generate(prompt)
+        if hasattr(provider, "acomplete"):
+            return await provider.acomplete(prompt)
+        if hasattr(provider, "complete"):
+            result = provider.complete(prompt)
+            if hasattr(result, "__await__"):
+                return await result
+            return result
+        raise AttributeError("LLM provider has no 'generate', 'acomplete', or 'complete' method")
+
+    @staticmethod
+    def _parse_llm_json(raw: str) -> Dict[str, Any]:
+        """Extract the JSON object from the LLM response string."""
+        text = raw.strip()
+        if text.startswith("```"):
+            lines = text.splitlines()
+            inner = [ln for ln in lines[1:] if ln.strip() != "```"]
+            text = "\n".join(inner).strip()
+        try:
+            return json.loads(text)
+        except json.JSONDecodeError:
+            pass
+        # Fallback: find the first {...} block
+        start = text.find("{")
+        end = text.rfind("}")
+        if start != -1 and end != -1 and end > start:
+            try:
+                return json.loads(text[start : end + 1])
+            except json.JSONDecodeError:
+                pass
+        return {}

mannf/product/llm/test_plan_models.py

@@ -0,0 +1,78 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Pydantic models for the LLM-generated test plan feature (Phase 10.1)."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from enum import Enum
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class PlanStatus(str, Enum):
+    draft = "draft"
+    approved = "approved"
+    executed = "executed"
+
+
+class Priority(str, Enum):
+    critical = "critical"
+    high = "high"
+    medium = "medium"
+    low = "low"
+
+
+class EndpointGroup(BaseModel):
+    """A logical group of related API endpoints within a test plan."""
+
+    name: str = Field(..., description="Human-readable group name (e.g. 'Authentication', 'User CRUD').")
+    endpoints: List[str] = Field(default_factory=list, description="List of 'METHOD /path' strings in this group.")
+    priority: Priority = Field(Priority.medium, description="Risk-based priority for this group.")
+    risk_assessment: str = Field("", description="Brief description of the risk profile for this group.")
+    suggested_strategies: List[str] = Field(
+        default_factory=list,
+        description="Recommended test strategies (e.g. 'boundary value', 'auth bypass').",
+    )
+    edge_cases: List[str] = Field(
+        default_factory=list,
+        description="Notable edge cases or business logic scenarios to cover.",
+    )
+
+
+class PlanApproval(BaseModel):
+    """Records user review/approval of a test plan."""
+
+    approved_by: str = Field("", description="Username or identifier of the approver.")
+    approved_at: Optional[str] = Field(None, description="ISO-8601 timestamp of approval.")
+    excluded_groups: List[str] = Field(
+        default_factory=list,
+        description="Names of endpoint groups that were excluded from execution.",
+    )
+
+
+class TestPlan(BaseModel):
+    """Top-level container for an LLM-generated API test plan."""
+
+    id: str = Field(..., description="Unique plan identifier (UUID).")
+    status: PlanStatus = Field(PlanStatus.draft, description="Lifecycle state of the plan.")
+    created_at: str = Field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat(),
+        description="ISO-8601 creation timestamp.",
+    )
+    spec_source: str = Field("", description="Original spec file name or URL.")
+    endpoint_groups: List[EndpointGroup] = Field(
+        default_factory=list,
+        description="Priority-ranked groups of related endpoints.",
+    )
+    estimated_coverage: Optional[float] = Field(
+        None,
+        ge=0.0,
+        le=1.0,
+        description="Estimated test coverage as a fraction (0–1), if the LLM provided one.",
+    )
+    approval: Optional[PlanApproval] = Field(None, description="Approval record, set when the plan is approved.")

mannf/product/metrics.py

@@ -0,0 +1,149 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Custom OpenTelemetry metrics for NAT scan lifecycle events.
+
+Public API
+----------
+- :func:`record_scan_start` — call when a scan begins.
+- :func:`record_scan_end`   — call when a scan finishes (any terminal status).
+
+All functions are safe no-ops when ``opentelemetry-api`` is not installed or
+when no metrics exporter is configured.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# No-op stubs used when OTel is unavailable
+# ---------------------------------------------------------------------------
+
+
+class _NoOpCounter:
+    def add(self, amount: float, attributes: Optional[dict] = None) -> None:
+        pass
+
+
+class _NoOpHistogram:
+    def record(self, amount: float, attributes: Optional[dict] = None) -> None:
+        pass
+
+
+class _NoOpUpDownCounter:
+    def add(self, amount: float, attributes: Optional[dict] = None) -> None:
+        pass
+
+
+# ---------------------------------------------------------------------------
+# Meter + instruments (initialised lazily)
+# ---------------------------------------------------------------------------
+
+scan_total: Any = _NoOpCounter()
+scan_duration_seconds: Any = _NoOpHistogram()
+scan_status_total: Any = _NoOpCounter()
+active_scans: Any = _NoOpUpDownCounter()
+
+_initialized: bool = False
+
+
+def _init_instruments() -> None:
+    """Create real OTel instruments if the SDK is available."""
+    global scan_total, scan_duration_seconds, scan_status_total, active_scans, _initialized
+    if _initialized:
+        return
+    _initialized = True
+
+    try:
+        from opentelemetry import metrics as _otel_metrics  # type: ignore[import]
+
+        meter = _otel_metrics.get_meter("mannf.product", version="1.0")
+
+        scan_total = meter.create_counter(
+            name="mannf.scan_total",
+            description="Total number of scans started",
+            unit="1",
+        )
+        scan_duration_seconds = meter.create_histogram(
+            name="mannf.scan_duration_seconds",
+            description="Wall-clock scan duration in seconds",
+            unit="s",
+        )
+        scan_status_total = meter.create_counter(
+            name="mannf.scan_status_total",
+            description="Total scans completed, by terminal status",
+            unit="1",
+        )
+        active_scans = meter.create_up_down_counter(
+            name="mannf.active_scans",
+            description="Number of scans currently running",
+            unit="1",
+        )
+    except ImportError:
+        # opentelemetry-api not installed — no-op stubs remain
+        pass
+    except Exception:  # noqa: BLE001
+        import traceback
+        logger.warning(
+            "Failed to initialise OpenTelemetry metric instruments:\n%s",
+            traceback.format_exc(),
+        )
+
+
+# Initialise eagerly so tests can inspect the state after import.
+_init_instruments()
+
+
+# ---------------------------------------------------------------------------
+# Public helper functions
+# ---------------------------------------------------------------------------
+
+
+def record_scan_start(scan_type: str, tenant_id: Optional[str] = None) -> None:
+    """Increment scan_total and active_scans counters when a scan begins.
+
+    Args:
+        scan_type: ``"functional"`` or ``"security"``.
+        tenant_id: Optional tenant identifier for multi-tenant deployments.
+    """
+    try:
+        attrs: dict[str, str] = {"scan_type": scan_type}
+        if tenant_id:
+            attrs["tenant_id"] = str(tenant_id)
+        scan_total.add(1, attrs)
+        active_scans.add(1, attrs)
+    except Exception:  # noqa: BLE001
+        pass
+
+
+def record_scan_end(
+    scan_type: str,
+    tenant_id: Optional[str],
+    status: str,
+    duration_s: float,
+) -> None:
+    """Record scan completion metrics.
+
+    Args:
+        scan_type: ``"functional"`` or ``"security"``.
+        tenant_id: Optional tenant identifier.
+        status: Terminal status — one of ``"completed"``, ``"failed"``,
+            ``"cancelled"``, ``"timed_out"``.
+        duration_s: Wall-clock scan duration in seconds.
+    """
+    try:
+        base_attrs: dict[str, str] = {"scan_type": scan_type}
+        if tenant_id:
+            base_attrs["tenant_id"] = str(tenant_id)
+
+        scan_status_total.add(1, {**base_attrs, "status": status})
+        scan_duration_seconds.record(duration_s, base_attrs)
+        active_scans.add(-1, base_attrs)
+    except Exception:  # noqa: BLE001
+        pass

mannf/product/middleware/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.

mannf/product/middleware/audit_middleware.py

@@ -0,0 +1,112 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+
+"""API audit logging middleware.
+
+Records every mutating HTTP call (POST, PUT, PATCH, DELETE) to the
+``api_audit_log`` table.  Never blocks the request — failures are logged
+and silently swallowed.
+
+Usage::
+
+    from mannf.product.middleware.audit_middleware import attach_audit_middleware
+    attach_audit_middleware(app)
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+
+from fastapi import FastAPI
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import Response
+
+logger = logging.getLogger(__name__)
+
+_MUTATING_METHODS = {"POST", "PUT", "PATCH", "DELETE"}
+
+
+class AuditMiddleware(BaseHTTPMiddleware):
+    """Starlette middleware that logs mutating API calls to api_audit_log."""
+
+    async def dispatch(self, request: Request, call_next) -> Response:
+        response = await call_next(request)
+
+        if request.method.upper() not in _MUTATING_METHODS:
+            return response
+
+        # Only log API paths
+        path = request.url.path
+        if not path.startswith("/api/"):
+            return response
+
+        try:
+            await _log_api_call(request, response.status_code)
+        except Exception as exc:  # noqa: BLE001
+            logger.debug("AuditMiddleware: failed to log call to %s: %s", path, exc)
+
+        return response
+
+
+async def _log_api_call(request: Request, status_code: int) -> None:
+    """Write an ApiAuditLog record for a request."""
+    try:
+        from mannf.product.database import _async_session_factory, _build_engine  # noqa: PLC0415
+        from mannf.product.models import ApiAuditLog  # noqa: PLC0415
+    except ImportError:
+        return
+
+    if _build_engine() is None or _async_session_factory is None:
+        return
+
+    tenant = getattr(request.state, "tenant", None)
+    user = getattr(request.state, "user", None)
+
+    tenant_id: uuid.UUID | None = None
+    if tenant:
+        try:
+            raw_tid = tenant.get("tenant_id") or tenant.get("id")
+            if raw_tid:
+                tenant_id = uuid.UUID(str(raw_tid))
+        except (ValueError, AttributeError):
+            pass
+
+    user_id: uuid.UUID | None = None
+    if user and user.get("id"):
+        try:
+            user_id = uuid.UUID(str(user["id"]))
+        except ValueError:
+            pass
+
+    # Best-effort request summary (method + path + query)
+    qs = str(request.query_params) if request.query_params else ""
+    request_summary = f"{request.method} {request.url.path}"
+    if qs:
+        request_summary += f"?{qs}"
+
+    source_ip = request.headers.get("x-forwarded-for") or (request.client.host if request.client else None)
+
+    try:
+        async with _async_session_factory() as session:
+            entry = ApiAuditLog(
+                id=uuid.uuid4(),
+                tenant_id=tenant_id,
+                user_id=user_id,
+                method=request.method[:10],
+                path=request.url.path[:2048],
+                status_code=status_code,
+                request_summary=request_summary[:4096] if request_summary else None,
+                source_ip=source_ip[:64] if source_ip else None,
+            )
+            session.add(entry)
+            await session.commit()
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("AuditMiddleware: DB write failed: %s", exc)
+
+
+def attach_audit_middleware(app: FastAPI) -> None:
+    """Attach the AuditMiddleware to a FastAPI application."""
+    app.add_middleware(AuditMiddleware)