nat-engine 1__py3-none-any.whl

mannf/product/regression/models.py

@@ -0,0 +1,232 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Data models for the regression record/replay system."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class RequestRecord:
+    """Full details of a captured HTTP request."""
+
+    method: str
+    url: str
+    headers: Dict[str, str] = field(default_factory=dict)
+    body: Any = None
+    query_params: Optional[Dict[str, Any]] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "method": self.method,
+            "url": self.url,
+            "headers": self.headers,
+            "body": self.body,
+            "query_params": self.query_params,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "RequestRecord":
+        return cls(
+            method=data["method"],
+            url=data["url"],
+            headers=data.get("headers") or {},
+            body=data.get("body"),
+            query_params=data.get("query_params"),
+        )
+
+
+@dataclass
+class ResponseRecord:
+    """Full details of a captured HTTP response."""
+
+    status_code: int
+    headers: Dict[str, str] = field(default_factory=dict)
+    body: Any = None
+    latency_ms: float = 0.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "status_code": self.status_code,
+            "headers": self.headers,
+            "body": self.body,
+            "latency_ms": self.latency_ms,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ResponseRecord":
+        return cls(
+            status_code=data["status_code"],
+            headers=data.get("headers") or {},
+            body=data.get("body"),
+            latency_ms=data.get("latency_ms", 0.0),
+        )
+
+
+@dataclass
+class RegressionRecord:
+    """A single recorded request/response pair linked to a test case."""
+
+    test_case_id: str
+    request: RequestRecord
+    response: ResponseRecord
+    passed: bool
+    error: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "test_case_id": self.test_case_id,
+            "request": self.request.to_dict(),
+            "response": self.response.to_dict(),
+            "passed": self.passed,
+            "error": self.error,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "RegressionRecord":
+        return cls(
+            test_case_id=data["test_case_id"],
+            request=RequestRecord.from_dict(data["request"]),
+            response=ResponseRecord.from_dict(data["response"]),
+            passed=data["passed"],
+            error=data.get("error"),
+        )
+
+
+@dataclass
+class RegressionBaseline:
+    """A portable regression baseline file containing all recorded request/response pairs.
+
+    Serialises to/from the standard NAT regression JSON format::
+
+        {
+          "version": "1.0",
+          "recorded_at": "2026-03-21T12:00:00Z",
+          "spec": "openapi.yaml",
+          "base_url": "https://api.example.com",
+          "records": [...]
+        }
+    """
+
+    version: str = "1.0"
+    recorded_at: str = ""
+    spec: Optional[str] = None
+    base_url: str = ""
+    records: List[RegressionRecord] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "version": self.version,
+            "recorded_at": self.recorded_at,
+            "spec": self.spec,
+            "base_url": self.base_url,
+            "records": [r.to_dict() for r in self.records],
+        }
+
+    def save(self, path: str) -> None:
+        """Serialise the baseline to a JSON file at *path*."""
+        with open(path, "w", encoding="utf-8") as fh:
+            json.dump(self.to_dict(), fh, indent=2)
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "RegressionBaseline":
+        return cls(
+            version=data.get("version", "1.0"),
+            recorded_at=data.get("recorded_at", ""),
+            spec=data.get("spec"),
+            base_url=data.get("base_url", ""),
+            records=[RegressionRecord.from_dict(r) for r in data.get("records", [])],
+        )
+
+    @classmethod
+    def load(cls, path: str) -> "RegressionBaseline":
+        """Load a baseline from a JSON file at *path*."""
+        with open(path, encoding="utf-8") as fh:
+            data = json.load(fh)
+        return cls.from_dict(data)
+
+
+@dataclass
+class FieldChange:
+    """A single changed field between baseline and replay."""
+
+    field: str
+    baseline: Any
+    replay: Any
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "field": self.field,
+            "baseline": self.baseline,
+            "replay": self.replay,
+        }
+
+
+@dataclass
+class RecordDiff:
+    """Diff result for a single test case."""
+
+    test_case_id: str
+    endpoint: str
+    changes: List[FieldChange] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "test_case_id": self.test_case_id,
+            "endpoint": self.endpoint,
+            "changes": [c.to_dict() for c in self.changes],
+        }
+
+
+@dataclass
+class RegressionDiff:
+    """Full diff report comparing a baseline to a replay run.
+
+    Serialises to the standard NAT regression diff JSON format::
+
+        {
+          "summary": {"total": 10, "unchanged": 8, "changed": 2, "new": 0, "removed": 0},
+          "changes": [...]
+        }
+    """
+
+    summary: Dict[str, int] = field(default_factory=dict)
+    changes: List[RecordDiff] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "summary": self.summary,
+            "changes": [c.to_dict() for c in self.changes],
+        }
+
+    def save(self, path: str) -> None:
+        """Write the diff report to a JSON file at *path*."""
+        with open(path, "w", encoding="utf-8") as fh:
+            json.dump(self.to_dict(), fh, indent=2)
+
+    def human_readable(self) -> str:
+        """Return a human-readable diff summary string."""
+        lines = []
+        s = self.summary
+        lines.append("=" * 60)
+        lines.append("  Regression Diff Report")
+        lines.append("=" * 60)
+        lines.append(f"  Total:     {s.get('total', 0)}")
+        lines.append(f"  Unchanged: {s.get('unchanged', 0)}")
+        lines.append(f"  Changed:   {s.get('changed', 0)}")
+        lines.append(f"  New:       {s.get('new', 0)}")
+        lines.append(f"  Removed:   {s.get('removed', 0)}")
+        lines.append("=" * 60)
+        for diff in self.changes:
+            lines.append(f"\n  [{diff.test_case_id}] {diff.endpoint}")
+            for chg in diff.changes:
+                lines.append(f"    {chg.field}:")
+                lines.append(f"      baseline: {chg.baseline!r}")
+                lines.append(f"      replay:   {chg.replay!r}")
+        return "\n".join(lines)

mannf/product/regression/recorder.py

@@ -0,0 +1,124 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""RegressionRecorder — hooks into SUT execute methods to capture request/response pairs."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+
+from mannf.product.regression.masking import mask_sensitive
+from mannf.product.regression.models import (
+    RegressionBaseline,
+    RegressionRecord,
+    RequestRecord,
+    ResponseRecord,
+)
+
+
+class RegressionRecorder:
+    """Collects :class:`~mannf.product.regression.models.RegressionRecord` objects during a test run.
+
+    Pass an instance to :class:`~mannf.product.integrations.http_sut.HttpApiSUT` or
+    :class:`~mannf.product.integrations.graphql_sut.GraphQLSUT` via the ``recorder``
+    constructor parameter.  After the run call :meth:`baseline` to obtain the
+    completed :class:`~mannf.product.regression.models.RegressionBaseline`.
+
+    Parameters
+    ----------
+    spec:
+        Path or identifier of the spec file used in this run.
+    base_url:
+        Root URL of the API under test.
+    mask_sensitive_data:
+        When ``True``, auth headers and token values are replaced with
+        ``***MASKED***`` before being stored.
+    """
+
+    def __init__(
+        self,
+        spec: Optional[str] = None,
+        base_url: str = "",
+        mask_sensitive_data: bool = False,
+    ) -> None:
+        self.spec = spec
+        self.base_url = base_url
+        self.mask_sensitive_data = mask_sensitive_data
+        self._records: list[RegressionRecord] = []
+
+    # ------------------------------------------------------------------
+    # Hook called by SUT implementations
+    # ------------------------------------------------------------------
+
+    def record(
+        self,
+        *,
+        test_case_id: str,
+        method: str,
+        url: str,
+        request_headers: Dict[str, str],
+        request_body: Any,
+        query_params: Optional[Dict[str, Any]],
+        status_code: int,
+        response_headers: Dict[str, str],
+        response_body: Any,
+        latency_ms: float,
+        passed: bool,
+        error: Optional[str],
+    ) -> None:
+        """Append one request/response pair to the in-memory log."""
+        if self.mask_sensitive_data:
+            request_headers, request_body, query_params = mask_sensitive(
+                headers=request_headers,
+                body=request_body,
+                query_params=query_params,
+            )
+            response_headers, response_body, _ = mask_sensitive(
+                headers=response_headers,
+                body=response_body,
+            )
+
+        self._records.append(
+            RegressionRecord(
+                test_case_id=test_case_id,
+                request=RequestRecord(
+                    method=method,
+                    url=url,
+                    headers=request_headers,
+                    body=request_body,
+                    query_params=query_params,
+                ),
+                response=ResponseRecord(
+                    status_code=status_code,
+                    headers=response_headers,
+                    body=response_body,
+                    latency_ms=latency_ms,
+                ),
+                passed=passed,
+                error=error,
+            )
+        )
+
+    # ------------------------------------------------------------------
+    # Output
+    # ------------------------------------------------------------------
+
+    def baseline(self) -> RegressionBaseline:
+        """Return a :class:`~mannf.product.regression.models.RegressionBaseline` for the current run."""
+        return RegressionBaseline(
+            version="1.0",
+            recorded_at=datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+            spec=self.spec,
+            base_url=self.base_url,
+            records=list(self._records),
+        )
+
+    def save(self, path: str) -> None:
+        """Write the current baseline to *path* as a JSON file."""
+        self.baseline().save(path)
+
+    def __len__(self) -> int:
+        return len(self._records)

mannf/product/regression/replayer.py

@@ -0,0 +1,168 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""RegressionReplayer — re-executes a previously recorded baseline against the live API."""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any, Dict, List, Optional
+
+import httpx
+
+from mannf.product.integrations.auth import AuthStrategy, NoAuth
+from mannf.product.regression.models import (
+    RegressionBaseline,
+    RegressionRecord,
+    RequestRecord,
+    ResponseRecord,
+)
+from mannf.product.regression.recorder import RegressionRecorder
+
+logger = logging.getLogger(__name__)
+
+
+class RegressionReplayer:
+    """Re-run all requests from a :class:`~mannf.product.regression.models.RegressionBaseline`.
+
+    For each record in the baseline the replayer constructs an identical HTTP
+    request, fires it at the (potentially changed) API, and stores the new
+    response.  The resulting :class:`~mannf.product.regression.models.RegressionBaseline`
+    can then be compared against the original with
+    :class:`~mannf.product.regression.differ.RegressionDiffer`.
+
+    Parameters
+    ----------
+    baseline:
+        The loaded regression baseline to replay.
+    auth:
+        Optional :class:`~mannf.product.integrations.auth.AuthStrategy` applied to
+        every replayed request (uses :class:`~mannf.product.integrations.auth.NoAuth`
+        by default).
+    timeout:
+        HTTP request timeout in seconds (default: 10).
+    mask_sensitive_data:
+        When ``True``, mask auth headers / token values in the replay
+        recording.
+    """
+
+    def __init__(
+        self,
+        baseline: RegressionBaseline,
+        auth: Optional[AuthStrategy] = None,
+        timeout: float = 10.0,
+        mask_sensitive_data: bool = False,
+    ) -> None:
+        self.baseline = baseline
+        self.auth: AuthStrategy = auth or NoAuth()
+        self.timeout = timeout
+        self.mask_sensitive_data = mask_sensitive_data
+
+    async def replay(self) -> RegressionBaseline:
+        """Execute all baseline requests and return a new baseline with replay responses.
+
+        Returns
+        -------
+        RegressionBaseline
+            A new baseline whose records contain the *replay* responses.
+            ``recorded_at`` is set to the current time.
+        """
+        recorder = RegressionRecorder(
+            spec=self.baseline.spec,
+            base_url=self.baseline.base_url,
+            mask_sensitive_data=self.mask_sensitive_data,
+        )
+
+        for record in self.baseline.records:
+            await self._replay_one(record, recorder)
+
+        return recorder.baseline()
+
+    async def _replay_one(
+        self, record: RegressionRecord, recorder: RegressionRecorder
+    ) -> None:
+        req: RequestRecord = record.request
+        request_kwargs: Dict[str, Any] = {
+            "method": req.method,
+            "url": req.url,
+            "headers": dict(req.headers),
+            "timeout": self.timeout,
+        }
+        if req.query_params:
+            request_kwargs["params"] = req.query_params
+        if req.body is not None:
+            if isinstance(req.body, (dict, list)):
+                request_kwargs["json"] = req.body
+            else:
+                request_kwargs["content"] = req.body
+
+        # Apply auth
+        request_kwargs = await self.auth.apply(request_kwargs)
+
+        start = time.monotonic()
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.request(**request_kwargs)
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+
+            try:
+                body: Any = response.json()
+            except Exception:
+                body = response.text
+
+            # Re-use original pass criteria: 2xx means pass
+            passed = 200 <= response.status_code < 300
+            error: Optional[str] = None if passed else f"Unexpected status {response.status_code}"
+
+            recorder.record(
+                test_case_id=record.test_case_id,
+                method=req.method,
+                url=req.url,
+                request_headers=dict(req.headers),
+                request_body=req.body,
+                query_params=req.query_params,
+                status_code=response.status_code,
+                response_headers=dict(response.headers),
+                response_body=body,
+                latency_ms=elapsed_ms,
+                passed=passed,
+                error=error,
+            )
+
+        except httpx.TimeoutException as exc:
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            logger.warning("RegressionReplayer: timeout replaying %s %s — %s", req.method, req.url, exc)
+            recorder.record(
+                test_case_id=record.test_case_id,
+                method=req.method,
+                url=req.url,
+                request_headers=dict(req.headers),
+                request_body=req.body,
+                query_params=req.query_params,
+                status_code=0,
+                response_headers={},
+                response_body=None,
+                latency_ms=elapsed_ms,
+                passed=False,
+                error=f"Request timed out: {exc}",
+            )
+        except httpx.RequestError as exc:
+            elapsed_ms = (time.monotonic() - start) * 1000.0
+            logger.warning("RegressionReplayer: connection error for %s %s — %s", req.method, req.url, exc)
+            recorder.record(
+                test_case_id=record.test_case_id,
+                method=req.method,
+                url=req.url,
+                request_headers=dict(req.headers),
+                request_body=req.body,
+                query_params=req.query_params,
+                status_code=0,
+                response_headers={},
+                response_body=None,
+                latency_ms=elapsed_ms,
+                passed=False,
+                error=f"Connection error: {exc}",
+            )

mannf/product/reports/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""NAT report generators — PDF and other exportable formats."""
+
+from mannf.product.reports.pdf import PDFReportGenerator
+
+__all__ = ["PDFReportGenerator"]

mannf/product/reports/pdf.py

@@ -0,0 +1,132 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""PDF report generator for NAT scan exports.
+
+Uses Playwright's ``page.pdf()`` to render the existing unified HTML report
+as a PDF document.  Playwright is already a declared dependency of the NAT
+framework (used by browser-based test agents), so no additional package is
+required.
+
+Enterprise white-label option:
+    When *whitelabel=True* is passed to :meth:`PDFReportGenerator.generate`,
+    all occurrences of the NAT brand strings are stripped from the HTML before
+    rendering, producing a clean, unbranded PDF suitable for enterprise
+    stakeholder distribution.
+
+Usage::
+
+    from mannf.product.reports.pdf import PDFReportGenerator
+
+    generator = PDFReportGenerator()
+    pdf_bytes = await generator.generate(html_content)
+
+    # Enterprise white-label (strip NAT branding):
+    pdf_bytes = await generator.generate(html_content, whitelabel=True)
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+# NAT brand strings to strip when producing white-label PDFs.
+_WHITELABEL_REPLACEMENTS: list[tuple[str, str]] = [
+    # Footer / title references
+    (r"Generated by NAT Framework\s*—\s*Neural Agent Testing\s*—\s*", "Generated "),
+    (r"NAT Unified Test Report", "Test Report"),
+    (r"NAT Scan Report", "Scan Report"),
+    (r"NeuroAgentTest\s*\([^)]*\)", ""),
+    (r"Neural Agent Testing Framework", ""),
+    (r"NAT Framework", ""),
+    # HTML <title> tag content
+    (r"(<title>)NAT\s+", r"\1"),
+    # Header h1 with "NAT" prefix
+    (r"<h1>NAT\s+", "<h1>"),
+]
+
+
+class PDFReportGenerator:
+    """Convert an HTML scan report to a PDF document using Playwright.
+
+    This class wraps Playwright's ``page.pdf()`` method to produce a
+    print-quality PDF from the same self-contained HTML report that is
+    served by ``GET /api/v1/scan/{scan_id}/export?format=html``.
+
+    Enterprise tenants may request white-label output (NAT branding stripped)
+    by passing ``whitelabel=True`` to :meth:`generate`.
+
+    Raises:
+        RuntimeError: If Playwright is not installed or the browser binary
+            has not been installed via ``playwright install``.
+    """
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    async def generate(self, html: str, whitelabel: bool = False) -> bytes:
+        """Render *html* as a PDF document and return the raw PDF bytes.
+
+        Parameters
+        ----------
+        html:
+            A self-contained HTML document (no external dependencies) to
+            render.  Typically the output of
+            :func:`~mannf.product.dashboard.app._render_export_html` or
+            :meth:`~mannf.core.reporting.unified_report.UnifiedReportGenerator.generate_html`.
+        whitelabel:
+            When ``True``, NAT brand strings are stripped from the HTML
+            before rendering.  Intended for Enterprise plan tenants.
+
+        Returns
+        -------
+        bytes
+            Raw PDF bytes suitable for returning as an HTTP response or
+            writing to disk.
+
+        Raises
+        ------
+        RuntimeError
+            If the ``playwright`` package is not importable or the required
+            browser binary is missing.
+        """
+        try:
+            from playwright.async_api import async_playwright  # noqa: PLC0415
+        except ImportError as exc:
+            raise RuntimeError(
+                "Playwright is required for PDF export. "
+                "Install it with: pip install playwright && playwright install chromium"
+            ) from exc
+
+        if whitelabel:
+            html = self._strip_branding(html)
+
+        async with async_playwright() as pw:
+            browser = await pw.chromium.launch()
+            try:
+                page = await browser.new_page()
+                await page.set_content(html, wait_until="networkidle")
+                pdf_bytes: bytes = await page.pdf(
+                    format="A4",
+                    print_background=True,
+                    margin={"top": "20mm", "bottom": "20mm", "left": "15mm", "right": "15mm"},
+                )
+            finally:
+                await browser.close()
+
+        return pdf_bytes
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _strip_branding(html: str) -> str:
+        """Remove NAT brand strings from *html* for white-label output."""
+        result = html
+        for pattern, replacement in _WHITELABEL_REPLACEMENTS:
+            result = re.sub(pattern, replacement, result)
+        return result