nat-engine 1__py3-none-any.whl

mannf/product/formatters/ctrf_formatter.py

@@ -0,0 +1,149 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""CTRF (Common Test Results Format) report formatter for NAT scan output.
+
+Produces a single ``ctrf-report.json`` file that conforms to the
+`CTRF schema <https://ctrf.io>`_ (v0).  The report contains one test entry
+per test case in the provided :class:`~mannf.core.testing.models.TestSuite`.
+
+Usage::
+
+    from mannf.product.formatters.ctrf_formatter import build_ctrf_report
+
+    json_str = build_ctrf_report(report, suite)
+    Path("ctrf-report.json").write_text(json_str, encoding="utf-8")
+
+CTRF is consumed by several GitHub Action steps (e.g. ``ctrf-io/github-actions-ctrf``),
+Slack notification bots, and custom dashboards.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from mannf.core.testing.models import TestSuite
+
+# ---------------------------------------------------------------------------
+# Status mapping
+# ---------------------------------------------------------------------------
+
+# CTRF statuses: passed | failed | skipped | pending | other
+_STATUS_MAP = {
+    "passed": "passed",
+    "failed": "failed",
+    "broken": "failed",
+    "skipped": "skipped",
+}
+
+
+def _result_status(result: Any) -> str:
+    """Return a CTRF-compatible status string for a single test result."""
+    if not getattr(result, "passed", True):
+        return "failed"
+    return "passed"
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def build_ctrf_report(
+    report: dict[str, Any],
+    suite: "TestSuite",
+) -> str:
+    """Build a CTRF JSON report string from a NAT scan result.
+
+    Parameters
+    ----------
+    report:
+        The NAT scan report dict (as produced by ``_run_scan``).
+    suite:
+        The :class:`~mannf.core.testing.models.TestSuite` that ran the tests.
+
+    Returns
+    -------
+    str
+        Pretty-printed CTRF JSON suitable for writing to a ``.json`` file.
+    """
+    s = report.get("suite", {})
+    suite_name = getattr(suite, "name", report.get("spec", "NAT Scan"))
+    base_url = report.get("base_url", report.get("endpoint", ""))
+    now_ms = int(datetime.now(timezone.utc).timestamp() * 1000)
+
+    total = int(s.get("total", 0))
+    passed = int(s.get("passed", 0))
+    failed = int(s.get("failed", 0)) + int(s.get("errored", 0))
+    skipped = 0
+    pending = 0
+    other = total - passed - failed - skipped - pending
+    if other < 0:
+        other = 0
+
+    tests: list[dict[str, Any]] = []
+    for result in getattr(suite, "results", []):
+        tc_id = getattr(result, "test_case_id", None) or "unknown"
+        status = _result_status(result)
+        exec_ms = getattr(result, "execution_time_ms", None) or 0.0
+        error = getattr(result, "error", None)
+
+        entry: dict[str, Any] = {
+            "name": tc_id,
+            "status": status,
+            "duration": round(float(exec_ms), 3),
+            "start": now_ms,
+            "stop": now_ms + int(exec_ms),
+        }
+        if error:
+            entry["message"] = str(error)
+            entry["trace"] = str(error)
+        if base_url:
+            entry["suite"] = suite_name
+        tests.append(entry)
+
+    ctrf: dict[str, Any] = {
+        "results": {
+            "tool": {
+                "name": "NAT",
+                "version": _nat_version(),
+                "url": "https://nat-testing.io",
+            },
+            "summary": {
+                "tests": total,
+                "passed": passed,
+                "failed": failed,
+                "pending": pending,
+                "skipped": skipped,
+                "other": other,
+                "start": now_ms,
+                "stop": now_ms,
+            },
+            "tests": tests,
+            "environment": {
+                "appName": suite_name,
+                "appUrl": base_url,
+            },
+        }
+    }
+
+    return json.dumps(ctrf, indent=2, ensure_ascii=False)
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _nat_version() -> str:
+    """Return the installed NAT version string, or 'unknown' if unavailable."""
+    try:
+        from mannf._version import __version__
+        return __version__
+    except Exception:
+        return "unknown"

mannf/product/healing/__init__.py

@@ -0,0 +1,30 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Self-healing test suite subsystem for NAT."""
+
+from mannf.product.healing.graphql_schema_diff import GraphQLSchemaDiffer
+from mannf.product.healing.healer import TestHealer
+from mannf.product.healing.models import (
+    EndpointChange,
+    FieldChange,
+    HealingAction,
+    HealingReport,
+    HealingResult,
+    SchemaDiff,
+)
+from mannf.product.healing.schema_diff import SchemaDiffer
+
+__all__ = [
+    "SchemaDiff",
+    "EndpointChange",
+    "FieldChange",
+    "HealingAction",
+    "HealingReport",
+    "HealingResult",
+    "SchemaDiffer",
+    "GraphQLSchemaDiffer",
+    "TestHealer",
+]

mannf/product/healing/graphql_schema_diff.py

@@ -0,0 +1,152 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""GraphQL schema differ."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List
+
+from mannf.product.healing.models import EndpointChange, FieldChange, SchemaDiff
+
+logger = logging.getLogger(__name__)
+
+
+class GraphQLSchemaDiffer:
+    """Compare two GraphQL introspection results and return a :class:`SchemaDiff`."""
+
+    def diff(
+        self, old_introspection: Dict[str, Any], new_introspection: Dict[str, Any]
+    ) -> SchemaDiff:
+        """Compare two GraphQL introspection results."""
+        old_schema = self._extract_schema(old_introspection)
+        new_schema = self._extract_schema(new_introspection)
+
+        old_types = old_schema.get("types", {})
+        new_types = new_schema.get("types", {})
+        old_ops = old_schema.get("operations", {})
+        new_ops = new_schema.get("operations", {})
+
+        added_endpoints: List[EndpointChange] = []
+        removed_endpoints: List[EndpointChange] = []
+        modified_endpoints: List[EndpointChange] = []
+
+        for name in set(new_ops) - set(old_ops):
+            kind = new_ops[name].get("kind", "query").upper()
+            added_endpoints.append(EndpointChange(method=kind, path=name, change_type="added"))
+        for name in set(old_ops) - set(new_ops):
+            kind = old_ops[name].get("kind", "query").upper()
+            removed_endpoints.append(EndpointChange(method=kind, path=name, change_type="removed"))
+        for name in set(old_ops) & set(new_ops):
+            old_op = old_ops[name]
+            new_op = new_ops[name]
+            changes = self._diff_gql_operation(name, old_op, new_op, old_types, new_types)
+            if changes:
+                kind = new_op.get("kind", "query").upper()
+                modified_endpoints.append(
+                    EndpointChange(method=kind, path=name, change_type="modified", details=changes)
+                )
+
+        result = SchemaDiff(
+            added_endpoints=sorted(added_endpoints, key=lambda e: e.path),
+            removed_endpoints=sorted(removed_endpoints, key=lambda e: e.path),
+            modified_endpoints=sorted(modified_endpoints, key=lambda e: e.path),
+        )
+        result.summary = {
+            "added": len(result.added_endpoints),
+            "removed": len(result.removed_endpoints),
+            "modified": len(result.modified_endpoints),
+            "total_changes": (
+                len(result.added_endpoints)
+                + len(result.removed_endpoints)
+                + len(result.modified_endpoints)
+            ),
+        }
+        return result
+
+    def _extract_schema(self, introspection: Dict[str, Any]) -> Dict[str, Any]:
+        """Parse introspection result into operations and types dict."""
+        data = introspection.get("data", introspection)
+        schema = data.get("__schema", {})
+
+        types_map: Dict[str, Dict[str, Any]] = {}
+        for t in schema.get("types", []):
+            name = t.get("name", "")
+            if name and not name.startswith("__"):
+                types_map[name] = t
+
+        operations: Dict[str, Dict[str, Any]] = {}
+
+        def _extract_ops_from_type(type_name: str, kind: str) -> None:
+            type_obj = types_map.get(type_name, {})
+            for f in type_obj.get("fields", []) or []:
+                field_name = f.get("name", "")
+                if field_name:
+                    operations[field_name] = {
+                        "kind": kind,
+                        "type": f.get("type"),
+                        "args": f.get("args", []),
+                    }
+
+        query_type = (schema.get("queryType") or {}).get("name")
+        mutation_type = (schema.get("mutationType") or {}).get("name")
+        if query_type:
+            _extract_ops_from_type(query_type, "query")
+        if mutation_type:
+            _extract_ops_from_type(mutation_type, "mutation")
+
+        return {"types": types_map, "operations": operations}
+
+    def _diff_gql_operation(
+        self,
+        name: str,
+        old_op: Dict[str, Any],
+        new_op: Dict[str, Any],
+        old_types: Dict[str, Any],
+        new_types: Dict[str, Any],
+    ) -> List[FieldChange]:
+        changes: List[FieldChange] = []
+
+        # Compare return type
+        old_type = _type_name(old_op.get("type"))
+        new_type = _type_name(new_op.get("type"))
+        if old_type != new_type:
+            changes.append(FieldChange(
+                field_path=f"{name}.returnType",
+                change_type="type_changed",
+                old_value=old_type,
+                new_value=new_type,
+            ))
+
+        # Compare args
+        old_args = {a["name"]: a for a in (old_op.get("args") or []) if a.get("name")}
+        new_args = {a["name"]: a for a in (new_op.get("args") or []) if a.get("name")}
+        for arg_name in set(old_args) - set(new_args):
+            changes.append(FieldChange(
+                field_path=f"{name}.args.{arg_name}",
+                change_type="removed",
+                old_value=old_args[arg_name],
+                new_value=None,
+            ))
+        for arg_name in set(new_args) - set(old_args):
+            changes.append(FieldChange(
+                field_path=f"{name}.args.{arg_name}",
+                change_type="added",
+                old_value=None,
+                new_value=new_args[arg_name],
+            ))
+
+        return changes
+
+
+def _type_name(type_ref: Any) -> str:
+    """Unwrap NON_NULL/LIST wrappers and return base type name."""
+    if not type_ref:
+        return ""
+    kind = type_ref.get("kind", "")
+    if kind in ("NON_NULL", "LIST"):
+        return _type_name(type_ref.get("ofType"))
+    return type_ref.get("name", "")

mannf/product/healing/healer.py

@@ -0,0 +1,141 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""TestHealer — automated self-healing for NAT test suites."""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from mannf.product.healing.models import HealingAction, HealingReport, HealingResult
+from mannf.product.healing.schema_diff import SchemaDiffer
+from mannf.core.testing.models import TestCase
+
+logger = logging.getLogger(__name__)
+
+
+class TestHealer:
+    """Automatically heal a NAT test suite when the API spec changes.
+
+    Steps:
+    1. Parse both specs using OpenApiParser
+    2. Run SchemaDiffer.diff() to identify changes
+    3. For removed endpoints: retire matching tests
+    4. For added endpoints: generate new tests
+    5. For modified endpoints: remove old, generate new
+    """
+
+    def heal(
+        self,
+        old_spec_path: str,
+        new_spec_path: str,
+        existing_tests: List[TestCase],
+        base_url: str = "",
+        scan_id: Optional[str] = None,
+    ) -> HealingResult:
+        """Run healing and return a :class:`HealingResult`."""
+        from mannf.product.integrations.openapi_parser import OpenApiParser, _load_spec
+
+        scan_id = scan_id or str(uuid.uuid4())
+        now = datetime.now(timezone.utc).isoformat()
+
+        # Load both specs
+        old_spec = _load_spec(old_spec_path)
+        new_spec = _load_spec(new_spec_path)
+
+        # Diff
+        differ = SchemaDiffer()
+        schema_diff = differ.diff(old_spec, new_spec)
+
+        actions: List[HealingAction] = []
+        removed_tests: List[TestCase] = []
+        new_tests: List[TestCase] = []
+
+        # Build sets of affected endpoints
+        removed_eps = {(ec.method, ec.path) for ec in schema_diff.removed_endpoints}
+        added_eps = {(ec.method, ec.path) for ec in schema_diff.added_endpoints}
+        modified_eps = {(ec.method, ec.path) for ec in schema_diff.modified_endpoints}
+
+        # Categorize existing tests
+        kept_tests: List[TestCase] = []
+        for tc in existing_tests:
+            spec_path = tc.metadata.get("spec_path", tc.target)
+            method = tc.metadata.get("method", "GET")
+            key = (method, spec_path)
+
+            if key in removed_eps:
+                removed_tests.append(tc)
+                actions.append(HealingAction(
+                    action_type="removed",
+                    endpoint=f"{method} {spec_path}",
+                    test_case_id=tc.id,
+                    reason="Endpoint removed from spec",
+                    timestamp=now,
+                ))
+            elif key in modified_eps:
+                removed_tests.append(tc)
+                actions.append(HealingAction(
+                    action_type="removed",
+                    endpoint=f"{method} {spec_path}",
+                    test_case_id=tc.id,
+                    reason="Endpoint modified in spec — regenerating",
+                    timestamp=now,
+                ))
+            else:
+                kept_tests.append(tc)
+
+        # Generate new tests for added endpoints and modified endpoints
+        needs_regen = added_eps | modified_eps
+        if needs_regen:
+            try:
+                parser = OpenApiParser(spec_path=new_spec_path, base_url=base_url)
+                all_new_cases = parser.parse()
+                for tc in all_new_cases:
+                    tc_spec_path = tc.metadata.get("spec_path", tc.target)
+                    tc_method = tc.metadata.get("method", "GET")
+                    key = (tc_method, tc_spec_path)
+                    if key in needs_regen:
+                        new_tests.append(tc)
+                        action_type = "regenerated"
+                        reason = (
+                            "Endpoint added to spec"
+                            if key in added_eps
+                            else "Endpoint modified — regenerated"
+                        )
+                        actions.append(HealingAction(
+                            action_type=action_type,
+                            endpoint=f"{tc_method} {tc_spec_path}",
+                            test_case_id=tc.id,
+                            reason=reason,
+                            timestamp=now,
+                        ))
+            except Exception as exc:
+                logger.warning("Failed to regenerate tests for new/modified endpoints: %s", exc)
+
+        summary = {
+            "regenerated": sum(1 for a in actions if a.action_type == "regenerated"),
+            "removed": sum(1 for a in actions if a.action_type == "removed"),
+            "kept": len(kept_tests),
+        }
+
+        report = HealingReport(
+            scan_id=scan_id,
+            old_spec=old_spec_path,
+            new_spec=new_spec_path,
+            actions=actions,
+            summary=summary,
+            timestamp=now,
+            schema_diff=schema_diff,
+        )
+
+        return HealingResult(
+            kept_tests=kept_tests,
+            removed_tests=removed_tests,
+            new_tests=new_tests,
+            healing_report=report,
+        )

mannf/product/healing/models.py

@@ -0,0 +1,175 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Data models for the self-healing test suite system."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class FieldChange:
+    """A field-level change detected between two API specs."""
+
+    field_path: str
+    change_type: str  # "added" | "removed" | "type_changed" | "constraint_changed"
+    old_value: Any = None
+    new_value: Any = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "field_path": self.field_path,
+            "change_type": self.change_type,
+            "old_value": self.old_value,
+            "new_value": self.new_value,
+        }
+
+
+@dataclass
+class EndpointChange:
+    """A change detected for a specific API endpoint."""
+
+    method: str
+    path: str
+    change_type: str  # "added" | "removed" | "modified"
+    details: List[FieldChange] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "method": self.method,
+            "path": self.path,
+            "change_type": self.change_type,
+            "details": [d.to_dict() for d in self.details],
+        }
+
+
+@dataclass
+class SchemaDiff:
+    """Result of comparing two API specs."""
+
+    added_endpoints: List[EndpointChange] = field(default_factory=list)
+    removed_endpoints: List[EndpointChange] = field(default_factory=list)
+    modified_endpoints: List[EndpointChange] = field(default_factory=list)
+    summary: Dict[str, int] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "added_endpoints": [e.to_dict() for e in self.added_endpoints],
+            "removed_endpoints": [e.to_dict() for e in self.removed_endpoints],
+            "modified_endpoints": [e.to_dict() for e in self.modified_endpoints],
+            "summary": self.summary,
+        }
+
+    @property
+    def has_changes(self) -> bool:
+        return bool(self.added_endpoints or self.removed_endpoints or self.modified_endpoints)
+
+
+@dataclass
+class HealingAction:
+    """A single healing action taken by the TestHealer."""
+
+    action_type: str  # "regenerated" | "removed" | "repaired"
+    endpoint: str  # "{method} {path}"
+    test_case_id: str
+    reason: str
+    timestamp: str
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "action_type": self.action_type,
+            "endpoint": self.endpoint,
+            "test_case_id": self.test_case_id,
+            "reason": self.reason,
+            "timestamp": self.timestamp,
+        }
+
+
+@dataclass
+class HealingReport:
+    """Full report of all healing actions taken during a self-healing run."""
+
+    scan_id: str
+    old_spec: str
+    new_spec: str
+    actions: List[HealingAction] = field(default_factory=list)
+    summary: Dict[str, int] = field(default_factory=dict)
+    timestamp: str = ""
+    schema_diff: Optional[SchemaDiff] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "scan_id": self.scan_id,
+            "old_spec": self.old_spec,
+            "new_spec": self.new_spec,
+            "actions": [a.to_dict() for a in self.actions],
+            "summary": self.summary,
+            "timestamp": self.timestamp,
+            "schema_diff": self.schema_diff.to_dict() if self.schema_diff else None,
+        }
+
+    def save(self, path: str) -> None:
+        with open(path, "w", encoding="utf-8") as fh:
+            json.dump(self.to_dict(), fh, indent=2)
+
+    def human_readable(self) -> str:
+        lines = []
+        lines.append("=" * 60)
+        lines.append("  Self-Healing Report")
+        lines.append("=" * 60)
+        lines.append(f"  Scan ID:  {self.scan_id}")
+        lines.append(f"  Old spec: {self.old_spec}")
+        lines.append(f"  New spec: {self.new_spec}")
+        lines.append(f"  Time:     {self.timestamp}")
+        lines.append("")
+        s = self.summary
+        lines.append(f"  Tests regenerated: {s.get('regenerated', 0)}")
+        lines.append(f"  Tests removed:     {s.get('removed', 0)}")
+        lines.append(f"  Tests kept:        {s.get('kept', 0)}")
+        lines.append("=" * 60)
+        if self.actions:
+            lines.append("\nActions:")
+            for action in self.actions:
+                lines.append(f"  [{action.action_type.upper()}] {action.endpoint} — {action.reason}")
+        return "\n".join(lines)
+
+    def markdown(self) -> str:
+        lines = []
+        lines.append("# Self-Healing Report")
+        lines.append(f"\n**Scan ID:** {self.scan_id}")
+        lines.append(f"**Timestamp:** {self.timestamp}")
+        lines.append(f"**Old Spec:** `{self.old_spec}`")
+        lines.append(f"**New Spec:** `{self.new_spec}`")
+        s = self.summary
+        lines.append("\n## Summary")
+        lines.append("| Action | Count |")
+        lines.append("|--------|-------|")
+        lines.append(f"| Regenerated | {s.get('regenerated', 0)} |")
+        lines.append(f"| Removed | {s.get('removed', 0)} |")
+        lines.append(f"| Kept | {s.get('kept', 0)} |")
+        if self.actions:
+            lines.append("\n## Actions")
+            lines.append("| Type | Endpoint | Reason |")
+            lines.append("|------|----------|--------|")
+            for action in self.actions:
+                lines.append(f"| {action.action_type} | {action.endpoint} | {action.reason} |")
+        return "\n".join(lines)
+
+
+@dataclass
+class HealingResult:
+    """Result of running the TestHealer."""
+
+    kept_tests: List[Any] = field(default_factory=list)     # List[TestCase]
+    removed_tests: List[Any] = field(default_factory=list)  # List[TestCase]
+    new_tests: List[Any] = field(default_factory=list)      # List[TestCase]
+    healing_report: Optional[HealingReport] = None
+
+    @property
+    def all_active_tests(self) -> List[Any]:
+        return self.kept_tests + self.new_tests