nat-engine 1__py3-none-any.whl

mannf/core/agents/autonomous_loop_models.py

@@ -0,0 +1,62 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Pydantic models for the autonomous test loop (Phase 5).
+
+Defines the per-iteration and full-run result structures produced by
+:class:`~mannf.core.agents.autonomous_loop_agent.AutonomousTestLoopAgent`.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class LoopIterationResult(BaseModel):
+    """Aggregated outcomes for a single autonomous loop iteration."""
+
+    iteration: int
+    scenarios_generated: int
+    scenarios_executed: int
+    passed: int
+    failed: int
+    flaky: int
+    new_failures: int  # failures not seen in previous iterations
+    coverage_pages: int  # unique pages tested this iteration
+    duration_s: float
+
+
+class AutonomousRunReport(BaseModel):
+    """Complete report for a finished (or in-progress) autonomous run."""
+
+    run_id: str
+    target_url: str
+    strategy: str
+    total_iterations: int
+    total_scenarios_executed: int
+    total_passed: int
+    total_failed: int
+    total_flaky: int
+    unique_failures: List[Dict[str, Any]] = Field(default_factory=list)
+    coverage_summary: Dict[str, Any] = Field(default_factory=dict)
+    iterations: List[LoopIterationResult] = Field(default_factory=list)
+    started_at: str
+    completed_at: str = ""
+    duration_s: float = 0.0
+    stop_reason: str = ""  # "max_iterations", "max_duration", "stable", "coverage_met"
+    status: str = "running"  # "running", "complete", "stopped", "failed"
+    error: str = ""
+    # Deploy / commit attribution (Phase 6.3)
+    build_id: Optional[str] = None
+    commit_sha: Optional[str] = None
+    deploy_tag: Optional[str] = None
+    # Belief evolution snapshots — list of per-iteration snapshots (Phase 6.5)
+    belief_snapshots: List[Dict[str, Any]] = Field(default_factory=list)
+    # LLM root cause suggestions per failure (Phase 6.5)
+    root_cause_suggestions: List[Dict[str, Any]] = Field(default_factory=list)
+    # Coverage gaps — pages blocked by auth/CAPTCHA/timeout/unreachable (Phase 7.4)
+    coverage_gaps: List[Dict[str, Any]] = Field(default_factory=list)

mannf/core/agents/autonomous_run_differ.py

@@ -0,0 +1,427 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""AutonomousRunDiffer — run-over-run diff engine for autonomous test loop reports.
+
+Compares two :class:`~mannf.core.agents.autonomous_loop_models.AutonomousRunReport`
+objects and produces a structured :class:`AutonomousRunDiff` identifying:
+
+- Per-page pass/fail deltas
+- New failures (regressions) since the baseline run
+- Resolved failures (recoveries) that no longer occur
+- Flakiness changes
+- Coverage delta (pages added / removed)
+- Belief score changes
+
+The comparison key for failures is the ``page_key`` (path component of the URL)
+plus an error signature string, mirroring the dedup logic in
+:func:`~mannf.core.agents.autonomous_loop_agent._page_key`.
+
+Follows the same structural pattern as
+:class:`~mannf.product.regression.differ.RegressionDiffer` — a class with a
+:meth:`diff` method that accepts two report objects and returns a structured
+result.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+from urllib.parse import urlparse
+
+from mannf.core.agents.autonomous_loop_models import AutonomousRunReport
+
+
+# ---------------------------------------------------------------------------
+# Structured result types
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class PageDelta:
+    """Pass/fail delta for a single page between two runs."""
+
+    page_key: str
+    baseline_passed: int
+    baseline_failed: int
+    current_passed: int
+    current_failed: int
+
+    @property
+    def pass_delta(self) -> int:
+        return self.current_passed - self.baseline_passed
+
+    @property
+    def fail_delta(self) -> int:
+        return self.current_failed - self.baseline_failed
+
+
+@dataclass
+class FailureEntry:
+    """A single unique failure identified by page + error signature."""
+
+    page_key: str
+    error_signature: str
+    url: str = ""
+    scenario_type: str = ""
+    details: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class FlakinessDelta:
+    """Change in flaky scenario count between two runs."""
+
+    page_key: str
+    baseline_flaky: int
+    current_flaky: int
+
+    @property
+    def delta(self) -> int:
+        return self.current_flaky - self.baseline_flaky
+
+
+@dataclass
+class CoverageDelta:
+    """Coverage delta: pages added or removed between two runs."""
+
+    pages_added: List[str] = field(default_factory=list)
+    pages_removed: List[str] = field(default_factory=list)
+    baseline_page_count: int = 0
+    current_page_count: int = 0
+
+
+@dataclass
+class AutonomousRunDiff:
+    """Structured diff between a baseline and a current autonomous run."""
+
+    baseline_run_id: str
+    current_run_id: str
+    baseline_commit: Optional[str]
+    current_commit: Optional[str]
+    baseline_build: Optional[str]
+    current_build: Optional[str]
+
+    summary: Dict[str, Any] = field(default_factory=dict)
+    new_failures: List[FailureEntry] = field(default_factory=list)
+    resolved_failures: List[FailureEntry] = field(default_factory=list)
+    page_deltas: List[PageDelta] = field(default_factory=list)
+    flakiness_changes: List[FlakinessDelta] = field(default_factory=list)
+    coverage_delta: CoverageDelta = field(default_factory=CoverageDelta)
+
+    def human_readable(self) -> str:
+        """Return a human-readable diff summary string."""
+        lines: List[str] = []
+        s = self.summary
+        lines.append("=" * 60)
+        lines.append("  Autonomous Run Diff Report")
+        lines.append("=" * 60)
+        lines.append(f"  Baseline:  {self.baseline_run_id[:8]}")
+        lines.append(f"  Current:   {self.current_run_id[:8]}")
+        if self.baseline_commit:
+            lines.append(f"  Baseline commit: {self.baseline_commit}")
+        if self.current_commit:
+            lines.append(f"  Current commit:  {self.current_commit}")
+        lines.append("-" * 60)
+        lines.append(f"  New failures:      {s.get('new_failure_count', 0)}")
+        lines.append(f"  Resolved failures: {s.get('resolved_failure_count', 0)}")
+        lines.append(f"  Pass delta:        {s.get('total_pass_delta', 0):+d}")
+        lines.append(f"  Fail delta:        {s.get('total_fail_delta', 0):+d}")
+        lines.append(f"  Coverage added:    {s.get('pages_added', 0)}")
+        lines.append(f"  Coverage removed:  {s.get('pages_removed', 0)}")
+        lines.append("=" * 60)
+        if self.new_failures:
+            lines.append("\n  New Failures:")
+            for f in self.new_failures:
+                lines.append(f"    [{f.page_key}] {f.error_signature}")
+        if self.resolved_failures:
+            lines.append("\n  Resolved Failures:")
+            for f in self.resolved_failures:
+                lines.append(f"    [{f.page_key}] {f.error_signature}")
+        return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# Differ
+# ---------------------------------------------------------------------------
+
+
+def _normalize_page_key(url: str, base_url: str) -> str:
+    """Return a normalised page-relative path (mirrors autonomous_loop_agent._page_key)."""
+    try:
+        base = urlparse(base_url)
+        parsed = urlparse(url)
+        if parsed.netloc == base.netloc or not parsed.netloc:
+            return parsed.path or "/"
+        return url
+    except Exception:  # noqa: BLE001
+        return url
+
+
+def _failure_signature(failure: Dict[str, Any]) -> str:
+    """Derive a stable dedup key from a unique-failure dict.
+
+    Uses the ``error`` field if present, otherwise falls back to
+    ``scenario_type`` + ``page_key``.
+    """
+    parts: List[str] = []
+    if failure.get("error"):
+        parts.append(str(failure["error"])[:120])
+    elif failure.get("scenario_type"):
+        parts.append(str(failure["scenario_type"]))
+    parts.append(str(failure.get("page_key", failure.get("url", ""))))
+    return "|".join(parts)
+
+
+def _build_failure_index(
+    unique_failures: List[Dict[str, Any]],
+    base_url: str,
+) -> Dict[str, FailureEntry]:
+    """Build a ``{signature: FailureEntry}`` index from the unique_failures list."""
+    index: Dict[str, FailureEntry] = {}
+    for f in unique_failures:
+        url = f.get("url", "")
+        page_key = f.get("page_key") or _normalize_page_key(url, base_url)
+        sig = _failure_signature(f)
+        composite_key = f"{page_key}::{sig}"
+        index[composite_key] = FailureEntry(
+            page_key=page_key,
+            error_signature=sig,
+            url=url,
+            scenario_type=f.get("scenario_type", ""),
+            details={k: v for k, v in f.items() if k not in {"page_key", "url", "error"}},
+        )
+    return index
+
+
+def _page_stats_from_failures(
+    unique_failures: List[Dict[str, Any]],
+    base_url: str,
+    total_passed: int,
+    total_failed: int,
+) -> Dict[str, Dict[str, int]]:
+    """Build per-page pass/fail stats from a run report.
+
+    When per-page breakdowns are not directly stored in the report, we
+    distribute the aggregate totals evenly across the pages that have failures
+    and treat the remainder as passed on an implicit "root" page.
+    """
+    page_failed: Dict[str, int] = {}
+    for f in unique_failures:
+        url = f.get("url", "")
+        pk = f.get("page_key") or _normalize_page_key(url, base_url)
+        page_failed[pk] = page_failed.get(pk, 0) + 1
+
+    stats: Dict[str, Dict[str, int]] = {}
+    for pk, fail_count in page_failed.items():
+        stats[pk] = {"passed": 0, "failed": fail_count}
+
+    # Assign remaining passed count to a synthetic root page when we have no
+    # page-level breakdown
+    if total_passed > 0 and not stats:
+        stats["/"] = {"passed": total_passed, "failed": 0}
+
+    return stats
+
+
+class AutonomousRunDiffer:
+    """Compare two :class:`~mannf.core.agents.autonomous_loop_models.AutonomousRunReport`
+    objects and produce an :class:`AutonomousRunDiff`.
+
+    Parameters
+    ----------
+    flakiness_spike_threshold:
+        Minimum absolute increase in flaky scenarios on a page to be reported
+        as a flakiness spike.  Defaults to ``2``.
+    """
+
+    def __init__(self, flakiness_spike_threshold: int = 2) -> None:
+        self.flakiness_spike_threshold = flakiness_spike_threshold
+
+    def diff(
+        self,
+        baseline: AutonomousRunReport,
+        current: AutonomousRunReport,
+    ) -> AutonomousRunDiff:
+        """Compute the run-over-run diff between *baseline* and *current*.
+
+        Parameters
+        ----------
+        baseline:
+            The earlier run to compare against.
+        current:
+            The newer run to compare.
+
+        Returns
+        -------
+        AutonomousRunDiff
+            Structured diff report.
+        """
+        base_url = baseline.target_url or current.target_url
+
+        # Build failure indexes
+        baseline_failures = _build_failure_index(
+            baseline.unique_failures, base_url
+        )
+        current_failures = _build_failure_index(
+            current.unique_failures, base_url
+        )
+
+        baseline_keys = set(baseline_failures.keys())
+        current_keys = set(current_failures.keys())
+
+        new_failure_keys = current_keys - baseline_keys
+        resolved_failure_keys = baseline_keys - current_keys
+
+        new_failures = [current_failures[k] for k in sorted(new_failure_keys)]
+        resolved_failures = [baseline_failures[k] for k in sorted(resolved_failure_keys)]
+
+        # Per-page pass/fail deltas
+        page_deltas = self._compute_page_deltas(baseline, current, base_url)
+
+        # Flakiness changes
+        flakiness_changes = self._compute_flakiness_changes(baseline, current, base_url)
+
+        # Coverage delta
+        coverage_delta = self._compute_coverage_delta(baseline, current)
+
+        # Summary
+        total_pass_delta = current.total_passed - baseline.total_passed
+        total_fail_delta = current.total_failed - baseline.total_failed
+
+        summary: Dict[str, Any] = {
+            "new_failure_count": len(new_failures),
+            "resolved_failure_count": len(resolved_failures),
+            "total_pass_delta": total_pass_delta,
+            "total_fail_delta": total_fail_delta,
+            "pages_added": len(coverage_delta.pages_added),
+            "pages_removed": len(coverage_delta.pages_removed),
+            "baseline_total_passed": baseline.total_passed,
+            "baseline_total_failed": baseline.total_failed,
+            "current_total_passed": current.total_passed,
+            "current_total_failed": current.total_failed,
+            "flakiness_spike_pages": len(
+                [
+                    c
+                    for c in flakiness_changes
+                    if c.delta >= self.flakiness_spike_threshold
+                ]
+            ),
+        }
+
+        return AutonomousRunDiff(
+            baseline_run_id=baseline.run_id,
+            current_run_id=current.run_id,
+            baseline_commit=baseline.commit_sha,
+            current_commit=current.commit_sha,
+            baseline_build=baseline.build_id,
+            current_build=current.build_id,
+            summary=summary,
+            new_failures=new_failures,
+            resolved_failures=resolved_failures,
+            page_deltas=page_deltas,
+            flakiness_changes=flakiness_changes,
+            coverage_delta=coverage_delta,
+        )
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _compute_page_deltas(
+        self,
+        baseline: AutonomousRunReport,
+        current: AutonomousRunReport,
+        base_url: str,
+    ) -> List[PageDelta]:
+        baseline_stats = _page_stats_from_failures(
+            baseline.unique_failures,
+            base_url,
+            baseline.total_passed,
+            baseline.total_failed,
+        )
+        current_stats = _page_stats_from_failures(
+            current.unique_failures,
+            base_url,
+            current.total_passed,
+            current.total_failed,
+        )
+
+        all_pages = set(baseline_stats.keys()) | set(current_stats.keys())
+        deltas: List[PageDelta] = []
+        for page in sorted(all_pages):
+            b = baseline_stats.get(page, {"passed": 0, "failed": 0})
+            c = current_stats.get(page, {"passed": 0, "failed": 0})
+            if b != c:
+                deltas.append(
+                    PageDelta(
+                        page_key=page,
+                        baseline_passed=b["passed"],
+                        baseline_failed=b["failed"],
+                        current_passed=c["passed"],
+                        current_failed=c["failed"],
+                    )
+                )
+        return deltas
+
+    def _compute_flakiness_changes(
+        self,
+        baseline: AutonomousRunReport,
+        current: AutonomousRunReport,
+        base_url: str,
+    ) -> List[FlakinessDelta]:
+        """Compute per-page flakiness change using iteration-level data."""
+
+        def _flaky_by_page(report: AutonomousRunReport) -> Dict[str, int]:
+            """Sum flaky counts by page using coverage_summary if available."""
+            cs = report.coverage_summary or {}
+            if cs.get("pages_by_risk"):
+                # Not per-page flakiness — fall through to aggregate
+                pass
+            # Use total_flaky as a single aggregate bucket
+            if report.total_flaky:
+                return {"/": report.total_flaky}
+            return {}
+
+        baseline_flaky = _flaky_by_page(baseline)
+        current_flaky = _flaky_by_page(current)
+        all_pages = set(baseline_flaky.keys()) | set(current_flaky.keys())
+
+        changes: List[FlakinessDelta] = []
+        for page in sorted(all_pages):
+            b_val = baseline_flaky.get(page, 0)
+            c_val = current_flaky.get(page, 0)
+            if b_val != c_val:
+                changes.append(
+                    FlakinessDelta(
+                        page_key=page,
+                        baseline_flaky=b_val,
+                        current_flaky=c_val,
+                    )
+                )
+        return changes
+
+    def _compute_coverage_delta(
+        self,
+        baseline: AutonomousRunReport,
+        current: AutonomousRunReport,
+    ) -> CoverageDelta:
+        """Compare covered_pages sets from coverage_summary."""
+
+        def _pages(report: AutonomousRunReport) -> set:
+            cs = report.coverage_summary or {}
+            raw = cs.get("covered_pages", [])
+            if isinstance(raw, list):
+                return set(raw)
+            return set()
+
+        baseline_pages = _pages(baseline)
+        current_pages = _pages(current)
+
+        return CoverageDelta(
+            pages_added=sorted(current_pages - baseline_pages),
+            pages_removed=sorted(baseline_pages - current_pages),
+            baseline_page_count=len(baseline_pages),
+            current_page_count=len(current_pages),
+        )

mannf/core/agents/base.py

@@ -0,0 +1,128 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Abstract base class for all agents in the framework.
+
+Every agent:
+  • has a unique ID and a human-readable role label
+  • owns a :class:`NeuralNetwork` for decision-making
+  • subscribes to the shared :class:`MessageBus`
+  • runs as an ``asyncio`` task via :meth:`run`
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from abc import ABC, abstractmethod
+from typing import Optional, Set
+
+from mannf.core.messaging.bus import MessageBus
+from mannf.core.messaging.messages import Message, MessageType
+from mannf.core.neural import NeuralNetwork
+
+logger = logging.getLogger(__name__)
+
+
+class BaseAgent(ABC):
+    """Abstract base for all MANNF agents.
+
+    Parameters
+    ----------
+    agent_id:
+        Unique identifier for this agent instance.
+    bus:
+        Shared message bus used for inter-agent communication.
+    network:
+        Optional neural network the agent uses for decision-making.
+        Subclasses may construct their own if *None* is passed.
+    """
+
+    def __init__(
+        self,
+        agent_id: str,
+        bus: MessageBus,
+        network: Optional[NeuralNetwork] = None,
+    ) -> None:
+        self.agent_id = agent_id
+        self._bus = bus
+        self.network = network
+
+        self._running = False
+        self._task: Optional[asyncio.Task] = None  # type: ignore[type-arg]
+        self._inbox: asyncio.Queue[Message] = asyncio.Queue()
+
+    # ------------------------------------------------------------------
+    # Subscriptions – override to declare which message types to receive
+    # ------------------------------------------------------------------
+
+    @property
+    @abstractmethod
+    def subscribed_types(self) -> Set[MessageType]:
+        """The set of :class:`MessageType` values this agent subscribes to."""
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    async def start(self) -> None:
+        """Register subscriptions, announce AGENT_STARTED, then start the run loop."""
+        self._inbox = self._bus.subscribe(self.agent_id, self.subscribed_types)
+        self._running = True
+        await self._bus.publish(
+            Message(MessageType.AGENT_STARTED, sender_id=self.agent_id)
+        )
+        logger.info("Agent %s started", self.agent_id)
+
+    async def stop(self) -> None:
+        """Announce AGENT_STOPPED and tear down the subscription."""
+        self._running = False
+        await self._bus.publish(
+            Message(MessageType.AGENT_STOPPED, sender_id=self.agent_id)
+        )
+        self._bus.unsubscribe(self.agent_id)
+        logger.info("Agent %s stopped", self.agent_id)
+
+    # ------------------------------------------------------------------
+    # Main loop
+    # ------------------------------------------------------------------
+
+    async def run(self) -> None:
+        """Entry point for the agent's asyncio task.
+
+        Calls :meth:`start`, then processes incoming messages until
+        :meth:`stop` is called.  An explicit ``asyncio.sleep(0)`` after each
+        message ensures other concurrent tasks (e.g. the coordinator's
+        award timers) always get a chance to execute.
+        """
+        await self.start()
+        try:
+            while self._running:
+                try:
+                    msg = await asyncio.wait_for(self._inbox.get(), timeout=1.0)
+                    await self._handle_message(msg)
+                    # Explicit yield so other tasks (timers, award callbacks) can run
+                    await asyncio.sleep(0)
+                except asyncio.TimeoutError:
+                    await self._on_idle()
+        finally:
+            await self.stop()
+
+    @abstractmethod
+    async def _handle_message(self, message: Message) -> None:
+        """Process a single inbound message."""
+
+    async def _on_idle(self) -> None:
+        """Called each time the inbox timeout expires.  Override for background work."""
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    async def _publish(self, message: Message) -> None:
+        await self._bus.publish(message)
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}(id={self.agent_id!r})"