nat-engine 1__py3-none-any.whl

mannf/core/browser/visual_comparer.py

@@ -0,0 +1,159 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""VisualComparer – pixel-level screenshot comparison utility.
+
+Provides pure functions and a dataclass for comparing PNG screenshots
+against stored baselines to detect unintended visual regressions.
+
+Pillow (``PIL``) is imported **lazily** inside :func:`compare_screenshots`
+so the module can be imported without Pillow installed.
+"""
+
+from __future__ import annotations
+
+import io
+import os
+from dataclasses import dataclass
+
+
+@dataclass
+class VisualComparisonResult:
+    """Result of comparing two screenshots.
+
+    Attributes
+    ----------
+    matched:
+        ``True`` if ``diff_percentage <= threshold``.
+    diff_percentage:
+        Percentage of pixels that differ (0.0–100.0).
+    diff_image:
+        PNG bytes of the diff visualisation (differing pixels highlighted in
+        magenta), or ``None`` if sizes differed.
+    baseline_size:
+        ``(width, height)`` of the baseline image.
+    actual_size:
+        ``(width, height)`` of the actual screenshot.
+    threshold:
+        The diff-percentage threshold used for the comparison.
+    """
+
+    matched: bool
+    diff_percentage: float
+    diff_image: bytes | None
+    baseline_size: tuple[int, int]
+    actual_size: tuple[int, int]
+    threshold: float
+
+
+async def compare_screenshots(
+    baseline: bytes,
+    actual: bytes,
+    threshold: float = 0.1,
+) -> VisualComparisonResult:
+    """Compare two PNG screenshots pixel-by-pixel using Pillow.
+
+    Parameters
+    ----------
+    baseline:
+        PNG bytes of the stored baseline image.
+    actual:
+        PNG bytes of the newly captured screenshot.
+    threshold:
+        Maximum allowed ``diff_percentage`` for the comparison to be
+        considered *matched* (default 0.1 %).
+
+    Returns
+    -------
+    VisualComparisonResult
+        Detailed comparison outcome including diff image bytes.
+    """
+    from PIL import Image  # lazy import — Pillow optional at module level
+
+    baseline_img = Image.open(io.BytesIO(baseline)).convert("RGB")
+    actual_img = Image.open(io.BytesIO(actual)).convert("RGB")
+
+    baseline_size: tuple[int, int] = baseline_img.size  # (width, height)
+    actual_size: tuple[int, int] = actual_img.size
+
+    if baseline_size != actual_size:
+        return VisualComparisonResult(
+            matched=False,
+            diff_percentage=100.0,
+            diff_image=None,
+            baseline_size=baseline_size,
+            actual_size=actual_size,
+            threshold=threshold,
+        )
+
+    width, height = baseline_size
+    total_pixels = width * height
+
+    baseline_pixels = list(baseline_img.get_flattened_data())
+    actual_pixels = list(actual_img.get_flattened_data())
+
+    diff_img = actual_img.copy()
+    diff_pixels = list(diff_img.get_flattened_data())
+
+    differing = 0
+    for idx, (bp, ap) in enumerate(zip(baseline_pixels, actual_pixels)):
+        r1, g1, b1 = bp[0], bp[1], bp[2]
+        r2, g2, b2 = ap[0], ap[1], ap[2]
+        if abs(r1 - r2) + abs(g1 - g2) + abs(b1 - b2) > 30:
+            differing += 1
+            diff_pixels[idx] = (255, 0, 255)  # magenta highlight
+
+    diff_img.putdata(diff_pixels)  # type: ignore[arg-type]
+
+    diff_percentage = (differing / total_pixels) * 100.0
+    matched = diff_percentage <= threshold
+
+    buf = io.BytesIO()
+    diff_img.save(buf, format="PNG")
+    diff_image_bytes = buf.getvalue()
+
+    return VisualComparisonResult(
+        matched=matched,
+        diff_percentage=diff_percentage,
+        diff_image=diff_image_bytes,
+        baseline_size=baseline_size,
+        actual_size=actual_size,
+        threshold=threshold,
+    )
+
+
+def save_baseline(path: str, screenshot: bytes) -> None:
+    """Write PNG bytes to *path*, creating parent directories as needed.
+
+    Parameters
+    ----------
+    path:
+        Absolute or relative filesystem path for the baseline PNG.
+    screenshot:
+        Raw PNG bytes to persist.
+    """
+    os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
+    with open(path, "wb") as fh:
+        fh.write(screenshot)
+
+
+def load_baseline(path: str) -> bytes | None:
+    """Read PNG bytes from *path*, returning ``None`` if the file is absent.
+
+    Parameters
+    ----------
+    path:
+        Filesystem path of the baseline PNG.
+
+    Returns
+    -------
+    bytes or None
+        Raw PNG bytes, or ``None`` if the file does not exist.
+    """
+    try:
+        with open(path, "rb") as fh:
+            return fh.read()
+    except FileNotFoundError:
+        return None

mannf/core/diagnostics/__init__.py

@@ -0,0 +1,28 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""NAT Diagnostics package (Phase 6.5).
+
+Provides intelligent failure analysis:
+
+- :mod:`~mannf.core.diagnostics.failure_clusterer` — groups failures by error
+  signature similarity using TF-IDF + cosine similarity.
+- :mod:`~mannf.core.diagnostics.flake_detector` — tracks pass/fail history per
+  scenario across runs and auto-labels flaky scenarios.
+- :mod:`~mannf.core.diagnostics.root_cause_analyzer` — calls the configured LLM
+  to suggest root causes for new failures.
+"""
+
+from __future__ import annotations
+
+from mannf.core.diagnostics.failure_clusterer import FailureClusterer
+from mannf.core.diagnostics.flake_detector import FlakeDetector
+from mannf.core.diagnostics.root_cause_analyzer import RootCauseAnalyzer
+
+__all__ = [
+    "FailureClusterer",
+    "FlakeDetector",
+    "RootCauseAnalyzer",
+]

mannf/core/diagnostics/failure_clusterer.py

@@ -0,0 +1,211 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Failure clustering by error-message similarity (Phase 6.5).
+
+Groups a list of failure dicts — each with at minimum an ``"error"`` key —
+into clusters whose members share a similar error signature.
+
+Algorithm
+---------
+1. Tokenise each error message into a bag-of-words (lowercased, stripped of
+   punctuation, deduplicated per message — effectively a binary term vector).
+2. Compute pairwise cosine similarity between the term vectors.
+3. Single-pass greedy merge: start a new cluster for each failure whose
+   similarity to every existing cluster centroid is below *threshold*.  The
+   cluster centroid is the member that is most similar to all other members
+   (the "medoid").
+
+This is intentionally lightweight — no scipy / sklearn dependency.  For
+large corpora a proper TF-IDF matrix would be more accurate, but for the
+hundreds-of-failures scale typical of an autonomous run, this is sufficient.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import defaultdict
+from typing import Any, Dict, List, Optional, Tuple
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _tokenise(text: str) -> frozenset[str]:
+    """Lower-case, strip punctuation, split into unique words."""
+    return frozenset(re.split(r"\W+", text.lower())) - {""}
+
+
+def _cosine(a: frozenset[str], b: frozenset[str]) -> float:
+    """Binary (0/1) cosine similarity between two token sets."""
+    if not a or not b:
+        return 0.0
+    intersection = len(a & b)
+    return intersection / math.sqrt(len(a) * len(b))
+
+
+# ---------------------------------------------------------------------------
+# Public dataclass-style result
+# ---------------------------------------------------------------------------
+
+
+class FailureCluster:
+    """A group of failures sharing a similar error signature.
+
+    Attributes
+    ----------
+    cluster_id : str
+        Stable identifier ``"cluster-0"``, ``"cluster-1"``, …
+    label : str
+        Auto-generated human-readable description derived from the most
+        common tokens in the cluster error messages.
+    failures : list[dict]
+        The original failure dicts that belong to this cluster.
+    representative_error : str
+        The error message of the *medoid* — the member most similar to all
+        other members.
+    pages : list[str]
+        Unique ``page`` keys across cluster members.
+    size : int
+        Number of failures in this cluster.
+    """
+
+    def __init__(
+        self,
+        cluster_id: str,
+        failures: List[Dict[str, Any]],
+        representative_error: str,
+        label: str,
+    ) -> None:
+        self.cluster_id = cluster_id
+        self.failures = failures
+        self.representative_error = representative_error
+        self.label = label
+        self.pages: List[str] = sorted(
+            {f.get("page", f.get("url", "")) for f in failures if f.get("page") or f.get("url")}
+        )
+        self.size = len(failures)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "cluster_id": self.cluster_id,
+            "label": self.label,
+            "representative_error": self.representative_error,
+            "pages": self.pages,
+            "size": self.size,
+            "failures": self.failures,
+        }
+
+
+# ---------------------------------------------------------------------------
+# Clusterer
+# ---------------------------------------------------------------------------
+
+
+class FailureClusterer:
+    """Cluster failure dicts by error-message similarity.
+
+    Parameters
+    ----------
+    threshold : float
+        Minimum cosine similarity (0–1) required to merge a failure into an
+        existing cluster.  Default ``0.4`` works well for typical error
+        messages.
+    """
+
+    def __init__(self, threshold: float = 0.4) -> None:
+        if not 0.0 < threshold <= 1.0:
+            raise ValueError("threshold must be in (0, 1]")
+        self.threshold = threshold
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def cluster(self, failures: List[Dict[str, Any]]) -> List[FailureCluster]:
+        """Group *failures* into clusters and return them.
+
+        Parameters
+        ----------
+        failures:
+            List of failure dicts, each expected to have at least an
+            ``"error"`` key (empty string is tolerated).
+
+        Returns
+        -------
+        list[FailureCluster]
+            Clusters sorted by descending size.
+        """
+        if not failures:
+            return []
+
+        # Each bucket: list of (failure_dict, token_set) pairs
+        buckets: List[List[Tuple[Dict[str, Any], frozenset[str]]]] = []
+
+        for f in failures:
+            tokens = _tokenise(f.get("error", ""))
+            placed = False
+            for bucket in buckets:
+                medoid_tokens = bucket[0][1]  # first member as centroid approx
+                if _cosine(tokens, medoid_tokens) >= self.threshold:
+                    bucket.append((f, tokens))
+                    placed = True
+                    break
+            if not placed:
+                buckets.append([(f, tokens)])
+
+        clusters: List[FailureCluster] = []
+        for idx, bucket in enumerate(buckets):
+            representative, rep_label = self._pick_representative(bucket)
+            cluster = FailureCluster(
+                cluster_id=f"cluster-{idx}",
+                failures=[pair[0] for pair in bucket],
+                representative_error=representative,
+                label=rep_label,
+            )
+            clusters.append(cluster)
+
+        # Sort by descending cluster size
+        clusters.sort(key=lambda c: c.size, reverse=True)
+        return clusters
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _pick_representative(
+        self, bucket: List[Tuple[Dict[str, Any], frozenset[str]]]
+    ) -> Tuple[str, str]:
+        """Return (representative_error, human_label) for the cluster."""
+        if len(bucket) == 1:
+            error = bucket[0][0].get("error", "")
+            return error, self._label_from_error(error)
+
+        # Pick medoid: member with highest average similarity to all others
+        best_idx = 0
+        best_score = -1.0
+        for i, (_, tokens_i) in enumerate(bucket):
+            total = sum(
+                _cosine(tokens_i, tokens_j) for j, (_, tokens_j) in enumerate(bucket) if i != j
+            )
+            avg = total / (len(bucket) - 1)
+            if avg > best_score:
+                best_score = avg
+                best_idx = i
+
+        error = bucket[best_idx][0].get("error", "")
+        return error, self._label_from_error(error)
+
+    @staticmethod
+    def _label_from_error(error: str) -> str:
+        """Create a short human-readable label from an error message."""
+        if not error:
+            return "Unknown error"
+        # Take the first ~80 chars, stripped of excess whitespace
+        short = " ".join(error.split())[:80]
+        return short if short else "Unknown error"

mannf/core/diagnostics/flake_detector.py

@@ -0,0 +1,233 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Flakiness detection across autonomous runs (Phase 6.5).
+
+Tracks pass/fail history per ``(page_key, scenario_id)`` pair across multiple
+autonomous runs.  A scenario is labelled **flaky** if it alternates pass/fail
+in 3 or more consecutive runs.
+
+Integration points
+------------------
+* **Delta alerting (Phase 6.3)**: call :meth:`FlakeDetector.is_flaky` before
+  dispatching regression alerts to suppress noise from known-flaky scenarios.
+* **Bug filing (Phase 6.4)**: call :meth:`FlakeDetector.is_flaky` before
+  creating tickets to exclude flaky failures.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict, deque
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+_FLAKY_WINDOW = 5     # look-back window of runs to assess flakiness
+_FLAKY_MIN_ALTERNATIONS = 2  # minimum pass/fail flip-flops within the window
+
+
+def _count_alternations(history: deque) -> int:
+    """Count how many consecutive pairs differ in outcome (pass vs fail)."""
+    if len(history) < 2:
+        return 0
+    alternations = 0
+    lst = list(history)
+    for i in range(1, len(lst)):
+        if lst[i] != lst[i - 1]:
+            alternations += 1
+    return alternations
+
+
+# ---------------------------------------------------------------------------
+# Public dataclass-style result
+# ---------------------------------------------------------------------------
+
+
+class FlakeSummary:
+    """Flakiness statistics for a single ``(page_key, scenario_id)`` pair.
+
+    Attributes
+    ----------
+    page_key : str
+    scenario_id : str
+    is_flaky : bool
+    flake_rate : float
+        ``flake_count / total_runs`` ratio.
+    total_runs : int
+    pass_count : int
+    fail_count : int
+    alternations : int
+        Number of pass/fail flip-flops in the observation window.
+    """
+
+    def __init__(
+        self,
+        page_key: str,
+        scenario_id: str,
+        history: deque,
+    ) -> None:
+        self.page_key = page_key
+        self.scenario_id = scenario_id
+        total = len(history)
+        passes = sum(1 for v in history if v)
+        fails = total - passes
+        alts = _count_alternations(history)
+
+        self.total_runs = total
+        self.pass_count = passes
+        self.fail_count = fails
+        self.alternations = alts
+        self.is_flaky = total >= 3 and alts >= _FLAKY_MIN_ALTERNATIONS
+        self.flake_rate = round(fails / total, 4) if total > 0 else 0.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "page_key": self.page_key,
+            "scenario_id": self.scenario_id,
+            "is_flaky": self.is_flaky,
+            "flake_rate": self.flake_rate,
+            "total_runs": self.total_runs,
+            "pass_count": self.pass_count,
+            "fail_count": self.fail_count,
+            "alternations": self.alternations,
+        }
+
+
+# ---------------------------------------------------------------------------
+# Detector
+# ---------------------------------------------------------------------------
+
+
+class FlakeDetector:
+    """Stateful, in-process flakiness tracker.
+
+    Maintains a rolling history of pass/fail outcomes per
+    ``(page_key, scenario_id)`` across autonomous runs.
+
+    Parameters
+    ----------
+    window : int
+        Maximum number of recent runs to retain per scenario.  Default ``5``.
+    min_alternations : int
+        Minimum pass/fail flip-flops within the window to label a scenario
+        flaky.  Default ``2`` (i.e. at least 3 runs and 2 flips).
+    """
+
+    def __init__(self, window: int = _FLAKY_WINDOW, min_alternations: int = _FLAKY_MIN_ALTERNATIONS) -> None:
+        self._window = window
+        self._min_alternations = min_alternations
+        # key → deque of booleans (True = passed, False = failed)
+        self._history: Dict[Tuple[str, str], deque] = defaultdict(
+            lambda: deque(maxlen=self._window)
+        )
+
+    # ------------------------------------------------------------------
+    # Recording outcomes
+    # ------------------------------------------------------------------
+
+    def record(
+        self,
+        page_key: str,
+        scenario_id: str,
+        passed: bool,
+    ) -> None:
+        """Record a single scenario outcome.
+
+        Parameters
+        ----------
+        page_key:
+            Page path (e.g. ``"/checkout"``).
+        scenario_id:
+            Scenario identifier (task_id or similar).
+        passed:
+            ``True`` if the scenario passed, ``False`` if it failed.
+        """
+        key = (page_key, scenario_id)
+        self._history[key].append(passed)
+
+    def record_run_results(self, task_details: List[Dict[str, Any]]) -> None:
+        """Bulk-record results from a single loop iteration.
+
+        Each dict in *task_details* should have:
+        - ``"url"`` or ``"page"`` — page key
+        - ``"task_id"`` — scenario identifier
+        - ``"passed"`` — bool
+        """
+        for t in task_details:
+            page_key = t.get("page") or t.get("url", "")
+            scenario_id = t.get("task_id", "")
+            passed = bool(t.get("passed", False))
+            if page_key and scenario_id:
+                self.record(page_key, scenario_id, passed)
+
+    # ------------------------------------------------------------------
+    # Querying
+    # ------------------------------------------------------------------
+
+    def is_flaky(self, page_key: str, scenario_id: str) -> bool:
+        """Return ``True`` if the scenario is currently labelled flaky."""
+        return self.get_summary(page_key, scenario_id).is_flaky
+
+    def get_summary(self, page_key: str, scenario_id: str) -> FlakeSummary:
+        """Return :class:`FlakeSummary` for a specific scenario."""
+        history = self._history[(page_key, scenario_id)]
+        return FlakeSummary(page_key, scenario_id, history)
+
+    def all_summaries(self) -> List[FlakeSummary]:
+        """Return :class:`FlakeSummary` for every tracked scenario."""
+        return [
+            FlakeSummary(pk, sid, hist)
+            for (pk, sid), hist in self._history.items()
+        ]
+
+    def flaky_summaries(self) -> List[FlakeSummary]:
+        """Return only the summaries labelled flaky."""
+        return [s for s in self.all_summaries() if s.is_flaky]
+
+    def flaky_scenario_keys(self) -> Set[Tuple[str, str]]:
+        """Return the set of ``(page_key, scenario_id)`` keys labelled flaky."""
+        return {(s.page_key, s.scenario_id) for s in self.flaky_summaries()}
+
+    def should_suppress(
+        self,
+        page_key: str,
+        scenario_id: str,
+        suppress_flaky: bool = True,
+    ) -> bool:
+        """Return ``True`` if a failure should be suppressed from alerts/filing.
+
+        Parameters
+        ----------
+        suppress_flaky:
+            When ``True`` (default), flaky scenarios are suppressed.
+        """
+        if not suppress_flaky:
+            return False
+        return self.is_flaky(page_key, scenario_id)
+
+    # ------------------------------------------------------------------
+    # Serialisation
+    # ------------------------------------------------------------------
+
+    def export_state(self) -> Dict[str, Any]:
+        """Export tracker state for persistence / debugging."""
+        return {
+            f"{pk}::{sid}": list(hist)
+            for (pk, sid), hist in self._history.items()
+        }
+
+    def import_state(self, state: Dict[str, Any]) -> None:
+        """Restore tracker state from a previously exported snapshot."""
+        for combined_key, history_list in state.items():
+            if "::" in combined_key:
+                pk, sid = combined_key.split("::", 1)
+            else:
+                pk, sid = combined_key, ""
+            dq: deque = deque(maxlen=self._window)
+            dq.extend(history_list)
+            self._history[(pk, sid)] = dq