catchbench 0.1.0__py3-none-any.whl

catchbench/__init__.py

@@ -0,0 +1,12 @@
+"""CatchBench: the benchmark for agent auditing across the PRE / LIVE / POST lifecycle.
+
+The public arena where methods compete at finding and attributing agent failures over real
+agent traces. Organized along the agent lifecycle (PRE deploy gate, LIVE real-time, POST
+forensics); each pillar holds scenarios, each scenario is one ``Task`` with its own labels,
+metric, and baseline set. The shared ``Task`` / ``Method`` contract keeps those inputs fixed
+while each compatible method supplies its own evaluation logic.
+"""
+from catchbench.core import Method, ResultRow, RunPipeline, Task
+
+__all__ = ["Method", "ResultRow", "RunPipeline", "Task"]
+__version__ = "0.0.1"

catchbench/_reuse.py

@@ -0,0 +1,109 @@
+"""Resolve GRADE's experiment modules and the ``auditable`` dependency.
+
+Installed modules take precedence. GRADE's experiment modules are not included in its wheel, so
+an editable sibling checkout or ``GRADE_DIR`` is used as a fallback. ``AUDITABLE_DIR`` remains a
+development fallback for ``auditable`` when the declared package dependency is unavailable.
+Importing this module may add fallback checkout directories to ``sys.path``.
+"""
+from __future__ import annotations
+
+import importlib.util
+import logging
+import os
+import sys
+from pathlib import Path
+
+from catchbench.corpora import install_hub_revision_pins
+
+
+install_hub_revision_pins()
+
+
+_LOGGER = logging.getLogger(__name__)
+_REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
+_GRADE_MODULES = (
+    "grade",
+    "agent_failure_detection",
+    "agent_failure_localization",
+    "agent_graph_characterization",
+    "agent_graph_swegym",
+    "agent_graph_tau_bench",
+)
+
+
+def _module_location(name: str) -> str | None:
+    try:
+        spec = importlib.util.find_spec(name)
+    except (AttributeError, ImportError, ValueError):
+        return None
+    if spec is None:
+        return None
+    return spec.origin or next(iter(spec.submodule_search_locations or ()), None)
+
+
+def _prepend(paths: tuple[Path, ...]) -> None:
+    for path in reversed(paths):
+        value = str(path)
+        if value not in sys.path:
+            sys.path.insert(0, value)
+
+
+def _resolve_grade() -> str:
+    locations = [_module_location(name) for name in _GRADE_MODULES]
+    if all(locations):
+        source = f"installed modules ({locations[0]})"
+        _LOGGER.debug("Using GRADE from %s", source)
+        return source
+
+    configured = os.environ.get("GRADE_DIR")
+    checkout = Path(configured).expanduser() if configured else _REPOSITORY_ROOT.parent / "grade"
+    paths = (checkout / "experiment", checkout / "src")
+    if all(path.is_dir() for path in paths):
+        _prepend(paths)
+        if all(_module_location(name) for name in _GRADE_MODULES):
+            source = f"checkout bridge ({checkout})"
+            _LOGGER.debug("Using GRADE from %s", source)
+            return source
+
+    raise ImportError(
+        "CatchBench needs GRADE's experiment modules, which are not distributed in GRADE's "
+        "wheel. Clone GRADE next to this repository and install its experiment dependencies:\n"
+        "  git clone https://github.com/yzhao062/grade.git ../grade\n"
+        "  python -m pip install -e \"../grade[experiments]\"\n"
+        "Alternatively, set GRADE_DIR to the GRADE checkout. The checkout must contain "
+        "experiment/ and src/."
+    )
+
+
+def _resolve_auditable() -> str:
+    location = _module_location("auditable")
+    if location:
+        source = f"installed package ({location})"
+        _LOGGER.debug("Using auditable from %s", source)
+        return source
+
+    configured = os.environ.get("AUDITABLE_DIR")
+    if configured:
+        path = Path(configured).expanduser() / "src"
+        if path.is_dir():
+            _prepend((path,))
+            location = _module_location("auditable")
+            if location:
+                source = f"checkout bridge ({path.parent})"
+                _LOGGER.debug("Using auditable from %s", source)
+                return source
+
+    raise ImportError(
+        "CatchBench requires auditable>=0.2.0. Install it with "
+        "`python -m pip install \"auditable>=0.2.0\"`, or set AUDITABLE_DIR to a source "
+        "checkout containing src/auditable."
+    )
+
+
+SOURCES = {
+    "auditable": _resolve_auditable(),
+    "grade": _resolve_grade(),
+}
+
+# GRADE's localization eval uses a conda BLAS stack that wants this set (matches the seed).
+os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE")

catchbench/agent_detectors.py

@@ -0,0 +1,180 @@
+"""Agent-specific published detectors as benchmark baselines: GUARDIAN and G-Safeguard.
+
+Both are run on the dependency-graph substrate so they compete on the same data as every other method,
+and both are implemented as their core mechanism with the simplifications noted, in the ADBench / BOND
+tradition of porting a published method onto the benchmark's representation rather than gesturing at it.
+
+  - GUARDIAN (Zhou et al., 2025, arXiv:2505.19234) safeguards multi-agent collaboration with an
+    UNSUPERVISED reconstruction autoencoder over a temporal attributed graph: it reconstructs node
+    attributes and structure and scores a node by reconstruction error. Here it is a directed-GCN
+    attribute-reconstruction autoencoder over the per-run dependency graph (the temporal direction is
+    the step -> dependency edge); a node it reconstructs poorly is anomalous, and a run's failure score
+    is its mean node error. The explicit adjacency-reconstruction term and the Information-Bottleneck
+    compression are simplified to an attribute-reconstruction objective that stays structure-aware
+    through message passing. Unsupervised: it never sees the failure label.
+
+  - G-Safeguard (Wang et al., 2025, arXiv:2502.11127) detects injected / anomalous agents with a
+    SUPERVISED GNN over the multi-agent utterance graph and then remediates topologically. Here it is a
+    supervised graph-classification GNN (GCN layers, mean pooling, a linear head) trained with
+    seed-averaged stratified cross-validation to predict run failure from the dependency graph. The
+    topological remediation step is out of scope because the board scores detection, not intervention.
+
+Both train on CPU with fixed seeds for reproducibility, the same posture as the PyGOD baseline.
+"""
+from __future__ import annotations
+
+import os
+
+os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE")
+
+from typing import List, Sequence, Tuple
+
+import numpy as np
+
+Graph = Tuple[np.ndarray, np.ndarray]  # (node features [n, d], edge_index [2, m])
+
+
+def _batched(graphs: Sequence[Graph]):
+    """Standardize node features across all graphs and pack them into one disconnected PyG batch,
+    self-looping isolated nodes (sparse dependency graphs have many) so message passing is defined."""
+    import torch
+    from torch_geometric.data import Batch, Data
+    from sklearn.preprocessing import StandardScaler
+
+    sizes = [len(np.asarray(x)) for x, _ in graphs]
+    stacked = StandardScaler().fit_transform(
+        np.vstack([np.asarray(x, dtype=float) for x, _ in graphs]))
+    datas, offset = [], 0
+    for (x, edges), n in zip(graphs, sizes):
+        node_x = torch.tensor(stacked[offset:offset + n], dtype=torch.float)
+        offset += n
+        e = np.asarray(edges, dtype=np.int64).reshape(2, -1)
+        touched = set(e.flatten().tolist()) if e.shape[1] else set()
+        isolated = [v for v in range(n) if v not in touched]
+        if isolated:
+            loops = np.array(isolated, dtype=np.int64)
+            e = np.concatenate([e, np.stack([loops, loops])], axis=1)
+        datas.append(Data(x=node_x, edge_index=torch.tensor(e, dtype=torch.long)))
+    return Batch.from_data_list(datas), len(datas)
+
+
+def guardian_run_scores(graphs: Sequence[Graph], *, seed: int = 0, hid: int = 16,
+                        epochs: int = 50) -> np.ndarray:
+    """GUARDIAN's unsupervised reconstruction-AE core: fit a directed-GCN attribute autoencoder over
+    the run graphs and return each run's mean per-node reconstruction error (higher = more anomalous).
+    Seed-averaged over a few inits, because a single GNN init is noisy at this graph scale."""
+    import torch
+    from torch_geometric.nn import GCNConv
+
+    batch, n_runs = _batched(graphs)
+    in_dim = batch.x.shape[1]
+
+    class _AE(torch.nn.Module):
+        def __init__(self) -> None:
+            super().__init__()
+            self.enc1 = GCNConv(in_dim, hid * 2)
+            self.enc2 = GCNConv(hid * 2, hid)          # latent bottleneck (the IB nod)
+            self.dec1 = GCNConv(hid, hid * 2)
+            self.dec2 = GCNConv(hid * 2, in_dim)       # reconstruct attributes
+
+        def forward(self, x, edge_index):
+            z = self.enc2(torch.relu(self.enc1(x, edge_index)), edge_index)
+            x_hat = self.dec2(torch.relu(self.dec1(z, edge_index)), edge_index)
+            return x_hat
+
+    per_seed = []
+    for s in range(3):
+        torch.manual_seed(seed + s)
+        model = _AE()
+        opt = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4)
+        model.train()
+        for _ in range(epochs):
+            opt.zero_grad()
+            x_hat = model(batch.x, batch.edge_index)
+            loss = ((batch.x - x_hat) ** 2).mean()
+            loss.backward()
+            opt.step()
+        model.eval()
+        with torch.no_grad():
+            x_hat = model(batch.x, batch.edge_index)
+            node_err = ((batch.x - x_hat) ** 2).mean(dim=1).numpy()
+        per_seed.append(np.array([node_err[batch.batch.numpy() == k].mean() for k in range(n_runs)]))
+    return np.mean(per_seed, axis=0)
+
+
+def _to_data(graph: Graph, scaler):
+    """One PyG Data from a raw graph, node features transformed by an already-fit scaler, isolated
+    nodes self-looped (matching ``_batched``) so message passing is defined."""
+    import torch
+    from torch_geometric.data import Data
+
+    x, edges = graph
+    node_x = torch.tensor(scaler.transform(np.asarray(x, dtype=float)), dtype=torch.float)
+    n = node_x.shape[0]
+    e = np.asarray(edges, dtype=np.int64).reshape(2, -1)
+    touched = set(e.flatten().tolist()) if e.shape[1] else set()
+    isolated = [v for v in range(n) if v not in touched]
+    if isolated:
+        loops = np.array(isolated, dtype=np.int64)
+        e = np.concatenate([e, np.stack([loops, loops])], axis=1)
+    return Data(x=node_x, edge_index=torch.tensor(e, dtype=torch.long))
+
+
+def gsafeguard_cv_auc(graphs: Sequence[Graph], y: np.ndarray, *, seed: int = 0, hid: int = 32,
+                      epochs: int = 60, n_splits: int = 5) -> float:
+    """G-Safeguard's supervised-GNN detector: a graph-classification GCN trained with seed-averaged
+    stratified K-fold cross-validation to predict run failure, returning the mean held-out ROC-AUC.
+    Grouped at the run, since each run is one graph and one label. The feature scaler is fit on each
+    fold's training graphs only, so the held-out fold never informs preprocessing (no train/test leak)."""
+    import torch
+    from torch_geometric.data import Batch
+    from torch_geometric.nn import GCNConv, global_mean_pool
+    from sklearn.metrics import roc_auc_score
+    from sklearn.model_selection import StratifiedKFold
+    from sklearn.preprocessing import StandardScaler
+
+    y = np.asarray(y)
+    n_runs = len(graphs)
+    if n_runs < n_splits or len(set(y.tolist())) < 2:
+        return 0.5
+    in_dim = np.asarray(graphs[0][0]).shape[1]
+
+    class _GNN(torch.nn.Module):
+        def __init__(self) -> None:
+            super().__init__()
+            self.c1 = GCNConv(in_dim, hid)
+            self.c2 = GCNConv(hid, hid)
+            self.lin = torch.nn.Linear(hid, 1)
+
+        def forward(self, b):
+            h = torch.relu(self.c1(b.x, b.edge_index))
+            h = torch.relu(self.c2(h, b.edge_index))
+            return self.lin(global_mean_pool(h, b.batch)).squeeze(-1)
+
+    aucs = []
+    for s in range(3):  # seed-average for a stable held-out estimate
+        skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed + s)
+        for tr, te in skf.split(np.zeros(n_runs), y):
+            if len(set(y[tr].tolist())) < 2 or len(set(y[te].tolist())) < 2:
+                continue  # skip degenerate single-class folds
+            scaler = StandardScaler().fit(  # train folds only: no held-out stats leak in
+                np.vstack([np.asarray(graphs[int(i)][0], dtype=float) for i in tr]))
+            train_batch = Batch.from_data_list([_to_data(graphs[int(i)], scaler) for i in tr])
+            test_batch = Batch.from_data_list([_to_data(graphs[int(i)], scaler) for i in te])
+            train_y = torch.tensor(y[tr], dtype=torch.float)
+
+            torch.manual_seed(seed + s)
+            model = _GNN()
+            opt = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4)
+            loss_fn = torch.nn.BCEWithLogitsLoss()
+            model.train()
+            for _ in range(epochs):
+                opt.zero_grad()
+                loss = loss_fn(model(train_batch), train_y)
+                loss.backward()
+                opt.step()
+            model.eval()
+            with torch.no_grad():
+                prob = torch.sigmoid(model(test_batch)).numpy()
+            aucs.append(roc_auc_score(y[te], prob))
+    return float(np.mean(aucs)) if aucs else 0.5

catchbench/core.py

@@ -0,0 +1,95 @@
+"""The five-layer plug-in contract (core interfaces).
+
+One scenario is one ``Task``; a ``Method`` evaluates a ``Task``; ``RunPipeline`` runs every
+valid ``(Task, Method)`` pair and emits leaderboard rows. The POST localization seed runs
+through these interfaces, and LIVE and PRE Tasks plug into the same contract.
+
+Under this contract, a ``Method`` sees the whole ``Task`` and returns its metric dict.
+Supervised baselines prevent in-sample scoring by running out-of-sample cross-validation inside
+``evaluate``: grouped by run for step-level localization and stratified over runs for run-level
+detection, where each run is one row.
+"""
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Mapping, Protocol, Sequence, runtime_checkable
+
+
+@runtime_checkable
+class Task(Protocol):
+    """One benchmark scenario: its data, its labels, and its metric.
+
+    ``pillar`` is "PRE", "LIVE", or "POST"; ``granularity`` is "run", "step", "edge", or
+    "plan". ``setup`` loads the corpus and is idempotent.
+    """
+
+    task_id: str
+    pillar: str
+    granularity: str
+    dataset: str
+
+    def setup(self) -> None: ...
+
+
+@runtime_checkable
+class Method(Protocol):
+    """A leaderboard entry. ``supports`` lists the ``task_id``s it can run."""
+
+    method_id: str
+    supports: set
+
+    def evaluate(self, task: Task) -> Mapping[str, float]: ...
+
+
+@dataclass(frozen=True)
+class ResultRow:
+    """One method's score on one task."""
+
+    pillar: str
+    task: str
+    dataset: str
+    method: str
+    metrics: Mapping[str, float]
+
+
+class RunPipeline:
+    """Run every valid ``(Task, Method)`` pair and collect leaderboard rows."""
+
+    def __init__(self, tasks: Sequence[Task], methods: Sequence[Method]) -> None:
+        self.tasks = list(tasks)
+        self.methods = list(methods)
+
+    def run(self) -> list[ResultRow]:
+        rows: list[ResultRow] = []
+        for task in self.tasks:
+            for method in self.methods:
+                if task.task_id not in method.supports:
+                    continue  # only valid (Task, Method) pairs run
+                rows.append(
+                    ResultRow(
+                        task.pillar, task.task_id, getattr(task, "dataset", ""),
+                        method.method_id, method.evaluate(task),
+                    )
+                )
+        return rows
+
+    @staticmethod
+    def leaderboard(rows: Sequence[ResultRow]) -> str:
+        """A text leaderboard, one block per task, methods in input order."""
+        by_task: dict[tuple[str, str, str], list[ResultRow]] = defaultdict(list)
+        for row in rows:
+            by_task[(row.pillar, row.task, row.dataset)].append(row)
+        out: list[str] = []
+        for (pillar, task, dataset), task_rows in by_task.items():
+            metric_keys = list(task_rows[0].metrics.keys())
+            label = f"{task} :: {dataset}" if dataset else task
+            mw = max([len("method")] + [len(row.method) for row in task_rows])  # fit the widest id
+            out.append(f"\n[{pillar}] {label}")
+            out.append(f"  {'method':{mw}s}" + "".join(f"{k:>10s}" for k in metric_keys))
+            for row in task_rows:
+                out.append(
+                    f"  {row.method:{mw}s}"
+                    + "".join(f"{row.metrics[k]:>10.3f}" for k in metric_keys)
+                )
+        return "\n".join(out)

catchbench/corpora.py

@@ -0,0 +1,158 @@
+"""Immutable Hugging Face revisions used by the POST and LIVE boards."""
+from __future__ import annotations
+
+import functools
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class CorpusRevision:
+    name: str
+    repo_id: str
+    revision: str
+
+
+CORPUS_REVISIONS = (
+    CorpusRevision(
+        "Who&When",
+        "Kevin355/Who_and_When",
+        "59b9fcba1aaed7bbf206b5f4d3c68b8face2f49c",
+    ),
+    CorpusRevision(
+        "SWE-Gym",
+        "SWE-Gym/OpenHands-Sampled-Trajectories",
+        "baf3a4e4bff514d48ddc08a93a2ade5c126212c7",
+    ),
+    CorpusRevision(
+        "tau-bench",
+        "AgentSuite/tau-bench-trajectories",
+        "382e57d1784b55c5155f4ef394ef48f1c747a287",
+    ),
+)
+
+_BY_REPO = {corpus.repo_id: corpus for corpus in CORPUS_REVISIONS}
+_PINNED_FETCHES: set[tuple[str, str]] = set()
+_PATCHED = False
+
+
+class CorpusRevisionError(RuntimeError):
+    """The runner cannot prove that it will score the recorded corpus revisions."""
+
+
+def verify_corpus_heads(api: Any = None) -> dict[str, str]:
+    """Resolve every dataset head and refuse to score if it differs from the board record."""
+    if api is None:
+        try:
+            from huggingface_hub import HfApi
+        except ImportError as exc:  # pragma: no cover - exercised by minimal installations
+            raise CorpusRevisionError(
+                "Corpus revision preflight needs huggingface_hub; install the dev-seed or full extra."
+            ) from exc
+        api = HfApi()
+
+    resolved: dict[str, str] = {}
+    errors = []
+    for corpus in CORPUS_REVISIONS:
+        try:
+            actual = str(api.dataset_info(corpus.repo_id, revision="main").sha)
+        except Exception as exc:
+            errors.append(
+                f"{corpus.name} ({corpus.repo_id}): could not resolve main ({exc}); "
+                f"recorded={corpus.revision}"
+            )
+            continue
+        resolved[corpus.name] = actual
+        if actual != corpus.revision:
+            errors.append(
+                f"{corpus.name} ({corpus.repo_id}): main={actual}; recorded={corpus.revision}"
+            )
+    if errors:
+        raise CorpusRevisionError(
+            "Corpus revision preflight failed; refusing to score an unrecorded population:\n  "
+            + "\n  ".join(errors)
+        )
+    return resolved
+
+
+def revision_header(revisions: dict[str, str] | None = None) -> str:
+    """One board-header line recording the exact population revisions being scored."""
+    revisions = revisions or {corpus.name: corpus.revision for corpus in CORPUS_REVISIONS}
+    cells = [f"{corpus.name}={revisions[corpus.name]}" for corpus in CORPUS_REVISIONS]
+    return "Corpus revisions :: " + " | ".join(cells)
+
+
+def _pin_hub_call(name: str, function):
+    @functools.wraps(function)
+    def pinned(*args, **kwargs):
+        import inspect
+
+        bound = inspect.signature(function).bind_partial(*args, **kwargs)
+        repo_id = bound.arguments.get("repo_id")
+        corpus = _BY_REPO.get(repo_id)
+        if corpus is None:
+            return function(*args, **kwargs)
+        requested = bound.arguments.get("revision")
+        if requested not in (None, "main", corpus.revision):
+            raise CorpusRevisionError(
+                f"{name} requested {repo_id}@{requested}, but the board records {corpus.revision}"
+            )
+        bound.arguments["revision"] = corpus.revision
+        result = function(*bound.args, **bound.kwargs)
+        _PINNED_FETCHES.add((repo_id, name))
+        return result
+
+    return pinned
+
+
+def install_hub_revision_pins() -> None:
+    """Force GRADE's Hub calls through the recorded revisions before its modules are imported."""
+    global _PATCHED
+    if _PATCHED:
+        return
+    try:
+        import huggingface_hub
+    except ImportError:  # the actionable error is emitted by the preflight or a GRADE loader
+        return
+    for name in ("snapshot_download", "hf_hub_download", "list_repo_files"):
+        function = getattr(huggingface_hub, name)
+        setattr(huggingface_hub, name, _pin_hub_call(name, function))
+    _PATCHED = True
+
+
+def _whoandwhen_cached_revisions() -> set[str]:
+    """Read the commit recorded by snapshot_download's local-dir metadata."""
+    try:
+        import agent_graph_characterization as whoandwhen
+    except ImportError:
+        return set()
+    metadata_root = Path(whoandwhen.CACHE).parent / ".cache" / "huggingface" / "download"
+    revisions = set()
+    for path in metadata_root.rglob("*.metadata"):
+        lines = path.read_text(encoding="utf-8").splitlines()
+        if lines:
+            revisions.add(lines[0])
+    return revisions
+
+
+def verify_pinned_fetches() -> None:
+    """Check that each loader used a pinned Hub call, or a verified Who&When local snapshot."""
+    errors = []
+    for corpus in CORPUS_REVISIONS:
+        calls = {name for repo_id, name in _PINNED_FETCHES if repo_id == corpus.repo_id}
+        if corpus.name == "Who&When" and not calls:
+            cached = _whoandwhen_cached_revisions()
+            if cached == {corpus.revision}:
+                continue
+            detail = ", ".join(sorted(cached)) if cached else "no snapshot metadata"
+            errors.append(f"Who&When local cache is not verified at {corpus.revision} ({detail})")
+        elif "hf_hub_download" not in calls and "snapshot_download" not in calls:
+            errors.append(
+                f"{corpus.name} loader made no observed pinned download for {corpus.repo_id}; "
+                "GRADE may have changed its fetch path"
+            )
+    if errors:
+        raise CorpusRevisionError(
+            "Corpus fetch verification failed; refusing to print scores:\n  " + "\n  ".join(errors)
+        )