graph-explain 0.7.0__py3-none-any.whl

graph_explain/core/benchmark.py

@@ -0,0 +1,284 @@
+from __future__ import annotations
+
+import torch
+
+from ..narration import summarize
+from .evaluation import (
+    evaluate_fidelity_minus,
+    evaluate_fidelity_plus,
+    evaluate_gea,
+    evaluate_gea_graph,
+    evaluate_sparsity,
+    evaluate_stability,
+)
+from .explainer import Explainer
+from .registry import get_algorithm, instantiate
+
+DEFAULT_METHODS = [
+    "gnn_explainer",
+    "pg_explainer",
+    "subgraphx",
+    "saliency",
+    "integrated_gradients",
+    "gnn_lrp",
+    "deep_lift",
+    "grad_x_input",
+    "graph_lime",
+    "node_mask",
+    "guided_backprop",
+    "random",
+    "counterfactual",
+    "attention",
+]
+
+_METRICS = (
+    "fidelity_plus",
+    "fidelity_minus",
+    "gea",
+    "sparsity",
+    "sparsity_local",
+    "stability",
+)
+
+
+def _has_gat(model) -> bool:
+    try:
+        from torch_geometric.nn import GATConv
+
+        return any(isinstance(m, GATConv) for m in model.modules())
+    except ImportError:
+        return False
+
+
+def _method_kwargs(epochs, lr, seed, top_k) -> dict:
+    kw: dict = {}
+    for name, value in (
+        ("epochs", epochs),
+        ("lr", lr),
+        ("seed", seed),
+        ("top_k", top_k),
+    ):
+        if value is not None:
+            kw[name] = value
+    return kw
+
+
+def compare(
+    data,
+    model,
+    node: int | None = None,
+    target_class: int | None = None,
+    backend: str = "pyg",
+    methods: list[str] | None = None,
+    top_k: int = 5,
+    num_perturbations: int = 5,
+    noise_std: float = 0.05,
+    epochs: int = 200,
+    lr: float = 0.01,
+    seed: int = 0,
+    mask_threshold: float = 0.5,
+    stability: bool = True,
+) -> dict:
+    """Runs several explanation methods and compares their metrics.
+
+    Uses `node` for node-level (required) or `node=None` for graph-level
+    (node-only methods are marked `skipped`). Returns a dictionary with one
+    entry per method: class, predictions, metrics (fidelity±, GEA, sparsity,
+    stability) and the structured `summarize` summary. Non-applicable methods
+    or failing metrics are marked as `skipped`/`None` without aborting the
+    rest.
+    """
+    from ..backends import get_backend
+
+    if methods is None:
+        methods = list(DEFAULT_METHODS)
+    backend_obj = get_backend(backend)
+    model.eval()
+
+    if node is None:
+        stability = False
+
+    torch.manual_seed(seed)
+    results: dict = {}
+    ran: list[str] = []
+    skipped: dict[str, str] = {}
+
+    for name in methods:
+        cls = get_algorithm(name)
+        entry = {
+            "method": name,
+            "class": cls.__name__,
+            "node": node,
+            "target_class": target_class,
+            "prediction_original": None,
+            "prediction_explanation": None,
+            "metrics": {m: None for m in _METRICS},
+            "summary": None,
+            "skipped": None,
+        }
+        if node is None and not cls.graph_level:
+            entry["skipped"] = "node-level only"
+            results[name] = entry
+            skipped[name] = entry["skipped"]
+            continue
+        if name == "attention" and not _has_gat(model):
+            entry["skipped"] = "requires a model with GATConv layers"
+            results[name] = entry
+            skipped[name] = entry["skipped"]
+            continue
+        algo = instantiate(name, **_method_kwargs(epochs, lr, seed, top_k))
+        explainer = Explainer(
+            algorithm=algo,
+            backend=backend_obj,
+            mask_threshold=mask_threshold,
+        )
+        torch.manual_seed(seed)
+        try:
+            expl = (
+                explainer.explain_node(data, model, node, target_class=target_class)
+                if node is not None
+                else explainer.explain_graph(data, model, target_class=target_class)
+            )
+        except (ValueError, TypeError) as exc:
+            entry["skipped"] = str(exc)
+            results[name] = entry
+            skipped[name] = str(exc)
+            continue
+
+        entry["prediction_original"] = _fmt(expl.prediction_original)
+        entry["prediction_explanation"] = _fmt(expl.prediction_explanation)
+        entry["summary"] = summarize(expl, data=data, top_k=top_k)
+
+        m = entry["metrics"]
+        expl_arg = expl
+        m["fidelity_plus"] = _safe(
+            lambda expl=expl_arg: float(evaluate_fidelity_plus(model, expl))
+        )
+        m["fidelity_minus"] = _safe(
+            lambda expl=expl_arg: float(evaluate_fidelity_minus(model, expl))
+        )
+        if node is not None:
+            m["gea"] = _safe(
+                lambda expl=expl_arg: float(evaluate_gea(expl, data=data, top_k=top_k))
+            )
+        else:
+            m["gea"] = _safe(
+                lambda expl=expl_arg: float(
+                    evaluate_gea_graph(expl, data=data, top_k=top_k)
+                )
+            )
+        m["sparsity"] = _safe(lambda expl=expl_arg: float(evaluate_sparsity(expl)))
+        m["sparsity_local"] = _safe(
+            lambda expl=expl_arg: float(evaluate_sparsity(expl, local=True))
+        )
+        if stability:
+
+            def _again(d, name=name):
+                algo_r = instantiate(name, **_method_kwargs(epochs, lr, seed, top_k))
+                return Explainer(
+                    algorithm=algo_r,
+                    backend=backend_obj,
+                    mask_threshold=mask_threshold,
+                ).explain_node(d, model, node)
+
+            m["stability"] = _safe(
+                lambda: float(
+                    evaluate_stability(
+                        _again,
+                        data,
+                        num_perturbations=num_perturbations,
+                        noise_std=noise_std,
+                        top_k=top_k,
+                    )
+                )
+            )
+        results[name] = entry
+        ran.append(name)
+
+    return {
+        "_meta": {
+            "node": node,
+            "target_class": target_class,
+            "backend": backend,
+            "methods": ran,
+            "skipped": skipped,
+        },
+        **{name: results[name] for name in methods},
+    }
+
+
+def report_html(results: dict, output_path: str) -> None:
+    """Builds a self-contained HTML report (comparative table)."""
+    meta = results["_meta"]
+    rows = []
+    for name, entry in results.items():
+        if name.startswith("_"):
+            continue
+        m = entry["metrics"]
+        if entry["skipped"]:
+            rows.append(
+                f"<tr><td>{name}</td>"
+                f"<td colspan='7' class='skip'>not applicable: {entry['skipped']}</td></tr>"
+            )
+            continue
+        cells = "".join(
+            f"<td>{'-' if m[k] is None else f'{m[k]:.4f}'}</td>"
+            for k in ("fidelity_plus", "fidelity_minus", "gea", "sparsity", "stability")
+        )
+        rows.append(
+            f"<tr><td>{name} <small>({entry['class']})</small></td>{cells}</tr>"
+        )
+
+    body = "\n".join(rows)
+    html = f"""<!doctype html>
+<html lang="es">
+<head>
+<meta charset="utf-8">
+<title>Benchmark - graph-explain</title>
+<style>
+  body {{ font-family: system-ui, sans-serif; margin: 2rem; }}
+  table {{ border-collapse: collapse; width: 100%; max-width: 900px; }}
+  th, td {{ border: 1px solid #ccc; padding: 6px 10px; text-align: right; }}
+  th {{ background: #f0f0f0; }}
+  td:first-child {{ text-align: left; }}
+  td.skip {{ text-align: left; color: #888; font-style: italic; }}
+  .meta {{ color: #555; margin-bottom: 1rem; }}
+  code {{ background: #f4f4f4; padding: 0 4px; }}
+</style>
+</head>
+<body>
+<h1>Comparative explanation benchmark</h1>
+<p class="meta">
+  node <code>{meta["node"]}</code> &middot; target class
+  <code>{meta["target_class"]}</code> &middot; backend <code>{meta["backend"]}</code>
+</p>
+<table>
+<tr>
+  <th>Method</th><th>fid+</th><th>fid-</th><th>GEA</th><th>sparsity</th><th>stability</th>
+</tr>
+{body}
+</table>
+<p class="meta">
+  Generated with <code>graph-explain</code>. fid+ = necessity (drop in P(c) after
+  removing top-k), fid- = sufficiency, GEA = overlap with ground truth, sparsity =
+  global sparsity, stability = mean similarity under perturbations.
+</p>
+</body>
+</html>"""
+    with open(output_path, "w", encoding="utf-8") as fh:
+        fh.write(html)
+
+
+def _fmt(value):
+    if value is None:
+        return None
+    if hasattr(value, "tolist"):
+        return [round(float(v), 4) for v in value.reshape(-1).tolist()]
+    return value
+
+
+def _safe(fn):
+    try:
+        return fn()
+    except Exception:  # noqa: BLE001
+        return None

graph_explain/core/evaluation.py

@@ -0,0 +1,391 @@
+from __future__ import annotations
+
+import math
+
+
+def _resolve_target(explanation) -> int:
+    target = explanation.target_class
+    if target is None:
+        import torch
+
+        pred = explanation.prediction_original
+        if torch.is_tensor(pred):
+            pred = pred.reshape(-1)
+            target = int(pred.argmax().item())
+        else:
+            target = 0
+    return int(target)
+
+
+def _mask_top_k(explanation, top_k: int | None, keep_ratio: float, kind: str):
+    importance = (
+        explanation.edge_importance if kind == "edge" else explanation.node_importance
+    )
+    if importance is None:
+        raise ValueError(
+            f"La explicación no tiene edge_importance/node_importance para "
+            f"fidelidad por {kind}."
+        )
+    import torch
+
+    imp = importance.detach().cpu().reshape(-1)
+    n = imp.shape[0]
+    if top_k is None:
+        top_k = round(n * keep_ratio)
+    top_k = int(max(0, min(top_k, n)))
+    if top_k == 0:
+        return torch.zeros(n, dtype=torch.bool)
+    idx = imp.argsort(descending=True)[:top_k]
+    mask = torch.zeros(n, dtype=torch.bool)
+    mask[idx] = True
+    return mask
+
+
+def _context(model, explanation):
+    backend = explanation.metadata.get("backend")
+    data = explanation.metadata.get("backing_data")
+    if backend is None or data is None:
+        raise ValueError(
+            "Para fidelidad± es necesario que la explicación tenga backend y "
+            "backing_data en metadata (úsala a través de Explainer)."
+        )
+    return backend, data
+
+
+def evaluate_fidelity_plus(
+    model,
+    explanation,
+    top_k: int | None = None,
+    keep_ratio: float = 0.1,
+    kind: str = "edge",
+) -> float:
+    """Necessity: drop in P(c) when removing the top-k important elements."""
+    target = _resolve_target(explanation)
+    backend, data = _context(model, explanation)
+    import torch
+
+    x = backend.node_features(data)
+    edge_index = backend.edge_index(data)
+    device = x.device
+    with torch.no_grad():
+        logits_orig = backend.forward(model, x, edge_index)
+        node = _explained_node(explanation)
+        p_orig = _prob(logits_orig[node], target, softmax=True)
+        mask = _mask_top_k(explanation, top_k, keep_ratio, kind)
+        if kind == "edge":
+            edge_weight = torch.ones(edge_index.size(1), device=device)
+            edge_weight[mask] = 0.0
+            masked = backend.forward(model, x, edge_index, edge_weight=edge_weight)
+            p_masked = _prob(masked[node], target, softmax=True)
+        else:
+            node_mask = torch.ones(x.size(0), device=device)
+            node_mask[mask] = 0.0
+            masked = backend.forward(model, x, edge_index, node_mask=node_mask)
+            p_masked = _prob(masked[node], target, softmax=True)
+    return float(p_orig - p_masked)
+
+
+def evaluate_fidelity_minus(
+    model,
+    explanation,
+    top_k: int | None = None,
+    keep_ratio: float = 0.1,
+    kind: str = "edge",
+) -> float:
+    """Sufficiency: P(c) preserved when keeping ONLY the top-k elements."""
+    target = _resolve_target(explanation)
+    backend, data = _context(model, explanation)
+    import torch
+
+    x = backend.node_features(data)
+    edge_index = backend.edge_index(data)
+    device = x.device
+    with torch.no_grad():
+        node = _explained_node(explanation)
+        mask = _mask_top_k(explanation, top_k, keep_ratio, kind)
+        if kind == "edge":
+            edge_weight = torch.zeros(edge_index.size(1), device=device)
+            edge_weight[mask] = 1.0
+            masked = backend.forward(model, x, edge_index, edge_weight=edge_weight)
+        else:
+            node_mask = torch.zeros(x.size(0), device=device)
+            node_mask[mask] = 1.0
+            masked = backend.forward(model, x, edge_index, node_mask=node_mask)
+        p_kept = _prob(masked[node], target, softmax=True)
+    return float(p_kept)
+
+
+def _explained_node(explanation) -> int:
+    if explanation.node_idx is None:
+        return 0
+    return int(explanation.node_idx)
+
+
+def _perturbed(data, perturbation: str, noise_std: float, num_edges: int | None, rng):
+    import copy
+
+    import torch
+
+    d = copy.deepcopy(data)
+    if perturbation == "feature":
+        x = d.x
+        noise = torch.randn_like(x) * noise_std
+        d.x = x + noise
+    elif perturbation == "edge" and hasattr(d, "edge_index"):
+        ei = d.edge_index
+        n = ei.size(1)
+        if num_edges is None:
+            num_edges = max(1, n // 10)
+        num_edges = min(num_edges, n)
+        drop = rng.choice(n, size=num_edges, replace=False)
+        keep = [i for i in range(n) if i not in set(drop.tolist())]
+        d.edge_index = ei[:, keep]
+        if hasattr(d, "edge_weight") and d.edge_weight is not None:
+            d.edge_weight = d.edge_weight[keep]
+    else:
+        raise ValueError(f"perturbación desconocida: {perturbation}")
+    return d
+
+
+def evaluate_stability(
+    get_explanation,
+    data,
+    num_perturbations: int = 10,
+    perturbation: str = "feature",
+    noise_std: float = 0.05,
+    num_edges: int | None = None,
+    top_k: int | None = None,
+    seed: int = 0,
+) -> float:
+    """Stability: mean similarity between explanations under small graph
+    perturbations. `get_explanation` receives a Data and returns an
+    Explanation."""
+    from itertools import pairwise
+
+    import numpy as np
+
+    rng = np.random.default_rng(seed)
+    exps = [
+        get_explanation(_perturbed(data, perturbation, noise_std, num_edges, rng))
+        for _ in range(num_perturbations)
+    ]
+    if not exps:
+        return 1.0
+    sims = []
+    for a, b in pairwise(exps):
+        sims.append(_explanation_similarity(a, b, top_k))
+    valid = [
+        s for s in sims if math.isfinite(s)
+    ]  # descarta NaN (p.ej. perturbación de aristas)
+    return float(np.mean(valid)) if valid else 1.0
+
+
+def _explanation_similarity(a, b, top_k: int | None) -> float:
+    import torch
+
+    va = _importance_vector(a)
+    vb = _importance_vector(b)
+    if va is None or vb is None or va.numel() != vb.numel():
+        return float("nan")
+    if top_k is not None:
+        ta = set(va.argsort(descending=True)[:top_k].tolist())
+        tb = set(vb.argsort(descending=True)[:top_k].tolist())
+        union = ta | tb
+        if not union:
+            return 1.0
+        return float(len(ta & tb) / len(union))
+    if va.norm().item() < 1e-9 or vb.norm().item() < 1e-9:
+        return float("nan")
+    cos = float(torch.nn.functional.cosine_similarity(va, vb, dim=0).item())
+    return max(0.0, min(1.0, cos))
+
+
+def _importance_vector(explanation):
+    parts = []
+    if explanation.node_importance is not None:
+        parts.append(explanation.node_importance.detach().reshape(-1).cpu().float())
+    if explanation.edge_importance is not None:
+        parts.append(explanation.edge_importance.detach().reshape(-1).cpu().float())
+    if not parts:
+        return None
+    import torch
+
+    return torch.cat(parts)
+
+
+def evaluate_gea(explanation, data=None, top_k: int | None = None) -> float:
+    """Graph Explanation Accuracy: overlap between the top-k elements of the
+    explanation and the benchmark's relevant ground-truth subgraph."""
+    if data is None:
+        data = explanation.metadata.get("backing_data")
+    if data is None:
+        raise ValueError("evaluate_gea necesita `data` (o backing_data en metadata).")
+    node = _explained_node(explanation)
+    gt_nodes, gt_edges = _ground_truth(data, node, explanation.metadata.get("backend"))
+    if explanation.edge_importance is not None and gt_edges:
+        imp = explanation.edge_importance.detach().reshape(-1)
+        k = top_k if top_k is not None else len(gt_edges)
+        k = int(max(0, min(k, imp.shape[0])))
+        if k == 0:
+            return 0.0
+        top = set(imp.argsort(descending=True)[:k].tolist())
+        return float(len(top & set(gt_edges)) / max(1, k))
+    if explanation.node_importance is not None and gt_nodes:
+        imp = explanation.node_importance.detach().reshape(-1)
+        k = top_k if top_k is not None else len(gt_nodes)
+        k = int(max(0, min(k, imp.shape[0])))
+        if k == 0:
+            return 0.0
+        top = set(imp.argsort(descending=True)[:k].tolist())
+        return float(len(top & set(gt_nodes)) / max(1, k))
+    raise ValueError("evaluate_gea necesita edge_importance o node_importance.")
+
+
+def _ground_truth(data, node, backend=None):
+    try:
+        from ..benchmarks.synthetic import ground_truth_edge_ids, ground_truth_nodes
+    except ImportError:
+        raise ValueError("Ground truth disponible solo con el benchmark sintético.")
+    edge_index = backend.edge_index(data) if backend is not None else data.edge_index
+    gt_nodes = ground_truth_nodes(data, node)
+    gt_edges = ground_truth_edge_ids(data, node, edge_index)
+    return gt_nodes, gt_edges
+
+
+def evaluate_fidelity(explanation, keep_ratio: float = 0.2) -> float:
+    if (
+        explanation.prediction_original is None
+        or explanation.prediction_explanation is None
+    ):
+        raise ValueError(
+            "Para evaluar fidelidad la explicación debe contener "
+            "prediction_original y prediction_explanation."
+        )
+    target = explanation.target_class
+    if target is None:
+        import torch
+
+        pred = explanation.prediction_original
+        if torch.is_tensor(pred):
+            pred = pred.reshape(-1)
+            target = int(pred.argmax().item())
+        else:
+            target = 0
+    p_orig = _prob(explanation.prediction_original, target, softmax=True)
+    p_expl = _prob(explanation.prediction_explanation, target, softmax=True)
+    return float(p_orig - p_expl)
+
+
+def evaluate_sparsity(explanation, local: bool = False, local_hops: int = 3) -> float:
+    node_mask = explanation.node_importance
+    edge_mask = explanation.edge_importance
+    if node_mask is None and edge_mask is None:
+        raise ValueError(
+            "Para evaluar esparcidad la explicación necesita node_importance "
+            "y/o edge_importance."
+        )
+    if local and explanation.node_idx is not None:
+        node_ids, edge_ids = _local_scope(explanation, local_hops)
+    else:
+        node_ids = None
+        edge_ids = None
+    masked = 0.0
+    total = 0.0
+    threshold = explanation.mask_threshold
+    if node_mask is not None:
+        total += len(node_ids) if node_ids is not None else node_mask.shape[0]
+        if node_ids is not None:
+            masked += float((node_mask[node_ids] < threshold).sum())
+        else:
+            masked += float((node_mask < threshold).sum())
+    if edge_mask is not None:
+        total += len(edge_ids) if edge_ids is not None else edge_mask.shape[0]
+        if edge_ids is not None:
+            masked += float((edge_mask[edge_ids] < threshold).sum())
+        else:
+            masked += float((edge_mask < threshold).sum())
+    return 1.0 - masked / total if total > 0 else 1.0
+
+
+def _local_scope(explanation, hops: int) -> tuple[list[int], list[int]]:
+    backend = explanation.metadata.get("backend")
+    data = explanation.metadata.get("backing_data")
+    edge_index = None
+    num_nodes = None
+    if backend is not None and data is not None:
+        edge_index = backend.edge_index(data)
+        num_nodes = backend.num_nodes(data)
+    else:
+        edge_index = getattr(data, "edge_index", None) if data is not None else None
+        num_nodes = getattr(data, "num_nodes", None)
+    if edge_index is None:
+        edge_index = explanation.metadata.get("edge_index")
+        num_nodes = num_nodes or explanation.metadata.get("num_nodes")
+    if edge_index is None or num_nodes is None:
+        return None, None
+    node_idx = int(explanation.node_idx)
+    visited = {node_idx}
+    frontier = {node_idx}
+    for _ in range(hops):
+        nxt = set()
+        for u in frontier:
+            m = (edge_index[0] == u) | (edge_index[1] == u)
+            nxt.update(edge_index[:, m].flatten().tolist())
+        frontier = nxt - visited
+        visited |= frontier
+    nodes = sorted(v for v in visited if v < num_nodes)
+    edge_ids = [
+        i
+        for i in range(edge_index.size(1))
+        if int(edge_index[0, i]) in visited and int(edge_index[1, i]) in visited
+    ]
+    return nodes, edge_ids
+
+
+def evaluate_gea_graph(
+    explanation,
+    data=None,
+    gt_edge_ids: list[int] | None = None,
+    top_k: int | None = None,
+) -> float:
+    """Graph Explanation Accuracy (graph-level): overlap of the explanation's
+    top-k edges with the dataset's known motif edges (`gt_edge_mask`) of the
+    explained graph."""
+    if gt_edge_ids is None:
+        if data is None:
+            data = explanation.metadata.get("backing_data")
+        if data is not None:
+            from ..benchmarks.synthetic import ground_truth_edges_graph
+
+            gt_edge_ids = ground_truth_edges_graph(data)
+    if not gt_edge_ids:
+        raise ValueError(
+            "evaluate_gea_graph no tiene ground truth de aristas para este grafo."
+        )
+    if explanation.edge_importance is None:
+        raise ValueError("evaluate_gea_graph necesita edge_importance.")
+    imp = explanation.edge_importance.detach().reshape(-1)
+    k = top_k if top_k is not None else len(gt_edge_ids)
+    k = int(max(0, min(k, imp.shape[0])))
+    if k == 0:
+        return 0.0
+    top = set(imp.argsort(descending=True)[:k].tolist())
+    return float(len(top & set(gt_edge_ids)) / max(1, k))
+
+
+def _prob(pred, class_idx, softmax: bool = False):
+    import torch
+
+    if torch.is_tensor(pred) and pred.dim() > 1:
+        pred = pred.reshape(-1)
+    if torch.is_tensor(pred) and pred.dim() == 1 and softmax:
+        pred = pred.softmax(dim=0)
+    if class_idx is None:
+        if torch.is_tensor(pred) and pred.numel() == 1:
+            return float(pred)
+        return float(pred)
+    if torch.is_tensor(pred):
+        if pred.dim() == 0:
+            return float(pred)
+        return float(pred[class_idx])
+    return float(pred)