explainiverse 0.2.5__py3-none-any.whl → 0.3.0__py3-none-any.whl

explainiverse/evaluation/__init__.py

@@ -1,8 +1,60 @@
 # src/explainiverse/evaluation/__init__.py
 """
 Evaluation metrics for explanation quality.
+
+Includes:
+- Faithfulness metrics (PGI, PGU, Comprehensiveness, Sufficiency)
+- Stability metrics (RIS, ROS, Lipschitz)
+- Perturbation metrics (AOPC, ROAR)
 """
 
-from explainiverse.evaluation.metrics import compute_aopc, compute_roar
+from explainiverse.evaluation.metrics import (
+    compute_aopc,
+    compute_batch_aopc,
+    compute_roar,
+    compute_roar_curve,
+)
+
+from explainiverse.evaluation.faithfulness import (
+    compute_pgi,
+    compute_pgu,
+    compute_faithfulness_score,
+    compute_comprehensiveness,
+    compute_sufficiency,
+    compute_faithfulness_correlation,
+    compare_explainer_faithfulness,
+    compute_batch_faithfulness,
+)
+
+from explainiverse.evaluation.stability import (
+    compute_ris,
+    compute_ros,
+    compute_lipschitz_estimate,
+    compute_stability_metrics,
+    compute_batch_stability,
+    compare_explainer_stability,
+)
 
-__all__ = ["compute_aopc", "compute_roar"]
+__all__ = [
+    # Perturbation metrics (existing)
+    "compute_aopc",
+    "compute_batch_aopc",
+    "compute_roar",
+    "compute_roar_curve",
+    # Faithfulness metrics (new)
+    "compute_pgi",
+    "compute_pgu",
+    "compute_faithfulness_score",
+    "compute_comprehensiveness",
+    "compute_sufficiency",
+    "compute_faithfulness_correlation",
+    "compare_explainer_faithfulness",
+    "compute_batch_faithfulness",
+    # Stability metrics (new)
+    "compute_ris",
+    "compute_ros",
+    "compute_lipschitz_estimate",
+    "compute_stability_metrics",
+    "compute_batch_stability",
+    "compare_explainer_stability",
+]

explainiverse/evaluation/_utils.py

@@ -0,0 +1,325 @@
+# src/explainiverse/evaluation/_utils.py
+"""
+Shared utility functions for evaluation metrics.
+"""
+import numpy as np
+import re
+from typing import Union, Callable, List, Tuple
+from explainiverse.core.explanation import Explanation
+
+
+def _extract_base_feature_name(feature_str: str) -> str:
+    """
+    Extract the base feature name from LIME-style feature strings.
+    
+    LIME returns strings like "petal width (cm) <= 0.80" or "feature_2 > 3.5".
+    This extracts just the feature name part.
+    
+    Args:
+        feature_str: Feature string possibly with conditions
+        
+    Returns:
+        Base feature name
+    """
+    # Remove comparison operators and values
+    # Pattern matches: name <= value, name < value, name >= value, name > value, name = value
+    patterns = [
+        r'^(.+?)\s*<=\s*[\d\.\-]+$',
+        r'^(.+?)\s*>=\s*[\d\.\-]+$',
+        r'^(.+?)\s*<\s*[\d\.\-]+$',
+        r'^(.+?)\s*>\s*[\d\.\-]+$',
+        r'^(.+?)\s*=\s*[\d\.\-]+$',
+    ]
+    
+    for pattern in patterns:
+        match = re.match(pattern, feature_str.strip())
+        if match:
+            return match.group(1).strip()
+    
+    # No match found, return as-is
+    return feature_str.strip()
+
+
+def _match_feature_to_index(
+    feature_key: str,
+    feature_names: List[str]
+) -> int:
+    """
+    Match a feature key (possibly with LIME conditions) to its index.
+    
+    Args:
+        feature_key: Feature name from explanation (may include conditions)
+        feature_names: List of original feature names
+        
+    Returns:
+        Index of the matching feature, or -1 if not found
+    """
+    # Try exact match first
+    if feature_key in feature_names:
+        return feature_names.index(feature_key)
+    
+    # Try extracting base name
+    base_name = _extract_base_feature_name(feature_key)
+    if base_name in feature_names:
+        return feature_names.index(base_name)
+    
+    # Try partial matching (feature name is contained in key)
+    for i, fname in enumerate(feature_names):
+        if fname in feature_key:
+            return i
+    
+    # Try index extraction from patterns like "feature_2" or "f2" or "feat_2"
+    patterns = [
+        r'feature[_\s]*(\d+)',
+        r'feat[_\s]*(\d+)',
+        r'^f(\d+)$',
+        r'^x(\d+)$',
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, feature_key, re.IGNORECASE)
+        if match:
+            idx = int(match.group(1))
+            if 0 <= idx < len(feature_names):
+                return idx
+    
+    return -1
+
+
+def get_sorted_feature_indices(
+    explanation: Explanation,
+    descending: bool = True
+) -> List[int]:
+    """
+    Extract feature indices sorted by absolute attribution value.
+    
+    Handles various feature naming conventions:
+    - Clean names: "sepal length", "feature_0"
+    - LIME-style: "sepal length <= 5.0", "feature_0 > 2.3"
+    - Indexed: "f0", "x1", "feat_2"
+    
+    Args:
+        explanation: Explanation object with feature_attributions
+        descending: If True, sort from most to least important
+        
+    Returns:
+        List of feature indices sorted by importance
+    """
+    attributions = explanation.explanation_data.get("feature_attributions", {})
+    
+    if not attributions:
+        raise ValueError("No feature attributions found in explanation.")
+    
+    # Sort features by absolute importance
+    sorted_features = sorted(
+        attributions.items(),
+        key=lambda x: abs(x[1]),
+        reverse=descending
+    )
+    
+    # Map feature names to indices
+    feature_indices = []
+    feature_names = getattr(explanation, 'feature_names', None)
+    
+    for i, (fname, _) in enumerate(sorted_features):
+        if feature_names is not None:
+            idx = _match_feature_to_index(fname, feature_names)
+            if idx >= 0:
+                feature_indices.append(idx)
+            else:
+                # Fallback: use position in sorted list
+                feature_indices.append(i % len(feature_names))
+        else:
+            # No feature_names available - try to extract index from name
+            patterns = [
+                r'feature[_\s]*(\d+)',
+                r'feat[_\s]*(\d+)',
+                r'^f(\d+)',
+                r'^x(\d+)',
+            ]
+            found = False
+            for pattern in patterns:
+                match = re.search(pattern, fname, re.IGNORECASE)
+                if match:
+                    feature_indices.append(int(match.group(1)))
+                    found = True
+                    break
+            if not found:
+                feature_indices.append(i)
+    
+    return feature_indices
+
+
+def compute_baseline_values(
+    baseline: Union[str, float, np.ndarray, Callable],
+    background_data: np.ndarray = None,
+    n_features: int = None
+) -> np.ndarray:
+    """
+    Compute per-feature baseline values for perturbation.
+    
+    Args:
+        baseline: Baseline specification - one of:
+            - "mean": Use mean of background_data
+            - "median": Use median of background_data
+            - float/int: Use this value for all features
+            - np.ndarray: Use these values directly (must match n_features)
+            - Callable: Function that takes background_data and returns baseline array
+        background_data: Reference data for computing statistics (required for "mean"/"median")
+        n_features: Number of features (required if baseline is scalar)
+        
+    Returns:
+        1D numpy array of baseline values, one per feature
+    """
+    if isinstance(baseline, str):
+        if background_data is None:
+            raise ValueError(f"background_data required for baseline='{baseline}'")
+        if baseline == "mean":
+            return np.mean(background_data, axis=0)
+        elif baseline == "median":
+            return np.median(background_data, axis=0)
+        else:
+            raise ValueError(f"Unsupported string baseline: {baseline}")
+    
+    elif callable(baseline):
+        if background_data is None:
+            raise ValueError("background_data required for callable baseline")
+        result = baseline(background_data)
+        return np.asarray(result)
+    
+    elif isinstance(baseline, np.ndarray):
+        return baseline
+    
+    elif isinstance(baseline, (float, int, np.number)):
+        if n_features is None:
+            raise ValueError("n_features required for scalar baseline")
+        return np.full(n_features, baseline)
+    
+    else:
+        raise ValueError(f"Invalid baseline type: {type(baseline)}")
+
+
+def apply_feature_mask(
+    instance: np.ndarray,
+    feature_indices: List[int],
+    baseline_values: np.ndarray
+) -> np.ndarray:
+    """
+    Replace specified features with baseline values.
+    
+    Args:
+        instance: Original instance (1D array)
+        feature_indices: Indices of features to replace
+        baseline_values: Per-feature baseline values
+        
+    Returns:
+        Modified instance with specified features replaced
+    """
+    modified = instance.copy()
+    for idx in feature_indices:
+        if idx < len(modified) and idx < len(baseline_values):
+            modified[idx] = baseline_values[idx]
+    return modified
+
+
+def resolve_k(k: Union[int, float], n_features: int) -> int:
+    """
+    Resolve k to an integer number of features.
+    
+    Args:
+        k: Either an integer count or a float fraction (0-1)
+        n_features: Total number of features
+        
+    Returns:
+        Integer number of features
+    """
+    if isinstance(k, float) and 0 < k <= 1:
+        return max(1, int(k * n_features))
+    elif isinstance(k, int) and k > 0:
+        return min(k, n_features)
+    else:
+        raise ValueError(f"k must be positive int or float in (0, 1], got {k}")
+
+
+def get_prediction_value(
+    model,
+    instance: np.ndarray,
+    output_type: str = "probability"
+) -> float:
+    """
+    Get a scalar prediction value from a model.
+    
+    Works with both raw sklearn models and explainiverse adapters.
+    For adapters, .predict() typically returns probabilities.
+    
+    Args:
+        model: Model adapter with predict/predict_proba methods
+        instance: Single instance (1D array)
+        output_type: "probability" (max prob) or "class" (predicted class)
+        
+    Returns:
+        Scalar prediction value
+    """
+    instance_2d = instance.reshape(1, -1)
+    
+    if output_type == "probability":
+        # Try predict_proba first (raw sklearn model)
+        if hasattr(model, 'predict_proba'):
+            proba = model.predict_proba(instance_2d)
+            if isinstance(proba, np.ndarray):
+                if proba.ndim == 2:
+                    return float(np.max(proba[0]))
+                return float(np.max(proba))
+            return float(np.max(proba[0]))
+        
+        # Fall back to predict (adapter returns probs from predict)
+        pred = model.predict(instance_2d)
+        if isinstance(pred, np.ndarray):
+            if pred.ndim == 2:
+                return float(np.max(pred[0]))
+            elif pred.ndim == 1:
+                return float(np.max(pred))
+        return float(pred[0]) if hasattr(pred, '__getitem__') else float(pred)
+    
+    elif output_type == "class":
+        # For class prediction, use argmax of probabilities
+        if hasattr(model, 'predict_proba'):
+            proba = model.predict_proba(instance_2d)
+            return float(np.argmax(proba[0]))
+        pred = model.predict(instance_2d)
+        if isinstance(pred, np.ndarray) and pred.ndim == 2:
+            return float(np.argmax(pred[0]))
+        return float(pred[0]) if hasattr(pred, '__getitem__') else float(pred)
+    
+    else:
+        raise ValueError(f"Unknown output_type: {output_type}")
+
+
+def compute_prediction_change(
+    model,
+    original: np.ndarray,
+    perturbed: np.ndarray,
+    metric: str = "absolute"
+) -> float:
+    """
+    Compute the change in prediction between original and perturbed instances.
+    
+    Args:
+        model: Model adapter
+        original: Original instance
+        perturbed: Perturbed instance
+        metric: "absolute" for |p1 - p2|, "relative" for |p1 - p2| / p1
+        
+    Returns:
+        Prediction change value
+    """
+    orig_pred = get_prediction_value(model, original)
+    pert_pred = get_prediction_value(model, perturbed)
+    
+    if metric == "absolute":
+        return abs(orig_pred - pert_pred)
+    elif metric == "relative":
+        if abs(orig_pred) < 1e-10:
+            return abs(pert_pred)
+        return abs(orig_pred - pert_pred) / abs(orig_pred)
+    else:
+        raise ValueError(f"Unknown metric: {metric}")

explainiverse/evaluation/faithfulness.py

@@ -0,0 +1,428 @@
+# src/explainiverse/evaluation/faithfulness.py
+"""
+Faithfulness evaluation metrics for explanations.
+
+Implements:
+- PGI (Prediction Gap on Important features)
+- PGU (Prediction Gap on Unimportant features)
+- Faithfulness Correlation
+- Comprehensiveness and Sufficiency
+"""
+import numpy as np
+import pandas as pd
+from typing import Union, Callable, List, Dict, Optional
+from explainiverse.core.explanation import Explanation
+from explainiverse.evaluation._utils import (
+    get_sorted_feature_indices,
+    compute_baseline_values,
+    apply_feature_mask,
+    resolve_k,
+    get_prediction_value,
+    compute_prediction_change,
+)
+
+
+def compute_pgi(
+    model,
+    instance: np.ndarray,
+    explanation: Explanation,
+    k: Union[int, float] = 0.2,
+    baseline: Union[str, float, np.ndarray, Callable] = "mean",
+    background_data: np.ndarray = None,
+) -> float:
+    """
+    Compute Prediction Gap on Important features (PGI).
+    
+    Measures prediction change when removing the top-k most important features.
+    Higher PGI indicates the explanation correctly identified important features.
+    
+    Args:
+        model: Model adapter with predict/predict_proba method
+        instance: Input instance (1D array)
+        explanation: Explanation object with feature_attributions
+        k: Number of top features to remove (int) or fraction (float 0-1)
+        baseline: Baseline for feature replacement ("mean", "median", scalar, array, callable)
+        background_data: Reference data for computing baseline (required for "mean"/"median")
+        
+    Returns:
+        PGI score (higher = explanation identified truly important features)
+    """
+    instance = np.asarray(instance).flatten()
+    n_features = len(instance)
+    
+    # Resolve k to integer
+    k_int = resolve_k(k, n_features)
+    
+    # Get feature indices sorted by importance (most important first)
+    sorted_indices = get_sorted_feature_indices(explanation, descending=True)
+    top_k_indices = sorted_indices[:k_int]
+    
+    # Compute baseline values
+    baseline_values = compute_baseline_values(
+        baseline, background_data, n_features
+    )
+    
+    # Perturb instance by removing top-k important features
+    perturbed = apply_feature_mask(instance, top_k_indices, baseline_values)
+    
+    # Compute prediction change
+    return compute_prediction_change(model, instance, perturbed, metric="absolute")
+
+
+def compute_pgu(
+    model,
+    instance: np.ndarray,
+    explanation: Explanation,
+    k: Union[int, float] = 0.2,
+    baseline: Union[str, float, np.ndarray, Callable] = "mean",
+    background_data: np.ndarray = None,
+) -> float:
+    """
+    Compute Prediction Gap on Unimportant features (PGU).
+    
+    Measures prediction change when removing the bottom-k least important features.
+    Lower PGU indicates the explanation correctly identified unimportant features.
+    
+    Args:
+        model: Model adapter with predict/predict_proba method
+        instance: Input instance (1D array)
+        explanation: Explanation object with feature_attributions
+        k: Number of bottom features to remove (int) or fraction (float 0-1)
+        baseline: Baseline for feature replacement ("mean", "median", scalar, array, callable)
+        background_data: Reference data for computing baseline (required for "mean"/"median")
+        
+    Returns:
+        PGU score (lower = explanation correctly identified unimportant features)
+    """
+    instance = np.asarray(instance).flatten()
+    n_features = len(instance)
+    
+    # Resolve k to integer
+    k_int = resolve_k(k, n_features)
+    
+    # Get feature indices sorted by importance (least important first for PGU)
+    sorted_indices = get_sorted_feature_indices(explanation, descending=False)
+    bottom_k_indices = sorted_indices[:k_int]
+    
+    # Compute baseline values
+    baseline_values = compute_baseline_values(
+        baseline, background_data, n_features
+    )
+    
+    # Perturb instance by removing bottom-k unimportant features
+    perturbed = apply_feature_mask(instance, bottom_k_indices, baseline_values)
+    
+    # Compute prediction change
+    return compute_prediction_change(model, instance, perturbed, metric="absolute")
+
+
+def compute_faithfulness_score(
+    model,
+    instance: np.ndarray,
+    explanation: Explanation,
+    k: Union[int, float] = 0.2,
+    baseline: Union[str, float, np.ndarray, Callable] = "mean",
+    background_data: np.ndarray = None,
+    epsilon: float = 1e-7,
+) -> Dict[str, float]:
+    """
+    Compute combined faithfulness metrics.
+    
+    Args:
+        model: Model adapter
+        instance: Input instance (1D array)
+        explanation: Explanation object
+        k: Number/fraction of features for PGI/PGU
+        baseline: Baseline for feature replacement
+        background_data: Reference data for baseline computation
+        epsilon: Small constant to avoid division by zero
+        
+    Returns:
+        Dictionary containing:
+            - pgi: Prediction Gap on Important features
+            - pgu: Prediction Gap on Unimportant features  
+            - faithfulness_ratio: PGI / (PGU + epsilon) - higher is better
+            - faithfulness_diff: PGI - PGU - higher is better
+    """
+    pgi = compute_pgi(model, instance, explanation, k, baseline, background_data)
+    pgu = compute_pgu(model, instance, explanation, k, baseline, background_data)
+    
+    return {
+        "pgi": pgi,
+        "pgu": pgu,
+        "faithfulness_ratio": pgi / (pgu + epsilon),
+        "faithfulness_diff": pgi - pgu,
+    }
+
+
+def compute_comprehensiveness(
+    model,
+    instance: np.ndarray,
+    explanation: Explanation,
+    k_values: List[Union[int, float]] = None,
+    baseline: Union[str, float, np.ndarray, Callable] = "mean",
+    background_data: np.ndarray = None,
+) -> Dict[str, float]:
+    """
+    Compute comprehensiveness - how much prediction drops when removing important features.
+    
+    This is essentially PGI computed at multiple k values and averaged.
+    Higher comprehensiveness = better explanation.
+    
+    Args:
+        model: Model adapter
+        instance: Input instance
+        explanation: Explanation object
+        k_values: List of k values to evaluate (default: [0.1, 0.2, 0.3])
+        baseline: Baseline for feature replacement
+        background_data: Reference data
+        
+    Returns:
+        Dictionary with per-k scores and mean comprehensiveness
+    """
+    if k_values is None:
+        k_values = [0.1, 0.2, 0.3]
+    
+    scores = {}
+    for k in k_values:
+        score = compute_pgi(model, instance, explanation, k, baseline, background_data)
+        scores[f"comp_k{k}"] = score
+    
+    scores["comprehensiveness"] = np.mean(list(scores.values()))
+    return scores
+
+
+def compute_sufficiency(
+    model,
+    instance: np.ndarray,
+    explanation: Explanation,
+    k_values: List[Union[int, float]] = None,
+    baseline: Union[str, float, np.ndarray, Callable] = "mean",
+    background_data: np.ndarray = None,
+) -> Dict[str, float]:
+    """
+    Compute sufficiency - how much prediction is preserved when keeping only important features.
+    
+    Lower sufficiency = the important features alone are sufficient for prediction.
+    
+    Args:
+        model: Model adapter
+        instance: Input instance
+        explanation: Explanation object  
+        k_values: List of k values (fraction of features to KEEP)
+        baseline: Baseline for feature replacement
+        background_data: Reference data
+        
+    Returns:
+        Dictionary with per-k scores and mean sufficiency
+    """
+    if k_values is None:
+        k_values = [0.1, 0.2, 0.3]
+    
+    instance = np.asarray(instance).flatten()
+    n_features = len(instance)
+    
+    # Get baseline values
+    baseline_values = compute_baseline_values(baseline, background_data, n_features)
+    
+    # Get sorted indices (most important first)
+    sorted_indices = get_sorted_feature_indices(explanation, descending=True)
+    
+    scores = {}
+    for k in k_values:
+        k_int = resolve_k(k, n_features)
+        
+        # Keep only top-k features, replace rest with baseline
+        top_k_set = set(sorted_indices[:k_int])
+        indices_to_mask = [i for i in range(n_features) if i not in top_k_set]
+        
+        perturbed = apply_feature_mask(instance, indices_to_mask, baseline_values)
+        change = compute_prediction_change(model, instance, perturbed, metric="absolute")
+        scores[f"suff_k{k}"] = change
+    
+    scores["sufficiency"] = np.mean([v for k, v in scores.items() if k.startswith("suff_k")])
+    return scores
+
+
+def compute_faithfulness_correlation(
+    model,
+    instance: np.ndarray,
+    explanation: Explanation,
+    baseline: Union[str, float, np.ndarray, Callable] = "mean",
+    background_data: np.ndarray = None,
+    n_steps: int = None,
+) -> float:
+    """
+    Compute faithfulness correlation between attributions and prediction changes.
+    
+    Measures correlation between feature importance ranking and actual impact
+    on predictions when features are removed one at a time.
+    
+    Args:
+        model: Model adapter
+        instance: Input instance
+        explanation: Explanation object
+        baseline: Baseline for feature replacement
+        background_data: Reference data
+        n_steps: Number of features to evaluate (default: all features)
+        
+    Returns:
+        Pearson correlation coefficient (-1 to 1, higher is better)
+    """
+    instance = np.asarray(instance).flatten()
+    n_features = len(instance)
+    
+    if n_steps is None:
+        n_steps = n_features
+    n_steps = min(n_steps, n_features)
+    
+    # Get attributions
+    attributions = explanation.explanation_data.get("feature_attributions", {})
+    sorted_indices = get_sorted_feature_indices(explanation, descending=True)[:n_steps]
+    
+    # Get baseline
+    baseline_values = compute_baseline_values(baseline, background_data, n_features)
+    
+    # Compute importance values and prediction changes for each feature
+    importance_values = []
+    prediction_changes = []
+    
+    feature_names = getattr(explanation, 'feature_names', None)
+    
+    for idx in sorted_indices:
+        # Get importance value for this feature
+        if feature_names and idx < len(feature_names):
+            fname = feature_names[idx]
+        else:
+            # Try common naming patterns
+            for pattern in [f"feature_{idx}", f"f{idx}", f"feat_{idx}"]:
+                if pattern in attributions:
+                    fname = pattern
+                    break
+            else:
+                fname = list(attributions.keys())[sorted_indices.index(idx)] if idx < len(attributions) else None
+        
+        if fname and fname in attributions:
+            importance_values.append(abs(attributions[fname]))
+        else:
+            continue
+        
+        # Compute prediction change when removing this single feature
+        perturbed = apply_feature_mask(instance, [idx], baseline_values)
+        change = compute_prediction_change(model, instance, perturbed, metric="absolute")
+        prediction_changes.append(change)
+    
+    if len(importance_values) < 2:
+        return 0.0  # Not enough data points
+    
+    # Compute Pearson correlation
+    return float(np.corrcoef(importance_values, prediction_changes)[0, 1])
+
+
+def compare_explainer_faithfulness(
+    model,
+    X: np.ndarray,
+    explanations: Dict[str, List[Explanation]],
+    k: Union[int, float] = 0.2,
+    baseline: Union[str, float, np.ndarray, Callable] = "mean",
+    max_samples: int = None,
+) -> pd.DataFrame:
+    """
+    Compare multiple explainers on faithfulness metrics across a dataset.
+    
+    Args:
+        model: Model adapter
+        X: Input data (2D array, n_samples x n_features)
+        explanations: Dict mapping explainer names to lists of Explanation objects
+        k: Number/fraction of features for PGI/PGU
+        baseline: Baseline for feature replacement
+        max_samples: Limit number of samples to evaluate (None = all)
+        
+    Returns:
+        DataFrame with columns: [explainer, mean_pgi, std_pgi, mean_pgu, std_pgu, 
+                                 mean_ratio, mean_diff, n_samples]
+    """
+    results = []
+    
+    for explainer_name, expl_list in explanations.items():
+        n_samples = len(expl_list)
+        if max_samples:
+            n_samples = min(n_samples, max_samples)
+        
+        pgi_scores = []
+        pgu_scores = []
+        
+        for i in range(n_samples):
+            instance = X[i]
+            exp = expl_list[i]
+            
+            try:
+                scores = compute_faithfulness_score(
+                    model, instance, exp, k, baseline, X
+                )
+                pgi_scores.append(scores["pgi"])
+                pgu_scores.append(scores["pgu"])
+            except Exception as e:
+                # Skip instances that fail
+                continue
+        
+        if pgi_scores:
+            results.append({
+                "explainer": explainer_name,
+                "mean_pgi": np.mean(pgi_scores),
+                "std_pgi": np.std(pgi_scores),
+                "mean_pgu": np.mean(pgu_scores),
+                "std_pgu": np.std(pgu_scores),
+                "mean_ratio": np.mean(pgi_scores) / (np.mean(pgu_scores) + 1e-7),
+                "mean_diff": np.mean(pgi_scores) - np.mean(pgu_scores),
+                "n_samples": len(pgi_scores),
+            })
+    
+    return pd.DataFrame(results)
+
+
+def compute_batch_faithfulness(
+    model,
+    X: np.ndarray,
+    explanations: List[Explanation],
+    k: Union[int, float] = 0.2,
+    baseline: Union[str, float, np.ndarray, Callable] = "mean",
+) -> Dict[str, float]:
+    """
+    Compute average faithfulness metrics over a batch of instances.
+    
+    Args:
+        model: Model adapter
+        X: Input data (2D array)
+        explanations: List of Explanation objects (one per instance)
+        k: Number/fraction of features for PGI/PGU
+        baseline: Baseline for feature replacement
+        
+    Returns:
+        Dictionary with aggregated metrics
+    """
+    pgi_scores = []
+    pgu_scores = []
+    
+    for i, exp in enumerate(explanations):
+        try:
+            scores = compute_faithfulness_score(
+                model, X[i], exp, k, baseline, X
+            )
+            pgi_scores.append(scores["pgi"])
+            pgu_scores.append(scores["pgu"])
+        except Exception:
+            continue
+    
+    if not pgi_scores:
+        return {"mean_pgi": 0.0, "mean_pgu": 0.0, "mean_ratio": 0.0, "n_samples": 0}
+    
+    return {
+        "mean_pgi": np.mean(pgi_scores),
+        "std_pgi": np.std(pgi_scores),
+        "mean_pgu": np.mean(pgu_scores),
+        "std_pgu": np.std(pgu_scores),
+        "mean_ratio": np.mean(pgi_scores) / (np.mean(pgu_scores) + 1e-7),
+        "mean_diff": np.mean(pgi_scores) - np.mean(pgu_scores),
+        "n_samples": len(pgi_scores),
+    }

explainiverse/evaluation/stability.py

@@ -0,0 +1,379 @@
+# src/explainiverse/evaluation/stability.py
+"""
+Stability evaluation metrics for explanations.
+
+Implements:
+- RIS (Relative Input Stability) - sensitivity to input perturbations
+- ROS (Relative Output Stability) - consistency with similar predictions
+- Lipschitz Estimate - local smoothness of explanations
+"""
+import numpy as np
+from typing import Union, Callable, List, Dict, Optional, Tuple
+from explainiverse.core.explanation import Explanation
+from explainiverse.core.explainer import BaseExplainer
+from explainiverse.evaluation._utils import get_prediction_value
+
+
+def _extract_attribution_vector(explanation: Explanation) -> np.ndarray:
+    """
+    Extract attribution values as a numpy array from an Explanation.
+    
+    Args:
+        explanation: Explanation object with feature_attributions
+        
+    Returns:
+        1D numpy array of attribution values
+    """
+    attributions = explanation.explanation_data.get("feature_attributions", {})
+    if not attributions:
+        raise ValueError("No feature attributions found in explanation.")
+    
+    # Get values in consistent order
+    feature_names = getattr(explanation, 'feature_names', None)
+    if feature_names:
+        values = [attributions.get(fn, 0.0) for fn in feature_names]
+    else:
+        values = list(attributions.values())
+    
+    return np.array(values, dtype=float)
+
+
+def _normalize_vector(v: np.ndarray, epsilon: float = 1e-10) -> np.ndarray:
+    """Normalize a vector to unit length."""
+    norm = np.linalg.norm(v)
+    if norm < epsilon:
+        return v
+    return v / norm
+
+
+def compute_ris(
+    explainer: BaseExplainer,
+    instance: np.ndarray,
+    n_perturbations: int = 10,
+    noise_scale: float = 0.01,
+    seed: int = None,
+) -> float:
+    """
+    Compute Relative Input Stability (RIS).
+    
+    Measures how stable explanations are to small perturbations in the input.
+    Lower RIS indicates more stable explanations.
+    
+    RIS = mean(||E(x) - E(x')|| / ||x - x'||) for perturbed inputs x'
+    
+    Args:
+        explainer: Explainer instance with .explain() method
+        instance: Original input instance (1D array)
+        n_perturbations: Number of perturbed samples to generate
+        noise_scale: Standard deviation of Gaussian noise (relative to feature range)
+        seed: Random seed for reproducibility
+        
+    Returns:
+        RIS score (lower = more stable)
+    """
+    if seed is not None:
+        np.random.seed(seed)
+    
+    instance = np.asarray(instance).flatten()
+    n_features = len(instance)
+    
+    # Get original explanation
+    original_exp = explainer.explain(instance)
+    original_exp.feature_names = getattr(original_exp, 'feature_names', None) or \
+                                  [f"feature_{i}" for i in range(n_features)]
+    original_attr = _extract_attribution_vector(original_exp)
+    
+    ratios = []
+    
+    for _ in range(n_perturbations):
+        # Generate perturbed input
+        noise = np.random.normal(0, noise_scale, n_features)
+        perturbed = instance + noise * np.abs(instance + 1e-10)  # Scale noise by feature magnitude
+        
+        # Get perturbed explanation
+        try:
+            perturbed_exp = explainer.explain(perturbed)
+            perturbed_exp.feature_names = original_exp.feature_names
+            perturbed_attr = _extract_attribution_vector(perturbed_exp)
+        except Exception:
+            continue
+        
+        # Compute ratio of changes
+        attr_diff = np.linalg.norm(original_attr - perturbed_attr)
+        input_diff = np.linalg.norm(instance - perturbed)
+        
+        if input_diff > 1e-10:
+            ratios.append(attr_diff / input_diff)
+    
+    if not ratios:
+        return float('inf')
+    
+    return float(np.mean(ratios))
+
+
+def compute_ros(
+    explainer: BaseExplainer,
+    model,
+    instance: np.ndarray,
+    reference_instances: np.ndarray,
+    n_neighbors: int = 5,
+    prediction_threshold: float = 0.05,
+) -> float:
+    """
+    Compute Relative Output Stability (ROS).
+    
+    Measures how similar explanations are for instances with similar predictions.
+    Higher ROS indicates more consistent explanations.
+    
+    Args:
+        explainer: Explainer instance with .explain() method
+        model: Model adapter with predict/predict_proba method
+        instance: Query instance
+        reference_instances: Pool of reference instances to find neighbors
+        n_neighbors: Number of neighbors to compare
+        prediction_threshold: Maximum prediction difference to consider "similar"
+        
+    Returns:
+        ROS score (higher = more consistent for similar predictions)
+    """
+    instance = np.asarray(instance).flatten()
+    n_features = len(instance)
+    
+    # Get prediction for query instance
+    query_pred = get_prediction_value(model, instance)
+    
+    # Find instances with similar predictions
+    similar_instances = []
+    for ref in reference_instances:
+        ref = np.asarray(ref).flatten()
+        ref_pred = get_prediction_value(model, ref)
+        if abs(query_pred - ref_pred) <= prediction_threshold:
+            similar_instances.append(ref)
+    
+    if len(similar_instances) < 2:
+        return 1.0  # Perfect stability if no similar instances
+    
+    # Limit to n_neighbors
+    similar_instances = similar_instances[:n_neighbors]
+    
+    # Get explanation for query
+    query_exp = explainer.explain(instance)
+    query_exp.feature_names = getattr(query_exp, 'feature_names', None) or \
+                               [f"feature_{i}" for i in range(n_features)]
+    query_attr = _normalize_vector(_extract_attribution_vector(query_exp))
+    
+    # Get explanations for similar instances and compute similarity
+    similarities = []
+    for ref in similar_instances:
+        try:
+            ref_exp = explainer.explain(ref)
+            ref_exp.feature_names = query_exp.feature_names
+            ref_attr = _normalize_vector(_extract_attribution_vector(ref_exp))
+            
+            # Cosine similarity
+            similarity = np.dot(query_attr, ref_attr)
+            similarities.append(similarity)
+        except Exception:
+            continue
+    
+    if not similarities:
+        return 1.0
+    
+    return float(np.mean(similarities))
+
+
+def compute_lipschitz_estimate(
+    explainer: BaseExplainer,
+    instance: np.ndarray,
+    n_samples: int = 20,
+    radius: float = 0.1,
+    seed: int = None,
+) -> float:
+    """
+    Estimate local Lipschitz constant of the explanation function.
+    
+    The Lipschitz constant bounds how fast explanations can change:
+    ||E(x) - E(y)|| <= L * ||x - y||
+    
+    Lower L indicates smoother, more stable explanations.
+    
+    Args:
+        explainer: Explainer instance
+        instance: Center point for local estimate
+        n_samples: Number of sample pairs to evaluate
+        radius: Radius of ball around instance to sample from
+        seed: Random seed
+        
+    Returns:
+        Estimated local Lipschitz constant (lower = smoother)
+    """
+    if seed is not None:
+        np.random.seed(seed)
+    
+    instance = np.asarray(instance).flatten()
+    n_features = len(instance)
+    
+    max_ratio = 0.0
+    
+    for _ in range(n_samples):
+        # Generate two random points in a ball around instance
+        direction1 = np.random.randn(n_features)
+        direction1 = direction1 / np.linalg.norm(direction1)
+        r1 = np.random.uniform(0, radius)
+        point1 = instance + r1 * direction1
+        
+        direction2 = np.random.randn(n_features)
+        direction2 = direction2 / np.linalg.norm(direction2)
+        r2 = np.random.uniform(0, radius)
+        point2 = instance + r2 * direction2
+        
+        try:
+            exp1 = explainer.explain(point1)
+            exp1.feature_names = [f"feature_{i}" for i in range(n_features)]
+            attr1 = _extract_attribution_vector(exp1)
+            
+            exp2 = explainer.explain(point2)
+            exp2.feature_names = exp1.feature_names
+            attr2 = _extract_attribution_vector(exp2)
+        except Exception:
+            continue
+        
+        attr_diff = np.linalg.norm(attr1 - attr2)
+        input_diff = np.linalg.norm(point1 - point2)
+        
+        if input_diff > 1e-10:
+            ratio = attr_diff / input_diff
+            max_ratio = max(max_ratio, ratio)
+    
+    return float(max_ratio)
+
+
+def compute_stability_metrics(
+    explainer: BaseExplainer,
+    model,
+    instance: np.ndarray,
+    background_data: np.ndarray,
+    n_perturbations: int = 10,
+    noise_scale: float = 0.01,
+    n_neighbors: int = 5,
+    seed: int = None,
+) -> Dict[str, float]:
+    """
+    Compute comprehensive stability metrics for a single instance.
+    
+    Args:
+        explainer: Explainer instance
+        model: Model adapter
+        instance: Query instance
+        background_data: Reference data for ROS computation
+        n_perturbations: Number of perturbations for RIS
+        noise_scale: Noise scale for RIS
+        n_neighbors: Number of neighbors for ROS
+        seed: Random seed
+        
+    Returns:
+        Dictionary with RIS, ROS, and Lipschitz estimate
+    """
+    return {
+        "ris": compute_ris(explainer, instance, n_perturbations, noise_scale, seed),
+        "ros": compute_ros(explainer, model, instance, background_data, n_neighbors),
+        "lipschitz": compute_lipschitz_estimate(explainer, instance, seed=seed),
+    }
+
+
+def compute_batch_stability(
+    explainer: BaseExplainer,
+    model,
+    X: np.ndarray,
+    n_perturbations: int = 10,
+    noise_scale: float = 0.01,
+    max_samples: int = None,
+    seed: int = None,
+) -> Dict[str, float]:
+    """
+    Compute average stability metrics over a batch of instances.
+    
+    Args:
+        explainer: Explainer instance
+        model: Model adapter
+        X: Input data (2D array)
+        n_perturbations: Number of perturbations per instance
+        noise_scale: Noise scale for perturbations
+        max_samples: Maximum number of samples to evaluate
+        seed: Random seed
+        
+    Returns:
+        Dictionary with mean and std of stability metrics
+    """
+    n_samples = len(X)
+    if max_samples:
+        n_samples = min(n_samples, max_samples)
+    
+    ris_scores = []
+    ros_scores = []
+    
+    for i in range(n_samples):
+        instance = X[i]
+        
+        try:
+            ris = compute_ris(explainer, instance, n_perturbations, noise_scale, seed)
+            if not np.isinf(ris):
+                ris_scores.append(ris)
+            
+            ros = compute_ros(explainer, model, instance, X, n_neighbors=5)
+            ros_scores.append(ros)
+        except Exception:
+            continue
+    
+    results = {"n_samples": len(ris_scores)}
+    
+    if ris_scores:
+        results["mean_ris"] = np.mean(ris_scores)
+        results["std_ris"] = np.std(ris_scores)
+    else:
+        results["mean_ris"] = float('inf')
+        results["std_ris"] = 0.0
+    
+    if ros_scores:
+        results["mean_ros"] = np.mean(ros_scores)
+        results["std_ros"] = np.std(ros_scores)
+    else:
+        results["mean_ros"] = 0.0
+        results["std_ros"] = 0.0
+    
+    return results
+
+
+def compare_explainer_stability(
+    explainers: Dict[str, BaseExplainer],
+    model,
+    X: np.ndarray,
+    n_perturbations: int = 5,
+    noise_scale: float = 0.01,
+    max_samples: int = 20,
+    seed: int = None,
+) -> Dict[str, Dict[str, float]]:
+    """
+    Compare stability metrics across multiple explainers.
+    
+    Args:
+        explainers: Dict mapping explainer names to explainer instances
+        model: Model adapter
+        X: Input data
+        n_perturbations: Number of perturbations per instance
+        noise_scale: Noise scale
+        max_samples: Max samples to evaluate per explainer
+        seed: Random seed
+        
+    Returns:
+        Dict mapping explainer names to their stability metrics
+    """
+    results = {}
+    
+    for name, explainer in explainers.items():
+        metrics = compute_batch_stability(
+            explainer, model, X, n_perturbations, noise_scale, max_samples, seed
+        )
+        results[name] = metrics
+    
+    return results

{explainiverse-0.2.5.dist-info → explainiverse-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: explainiverse
-Version: 0.2.5
+Version: 0.3.0
 Summary: Unified, extensible explainability framework supporting LIME, SHAP, Anchors, Counterfactuals, PDP, ALE, SAGE, and more
 Home-page: https://github.com/jemsbhai/explainiverse
 License: MIT
@@ -20,6 +20,7 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Provides-Extra: torch
 Requires-Dist: lime (>=0.2.0.1,<0.3.0.0)
 Requires-Dist: numpy (>=1.24,<2.0)
+Requires-Dist: pandas (>=1.5,<3.0)
 Requires-Dist: scikit-learn (>=1.1,<1.6)
 Requires-Dist: scipy (>=1.10,<2.0)
 Requires-Dist: shap (>=0.48.0,<0.49.0)

{explainiverse-0.2.5.dist-info → explainiverse-0.3.0.dist-info}/RECORD

@@ -9,8 +9,11 @@ explainiverse/core/explanation.py,sha256=6zxFh_TH8tFHc-r_H5-WHQ05Sp1Kp2TxLz3gyFe
 explainiverse/core/registry.py,sha256=f1GAo2tg6Sjyz-uOPyLukYYSUgMmpb95pI3B6O-5jjo,22992
 explainiverse/engine/__init__.py,sha256=1sZO8nH1mmwK2e-KUavBQm7zYDWUe27nyWoFy9tgsiA,197
 explainiverse/engine/suite.py,sha256=sq8SK_6Pf0qRckTmVJ7Mdosu9bhkjAGPGN8ymLGFP9E,4914
-explainiverse/evaluation/__init__.py,sha256=Y50L_b4HKthg4epwcayPHXh0l4i4MUuzvaNlqPmUNZY,212
+explainiverse/evaluation/__init__.py,sha256=ePE97KwSjg_IChZ03DeQax8GruTjx-BVrMSi_nzoyoA,1501
+explainiverse/evaluation/_utils.py,sha256=ej7YOPZ90gVHuuIMj45EXHq9Jx3QG7lhaj5sk26hRpg,10519
+explainiverse/evaluation/faithfulness.py,sha256=_40afOW6vJ3dQguHlJySlgWqiJF_xIvN-uVA3nPKRvI,14841
 explainiverse/evaluation/metrics.py,sha256=tSBXtyA_-0zOGCGjlPZU6LdGKRH_QpWfgKa78sdlovs,7453
+explainiverse/evaluation/stability.py,sha256=q2d3rpxpp0X1s6ADST1iZA4tzksLJpR0mYBnA_U5FIs,12090
 explainiverse/explainers/__init__.py,sha256=d7DTbUXzdVdN0l5GQnoJ4zzutI0TXNvx0UzwNXoWY9w,2207
 explainiverse/explainers/attribution/__init__.py,sha256=YeVs9bS_IWDtqGbp6T37V6Zp5ZDWzLdAXHxxyFGpiQM,431
 explainiverse/explainers/attribution/lime_wrapper.py,sha256=OnXIV7t6yd-vt38sIi7XmHFbgzlZfCEbRlFyGGd5XiE,3245
@@ -29,7 +32,7 @@ explainiverse/explainers/gradient/gradcam.py,sha256=ywW_8PhALwegkpSUDQMFvvVFkA5N
 explainiverse/explainers/gradient/integrated_gradients.py,sha256=feBgY3Vw2rDti7fxRZtLkxse75m2dbP_R05ARqo2BRM,13367
 explainiverse/explainers/rule_based/__init__.py,sha256=gKzlFCAzwurAMLJcuYgal4XhDj1thteBGcaHWmN7iWk,243
 explainiverse/explainers/rule_based/anchors_wrapper.py,sha256=ML7W6aam-eMGZHy5ilol8qupZvNBJpYAFatEEPnuMyo,13254
-explainiverse-0.2.5.dist-info/LICENSE,sha256=28rbHe8rJgmUlRdxJACfq1Sj-MtCEhyHxkJedQd1ZYA,1070
-explainiverse-0.2.5.dist-info/METADATA,sha256=ih7rkKiPaVBmdYHwMRKB3QfrOaCm68m3iHztt6SO5S0,11483
-explainiverse-0.2.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-explainiverse-0.2.5.dist-info/RECORD,,
+explainiverse-0.3.0.dist-info/LICENSE,sha256=28rbHe8rJgmUlRdxJACfq1Sj-MtCEhyHxkJedQd1ZYA,1070
+explainiverse-0.3.0.dist-info/METADATA,sha256=F53kTds8YbDDtEes-9dC6lDlxAYXekNSMtIGvwB1eY4,11518
+explainiverse-0.3.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+explainiverse-0.3.0.dist-info/RECORD,,