explainiverse 0.8.2__py3-none-any.whl → 0.8.4__py3-none-any.whl

explainiverse/__init__.py

@@ -34,7 +34,7 @@ from explainiverse.adapters.sklearn_adapter import SklearnAdapter
 from explainiverse.adapters import TORCH_AVAILABLE
 from explainiverse.engine.suite import ExplanationSuite
 
-__version__ = "0.8.2"
+__version__ = "0.8.4"
 
 __all__ = [
     # Core

explainiverse/evaluation/__init__.py

@@ -41,6 +41,10 @@ from explainiverse.evaluation.faithfulness_extended import (
     compute_batch_faithfulness_estimate,
     compute_monotonicity,
     compute_batch_monotonicity,
+    compute_monotonicity_nguyen,
+    compute_batch_monotonicity_nguyen,
+    compute_pixel_flipping,
+    compute_batch_pixel_flipping,
 )
 
 __all__ = [
@@ -70,4 +74,8 @@ __all__ = [
     "compute_batch_faithfulness_estimate",
     "compute_monotonicity",
     "compute_batch_monotonicity",
+    "compute_monotonicity_nguyen",
+    "compute_batch_monotonicity_nguyen",
+    "compute_pixel_flipping",
+    "compute_batch_pixel_flipping",
 ]

explainiverse/evaluation/faithfulness_extended.py

@@ -256,6 +256,348 @@ def compute_batch_faithfulness_estimate(
     }
 
 
+# =============================================================================
+# Metric 4: Pixel Flipping (Bach et al., 2015)
+# =============================================================================
+
+def compute_pixel_flipping(
+    model,
+    instance: np.ndarray,
+    explanation: Explanation,
+    baseline: Union[str, float, np.ndarray, Callable] = "mean",
+    background_data: np.ndarray = None,
+    target_class: int = None,
+    use_absolute: bool = True,
+    return_curve: bool = False,
+) -> Union[float, Dict[str, Union[float, np.ndarray]]]:
+    """
+    Compute Pixel Flipping score (Bach et al., 2015).
+    
+    Sequentially removes features in order of attributed importance (most
+    important first) and measures the cumulative prediction degradation.
+    A faithful explanation should cause rapid prediction drop when the
+    most important features are removed first.
+    
+    The score is the Area Under the perturbation Curve (AUC), normalized
+    to [0, 1]. Lower AUC indicates better faithfulness (faster degradation).
+    
+    Args:
+        model: Model adapter with predict/predict_proba method
+        instance: Input instance (1D array)
+        explanation: Explanation object with feature_attributions
+        baseline: Baseline for feature removal ("mean", "median", scalar, array, callable)
+        background_data: Reference data for computing baseline (required for "mean"/"median")
+        target_class: Target class index for probability (default: predicted class)
+        use_absolute: If True, sort features by absolute attribution value
+        return_curve: If True, return full degradation curve and predictions
+        
+    Returns:
+        If return_curve=False: AUC score (float, 0 to 1, lower is better)
+        If return_curve=True: Dictionary with 'auc', 'curve', 'predictions', 'feature_order'
+        
+    References:
+        Bach, S., et al. (2015). On Pixel-Wise Explanations for Non-Linear 
+        Classifier Decisions by Layer-Wise Relevance Propagation. PLOS ONE.
+    """
+    instance = np.asarray(instance).flatten()
+    n_features = len(instance)
+    
+    # Get baseline values
+    baseline_values = compute_baseline_values(
+        baseline, background_data, n_features
+    )
+    
+    # Extract attributions as array
+    attr_array = _extract_attribution_array(explanation, n_features)
+    
+    # Sort features by attribution (descending - most important first)
+    if use_absolute:
+        sorted_indices = np.argsort(-np.abs(attr_array))
+    else:
+        sorted_indices = np.argsort(-attr_array)
+    
+    # Determine target class
+    if target_class is None:
+        pred = get_prediction_value(model, instance.reshape(1, -1))
+        if isinstance(pred, np.ndarray) and pred.ndim > 0:
+            target_class = int(np.argmax(pred))
+        else:
+            target_class = 0
+    
+    # Get original prediction for the target class
+    original_pred = get_prediction_value(model, instance.reshape(1, -1))
+    if isinstance(original_pred, np.ndarray) and original_pred.ndim > 0 and len(original_pred) > target_class:
+        original_value = original_pred[target_class]
+    else:
+        original_value = float(original_pred)
+    
+    # Start with original instance
+    current = instance.copy()
+    
+    # Track predictions as features are removed
+    predictions = [original_value]
+    
+    # Remove features one by one (most important first)
+    for idx in sorted_indices:
+        # Remove this feature (replace with baseline)
+        current[idx] = baseline_values[idx]
+        
+        # Get prediction
+        pred = get_prediction_value(model, current.reshape(1, -1))
+        if isinstance(pred, np.ndarray) and pred.ndim > 0 and len(pred) > target_class:
+            predictions.append(pred[target_class])
+        else:
+            predictions.append(float(pred))
+    
+    predictions = np.array(predictions)
+    
+    # Normalize predictions to [0, 1] relative to original
+    # curve[i] = prediction after removing i features / original prediction
+    if abs(original_value) > 1e-10:
+        curve = predictions / original_value
+    else:
+        # Handle zero original prediction
+        curve = predictions
+    
+    # Compute AUC using trapezoidal rule
+    # x-axis: fraction of features removed (0 to 1)
+    # y-axis: relative prediction value
+    x = np.linspace(0, 1, len(predictions))
+    auc = np.trapz(curve, x)
+    
+    if return_curve:
+        return {
+            "auc": float(auc),
+            "curve": curve,
+            "predictions": predictions,
+            "feature_order": sorted_indices,
+            "n_features": n_features,
+        }
+    
+    return float(auc)
+
+
+def compute_batch_pixel_flipping(
+    model,
+    X: np.ndarray,
+    explanations: List[Explanation],
+    baseline: Union[str, float, np.ndarray, Callable] = "mean",
+    max_samples: int = None,
+    use_absolute: bool = True,
+) -> Dict[str, float]:
+    """
+    Compute average Pixel Flipping score over a batch of instances.
+    
+    Args:
+        model: Model adapter
+        X: Input data (2D array)
+        explanations: List of Explanation objects (one per instance)
+        baseline: Baseline for feature removal
+        max_samples: Maximum number of samples to evaluate
+        use_absolute: If True, sort features by absolute attribution value
+        
+    Returns:
+        Dictionary with mean, std, min, max, and count of valid scores
+    """
+    n_samples = len(explanations)
+    if max_samples:
+        n_samples = min(n_samples, max_samples)
+    
+    scores = []
+    
+    for i in range(n_samples):
+        try:
+            score = compute_pixel_flipping(
+                model, X[i], explanations[i],
+                baseline=baseline, background_data=X,
+                use_absolute=use_absolute
+            )
+            if not np.isnan(score):
+                scores.append(score)
+        except Exception:
+            continue
+    
+    if not scores:
+        return {"mean": 0.0, "std": 0.0, "min": 0.0, "max": 0.0, "n_samples": 0}
+    
+    return {
+        "mean": float(np.mean(scores)),
+        "std": float(np.std(scores)),
+        "min": float(np.min(scores)),
+        "max": float(np.max(scores)),
+        "n_samples": len(scores),
+    }
+
+
+# =============================================================================
+# Metric 3: Monotonicity-Nguyen (Nguyen et al., 2020)
+# =============================================================================
+
+def compute_monotonicity_nguyen(
+    model,
+    instance: np.ndarray,
+    explanation: Explanation,
+    baseline: Union[str, float, np.ndarray, Callable] = "mean",
+    background_data: np.ndarray = None,
+    target_class: int = None,
+    use_absolute: bool = True,
+) -> float:
+    """
+    Compute Monotonicity Correlation (Nguyen et al., 2020).
+    
+    Measures the Spearman rank correlation between attribution magnitudes
+    and the prediction changes when each feature is individually removed
+    (replaced with baseline). A faithful explanation should show that
+    features with higher attributions cause larger prediction changes
+    when removed.
+    
+    Unlike Arya's Monotonicity (sequential feature addition), this metric
+    evaluates each feature independently and uses rank correlation to
+    measure agreement between attributed importance and actual impact.
+    
+    Args:
+        model: Model adapter with predict/predict_proba method
+        instance: Input instance (1D array)
+        explanation: Explanation object with feature_attributions
+        baseline: Baseline for feature removal ("mean", "median", scalar, array, callable)
+        background_data: Reference data for computing baseline (required for "mean"/"median")
+        target_class: Target class index for probability (default: predicted class)
+        use_absolute: If True, use absolute attribution values (default: True)
+        
+    Returns:
+        Monotonicity correlation score (Spearman rho, -1 to 1, higher is better)
+        
+    References:
+        Nguyen, A. P., & Martinez, M. R. (2020). Quantitative Evaluation of 
+        Machine Learning Explanations: A Human-Grounded Benchmark. 
+        arXiv:2010.07455.
+    """
+    instance = np.asarray(instance).flatten()
+    n_features = len(instance)
+    
+    # Get baseline values
+    baseline_values = compute_baseline_values(
+        baseline, background_data, n_features
+    )
+    
+    # Extract attributions as array
+    attr_array = _extract_attribution_array(explanation, n_features)
+    
+    # Determine target class
+    if target_class is None:
+        pred = get_prediction_value(model, instance.reshape(1, -1))
+        if isinstance(pred, np.ndarray) and pred.ndim > 0:
+            target_class = int(np.argmax(pred))
+        else:
+            target_class = 0
+    
+    # Get original prediction for the target class
+    original_pred = get_prediction_value(model, instance.reshape(1, -1))
+    if isinstance(original_pred, np.ndarray) and original_pred.ndim > 0 and len(original_pred) > target_class:
+        original_value = original_pred[target_class]
+    else:
+        original_value = float(original_pred)
+    
+    # Compute prediction change for each feature when removed
+    prediction_changes = []
+    attribution_values = []
+    
+    for i in range(n_features):
+        # Create perturbed instance with feature i replaced by baseline
+        perturbed = instance.copy()
+        perturbed[i] = baseline_values[i]
+        
+        # Get prediction for perturbed instance
+        perturbed_pred = get_prediction_value(model, perturbed.reshape(1, -1))
+        if isinstance(perturbed_pred, np.ndarray) and perturbed_pred.ndim > 0 and len(perturbed_pred) > target_class:
+            perturbed_value = perturbed_pred[target_class]
+        else:
+            perturbed_value = float(perturbed_pred)
+        
+        # Prediction change (drop in confidence when feature is removed)
+        # Positive change means removing the feature decreased prediction
+        change = original_value - perturbed_value
+        prediction_changes.append(abs(change))
+        
+        # Attribution value
+        if use_absolute:
+            attribution_values.append(abs(attr_array[i]))
+        else:
+            attribution_values.append(attr_array[i])
+    
+    prediction_changes = np.array(prediction_changes)
+    attribution_values = np.array(attribution_values)
+    
+    # Handle edge cases
+    if len(prediction_changes) < 2:
+        return 0.0
+    
+    # Check for constant arrays (would cause division by zero in correlation)
+    if np.std(prediction_changes) < 1e-10 or np.std(attribution_values) < 1e-10:
+        # If both are constant, consider it perfect correlation
+        if np.std(prediction_changes) < 1e-10 and np.std(attribution_values) < 1e-10:
+            return 1.0
+        # If only one is constant, correlation is undefined
+        return 0.0
+    
+    # Compute Spearman rank correlation
+    corr, _ = stats.spearmanr(attribution_values, prediction_changes)
+    
+    return float(corr) if not np.isnan(corr) else 0.0
+
+
+def compute_batch_monotonicity_nguyen(
+    model,
+    X: np.ndarray,
+    explanations: List[Explanation],
+    baseline: Union[str, float, np.ndarray, Callable] = "mean",
+    max_samples: int = None,
+    use_absolute: bool = True,
+) -> Dict[str, float]:
+    """
+    Compute average Monotonicity-Nguyen over a batch of instances.
+    
+    Args:
+        model: Model adapter
+        X: Input data (2D array)
+        explanations: List of Explanation objects (one per instance)
+        baseline: Baseline for feature removal
+        max_samples: Maximum number of samples to evaluate
+        use_absolute: If True, use absolute attribution values
+        
+    Returns:
+        Dictionary with mean, std, min, max, and count of valid scores
+    """
+    n_samples = len(explanations)
+    if max_samples:
+        n_samples = min(n_samples, max_samples)
+    
+    scores = []
+    
+    for i in range(n_samples):
+        try:
+            score = compute_monotonicity_nguyen(
+                model, X[i], explanations[i],
+                baseline=baseline, background_data=X,
+                use_absolute=use_absolute
+            )
+            if not np.isnan(score):
+                scores.append(score)
+        except Exception:
+            continue
+    
+    if not scores:
+        return {"mean": 0.0, "std": 0.0, "min": 0.0, "max": 0.0, "n_samples": 0}
+    
+    return {
+        "mean": float(np.mean(scores)),
+        "std": float(np.std(scores)),
+        "min": float(np.min(scores)),
+        "max": float(np.max(scores)),
+        "n_samples": len(scores),
+    }
+
+
 # =============================================================================
 # Metric 2: Monotonicity (Arya et al., 2019)
 # =============================================================================

{explainiverse-0.8.2.dist-info → explainiverse-0.8.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: explainiverse
-Version: 0.8.2
+Version: 0.8.4
 Summary: Unified, extensible explainability framework supporting 18 XAI methods including LIME, SHAP, LRP, TCAV, GradCAM, and more
 Home-page: https://github.com/jemsbhai/explainiverse
 License: MIT
@@ -44,7 +44,7 @@ Description-Content-Type: text/markdown
 | Feature | Description |
 |---------|-------------|
 | **18 Explainers** | LIME, KernelSHAP, TreeSHAP, Integrated Gradients, DeepLIFT, DeepSHAP, SmoothGrad, Saliency Maps, GradCAM/GradCAM++, LRP, TCAV, Anchors, Counterfactual, Permutation Importance, PDP, ALE, SAGE, ProtoDash |
-| **8 Evaluation Metrics** | Faithfulness (PGI, PGU, Comprehensiveness, Sufficiency, Correlation) and Stability (RIS, ROS, Lipschitz) |
+| **13 Evaluation Metrics** | Faithfulness (PGI, PGU, Comprehensiveness, Sufficiency, Correlation, Faithfulness Estimate, Monotonicity, Monotonicity-Nguyen, Pixel Flipping) and Stability (RIS, ROS, Lipschitz) |
 | **Unified API** | Consistent `BaseExplainer` interface with standardized `Explanation` output |
 | **Plugin Registry** | Filter explainers by scope, model type, data type; automatic recommendations |
 | **Framework Support** | Adapters for scikit-learn and PyTorch (with gradient computation) |
@@ -96,6 +96,10 @@ Explainiverse includes a comprehensive suite of evaluation metrics based on the
 | **Comprehensiveness** | Drop when removing top-k features | [DeYoung et al., 2020](https://arxiv.org/abs/1911.03429) |
 | **Sufficiency** | Prediction using only top-k features | [DeYoung et al., 2020](https://arxiv.org/abs/1911.03429) |
 | **Faithfulness Correlation** | Correlation between attribution and impact | [Bhatt et al., 2020](https://arxiv.org/abs/2005.00631) |
+| **Faithfulness Estimate** | Correlation of attributions with single-feature perturbation impact | [Alvarez-Melis & Jaakkola, 2018](https://arxiv.org/abs/1806.08049) |
+| **Monotonicity** | Sequential feature addition shows monotonic prediction increase | [Arya et al., 2019](https://arxiv.org/abs/1909.03012) |
+| **Monotonicity-Nguyen** | Spearman correlation between attributions and feature removal impact | [Nguyen & Martinez, 2020](https://arxiv.org/abs/2010.07455) |
+| **Pixel Flipping** | AUC of prediction degradation when removing features by importance | [Bach et al., 2015](https://doi.org/10.1371/journal.pone.0130140) |
 
 ### Stability Metrics
 
@@ -715,6 +719,16 @@ poetry run pytest tests/test_lrp.py::TestLRPConv2d -v
 - [x] Evaluation: Stability metrics (RIS, ROS, Lipschitz)
 - [x] PyTorch adapter with gradient support
 
+### In Progress 🔄
+- [ ] **Evaluation metrics expansion** - Adding 42 more metrics across 7 categories to exceed Quantus (37 metrics)
+  - Phase 1: Faithfulness (+9 metrics) - 4/12 complete
+  - Phase 2: Robustness (+7 metrics)
+  - Phase 3: Localisation (+8 metrics)
+  - Phase 4: Complexity (+4 metrics)
+  - Phase 5: Randomisation (+5 metrics)
+  - Phase 6: Axiomatic (+4 metrics)
+  - Phase 7: Fairness (+4 metrics)
+
 ### Planned 📋
 - [ ] Attention-based explanations (for Transformers)
 - [ ] TensorFlow/Keras adapter
@@ -734,7 +748,7 @@ If you use Explainiverse in your research, please cite:
   author = {Syed, Muntaser},
   year = {2025},
   url = {https://github.com/jemsbhai/explainiverse},
-  version = {0.8.0}
+  version = {0.8.4}
 }
 ```
 

{explainiverse-0.8.2.dist-info → explainiverse-0.8.4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-explainiverse/__init__.py,sha256=icvNmaSq0DAERqIrU60N60KCIspHbtEWTi3kt_YXTUI,1694
+explainiverse/__init__.py,sha256=bi_M_46DTXxO2sTGol7RX7LrCajNZSw12CYg7I9WE90,1694
 explainiverse/adapters/__init__.py,sha256=HcQGISyp-YQ4jEj2IYveX_c9X5otLcTNWRnVRRhzRik,781
 explainiverse/adapters/base_adapter.py,sha256=Nqt0GeDn_-PjTyJcZsE8dRTulavqFQsv8sMYWS_ps-M,603
 explainiverse/adapters/pytorch_adapter.py,sha256=DLQKJ7gB0foPwAmcrru7QdZnPRnhqDKpFCT-EaD3420,15612
@@ -9,10 +9,10 @@ explainiverse/core/explanation.py,sha256=498BbRYrNR-BOql78sENOsyWxgqLsBVZXn14lh-
 explainiverse/core/registry.py,sha256=6HttL27Ty4jYtugRf-EDIKPy80M8BfvUppAKwwGDyQ8,27207
 explainiverse/engine/__init__.py,sha256=1sZO8nH1mmwK2e-KUavBQm7zYDWUe27nyWoFy9tgsiA,197
 explainiverse/engine/suite.py,sha256=G-7OjESisSTaQ1FQrlPl4YydX13uz8Bb70hJZNlcl2M,8918
-explainiverse/evaluation/__init__.py,sha256=XFVnmwrRtHHhtxI_yOw_nsR67pJvH-IBO_lEUVI-eDE,1957
+explainiverse/evaluation/__init__.py,sha256=HicoR2_xVWQO6z7ckQj05jxa7djA7zpKozAwRyURYmA,2233
 explainiverse/evaluation/_utils.py,sha256=ej7YOPZ90gVHuuIMj45EXHq9Jx3QG7lhaj5sk26hRpg,10519
 explainiverse/evaluation/faithfulness.py,sha256=_40afOW6vJ3dQguHlJySlgWqiJF_xIvN-uVA3nPKRvI,14841
-explainiverse/evaluation/faithfulness_extended.py,sha256=0zHcmINNA88EJcKOY04Z384S3QhBMo7W2m3lGNkUiNQ,14690
+explainiverse/evaluation/faithfulness_extended.py,sha256=uMcYO6FJmzDFPAr5Y7AGkU7gYbweaPnqEhRoC4URGm0,27264
 explainiverse/evaluation/metrics.py,sha256=snNK9Ua1VzHDT6DlrhYL4m2MmRF3X15vuuVXiHbeicU,9944
 explainiverse/evaluation/stability.py,sha256=q2d3rpxpp0X1s6ADST1iZA4tzksLJpR0mYBnA_U5FIs,12090
 explainiverse/explainers/__init__.py,sha256=-ncRXbFKahH3bR0oXM2UQM4LtTdTlvdeprL6cHeqNBs,2549
@@ -39,7 +39,7 @@ explainiverse/explainers/gradient/smoothgrad.py,sha256=COIKZSFcApmMkA62M0AForHiY
 explainiverse/explainers/gradient/tcav.py,sha256=zc-8wMsc2ZOhUeSZNBJ6H6BPXlVMJ9DRcAMiL25wU9I,32242
 explainiverse/explainers/rule_based/__init__.py,sha256=gKzlFCAzwurAMLJcuYgal4XhDj1thteBGcaHWmN7iWk,243
 explainiverse/explainers/rule_based/anchors_wrapper.py,sha256=ML7W6aam-eMGZHy5ilol8qupZvNBJpYAFatEEPnuMyo,13254
-explainiverse-0.8.2.dist-info/LICENSE,sha256=28rbHe8rJgmUlRdxJACfq1Sj-MtCEhyHxkJedQd1ZYA,1070
-explainiverse-0.8.2.dist-info/METADATA,sha256=QSLwIr4RmoHpxqIfoarJX17alA-0esXfdNa1cemWu5s,23770
-explainiverse-0.8.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-explainiverse-0.8.2.dist-info/RECORD,,
+explainiverse-0.8.4.dist-info/LICENSE,sha256=28rbHe8rJgmUlRdxJACfq1Sj-MtCEhyHxkJedQd1ZYA,1070
+explainiverse-0.8.4.dist-info/METADATA,sha256=-NAqFPbZ_fOqstOEIHUP8CQLplzFqzGGdeVAoP3l7Fg,24894
+explainiverse-0.8.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+explainiverse-0.8.4.dist-info/RECORD,,