explainiverse 0.6.0__tar.gz → 0.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {explainiverse-0.6.0 → explainiverse-0.7.1}/PKG-INFO +60 -9
- {explainiverse-0.6.0 → explainiverse-0.7.1}/README.md +59 -8
- {explainiverse-0.6.0 → explainiverse-0.7.1}/pyproject.toml +1 -1
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/__init__.py +1 -1
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/adapters/pytorch_adapter.py +88 -25
- explainiverse-0.7.1/src/explainiverse/core/explanation.py +179 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/core/registry.py +18 -0
- explainiverse-0.7.1/src/explainiverse/engine/suite.py +252 -0
- explainiverse-0.7.1/src/explainiverse/evaluation/metrics.py +314 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/attribution/lime_wrapper.py +90 -7
- explainiverse-0.7.1/src/explainiverse/explainers/attribution/shap_wrapper.py +185 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/gradient/__init__.py +12 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/gradient/integrated_gradients.py +189 -76
- explainiverse-0.7.1/src/explainiverse/explainers/gradient/tcav.py +865 -0
- explainiverse-0.6.0/src/explainiverse/core/explanation.py +0 -24
- explainiverse-0.6.0/src/explainiverse/engine/suite.py +0 -143
- explainiverse-0.6.0/src/explainiverse/evaluation/metrics.py +0 -233
- explainiverse-0.6.0/src/explainiverse/explainers/attribution/shap_wrapper.py +0 -89
- {explainiverse-0.6.0 → explainiverse-0.7.1}/LICENSE +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/adapters/__init__.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/adapters/base_adapter.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/adapters/sklearn_adapter.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/core/__init__.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/core/explainer.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/engine/__init__.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/evaluation/__init__.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/evaluation/_utils.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/evaluation/faithfulness.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/evaluation/stability.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/__init__.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/attribution/__init__.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/attribution/treeshap_wrapper.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/counterfactual/__init__.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/counterfactual/dice_wrapper.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/example_based/__init__.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/example_based/protodash.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/global_explainers/__init__.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/global_explainers/ale.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/global_explainers/partial_dependence.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/global_explainers/permutation_importance.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/global_explainers/sage.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/gradient/deeplift.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/gradient/gradcam.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/gradient/saliency.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/gradient/smoothgrad.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/rule_based/__init__.py +0 -0
- {explainiverse-0.6.0 → explainiverse-0.7.1}/src/explainiverse/explainers/rule_based/anchors_wrapper.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: explainiverse
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.1
|
|
4
4
|
Summary: Unified, extensible explainability framework supporting LIME, SHAP, Anchors, Counterfactuals, PDP, ALE, SAGE, and more
|
|
5
5
|
Home-page: https://github.com/jemsbhai/explainiverse
|
|
6
6
|
License: MIT
|
|
@@ -35,7 +35,7 @@ Description-Content-Type: text/markdown
|
|
|
35
35
|
[](https://www.python.org/downloads/)
|
|
36
36
|
[](https://opensource.org/licenses/MIT)
|
|
37
37
|
|
|
38
|
-
**Explainiverse** is a unified, extensible Python framework for Explainable AI (XAI). It provides a standardized interface for **
|
|
38
|
+
**Explainiverse** is a unified, extensible Python framework for Explainable AI (XAI). It provides a standardized interface for **17 state-of-the-art explanation methods** across local, global, gradient-based, concept-based, and example-based paradigms, along with **comprehensive evaluation metrics** for assessing explanation quality.
|
|
39
39
|
|
|
40
40
|
---
|
|
41
41
|
|
|
@@ -43,7 +43,7 @@ Description-Content-Type: text/markdown
|
|
|
43
43
|
|
|
44
44
|
| Feature | Description |
|
|
45
45
|
|---------|-------------|
|
|
46
|
-
| **
|
|
46
|
+
| **17 Explainers** | LIME, KernelSHAP, TreeSHAP, Integrated Gradients, DeepLIFT, DeepSHAP, SmoothGrad, Saliency Maps, GradCAM/GradCAM++, TCAV, Anchors, Counterfactual, Permutation Importance, PDP, ALE, SAGE, ProtoDash |
|
|
47
47
|
| **8 Evaluation Metrics** | Faithfulness (PGI, PGU, Comprehensiveness, Sufficiency, Correlation) and Stability (RIS, ROS, Lipschitz) |
|
|
48
48
|
| **Unified API** | Consistent `BaseExplainer` interface with standardized `Explanation` output |
|
|
49
49
|
| **Plugin Registry** | Filter explainers by scope, model type, data type; automatic recommendations |
|
|
@@ -66,6 +66,7 @@ Description-Content-Type: text/markdown
|
|
|
66
66
|
| **SmoothGrad** | Gradient | [Smilkov et al., 2017](https://arxiv.org/abs/1706.03825) |
|
|
67
67
|
| **Saliency Maps** | Gradient | [Simonyan et al., 2014](https://arxiv.org/abs/1312.6034) |
|
|
68
68
|
| **GradCAM / GradCAM++** | Gradient (CNN) | [Selvaraju et al., 2017](https://arxiv.org/abs/1610.02391) |
|
|
69
|
+
| **TCAV** | Concept-Based | [Kim et al., 2018](https://arxiv.org/abs/1711.11279) |
|
|
69
70
|
| **Anchors** | Rule-Based | [Ribeiro et al., 2018](https://ojs.aaai.org/index.php/AAAI/article/view/11491) |
|
|
70
71
|
| **Counterfactual** | Contrastive | [Mothilal et al., 2020](https://arxiv.org/abs/1905.07697) |
|
|
71
72
|
| **ProtoDash** | Example-Based | [Gurumoorthy et al., 2019](https://arxiv.org/abs/1707.01212) |
|
|
@@ -142,8 +143,8 @@ adapter = SklearnAdapter(model, class_names=iris.target_names.tolist())
|
|
|
142
143
|
# List all available explainers
|
|
143
144
|
print(default_registry.list_explainers())
|
|
144
145
|
# ['lime', 'shap', 'treeshap', 'integrated_gradients', 'deeplift', 'deepshap',
|
|
145
|
-
# 'smoothgrad', 'gradcam', '
|
|
146
|
-
# 'permutation_importance', 'partial_dependence', 'ale', 'sage']
|
|
146
|
+
# 'smoothgrad', 'saliency', 'gradcam', 'tcav', 'anchors', 'counterfactual',
|
|
147
|
+
# 'protodash', 'permutation_importance', 'partial_dependence', 'ale', 'sage']
|
|
147
148
|
|
|
148
149
|
# Create an explainer via registry
|
|
149
150
|
explainer = default_registry.create(
|
|
@@ -317,6 +318,56 @@ heatmap = explanation.explanation_data["heatmap"]
|
|
|
317
318
|
overlay = explainer.get_overlay(original_image, heatmap, alpha=0.5)
|
|
318
319
|
```
|
|
319
320
|
|
|
321
|
+
### TCAV (Concept-Based Explanations)
|
|
322
|
+
|
|
323
|
+
```python
|
|
324
|
+
from explainiverse.explainers.gradient import TCAVExplainer
|
|
325
|
+
|
|
326
|
+
# For neural network models with concept examples
|
|
327
|
+
adapter = PyTorchAdapter(model, task="classification", class_names=class_names)
|
|
328
|
+
|
|
329
|
+
# Create TCAV explainer targeting a specific layer
|
|
330
|
+
explainer = TCAVExplainer(
|
|
331
|
+
model=adapter,
|
|
332
|
+
layer_name="layer3", # Target layer for concept analysis
|
|
333
|
+
class_names=class_names
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
# Learn a concept from examples (e.g., "striped" pattern)
|
|
337
|
+
explainer.learn_concept(
|
|
338
|
+
concept_name="striped",
|
|
339
|
+
concept_examples=striped_images, # Images with stripes
|
|
340
|
+
negative_examples=random_images, # Random images without stripes
|
|
341
|
+
min_accuracy=0.6 # Minimum CAV classifier accuracy
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
# Compute TCAV score: fraction of inputs where concept positively influences prediction
|
|
345
|
+
tcav_score = explainer.compute_tcav_score(
|
|
346
|
+
test_inputs=test_images,
|
|
347
|
+
target_class=0, # e.g., "zebra"
|
|
348
|
+
concept_name="striped"
|
|
349
|
+
)
|
|
350
|
+
print(f"TCAV score: {tcav_score:.3f}") # >0.5 means concept positively influences class
|
|
351
|
+
|
|
352
|
+
# Statistical significance testing against random concepts
|
|
353
|
+
result = explainer.statistical_significance_test(
|
|
354
|
+
test_inputs=test_images,
|
|
355
|
+
target_class=0,
|
|
356
|
+
concept_name="striped",
|
|
357
|
+
n_random=10,
|
|
358
|
+
negative_examples=random_images
|
|
359
|
+
)
|
|
360
|
+
print(f"p-value: {result['p_value']:.4f}, significant: {result['significant']}")
|
|
361
|
+
|
|
362
|
+
# Full explanation with multiple concepts
|
|
363
|
+
explanation = explainer.explain(
|
|
364
|
+
test_inputs=test_images,
|
|
365
|
+
target_class=0,
|
|
366
|
+
run_significance_test=True
|
|
367
|
+
)
|
|
368
|
+
print(explanation.explanation_data["tcav_scores"])
|
|
369
|
+
```
|
|
370
|
+
|
|
320
371
|
---
|
|
321
372
|
|
|
322
373
|
## Example-Based Explanations
|
|
@@ -551,7 +602,7 @@ explainiverse/
|
|
|
551
602
|
│ └── pytorch_adapter.py # With gradient support
|
|
552
603
|
├── explainers/
|
|
553
604
|
│ ├── attribution/ # LIME, SHAP, TreeSHAP
|
|
554
|
-
│ ├── gradient/ # IG, DeepLIFT, DeepSHAP, SmoothGrad, GradCAM
|
|
605
|
+
│ ├── gradient/ # IG, DeepLIFT, DeepSHAP, SmoothGrad, Saliency, GradCAM, TCAV
|
|
555
606
|
│ ├── rule_based/ # Anchors
|
|
556
607
|
│ ├── counterfactual/ # DiCE-style
|
|
557
608
|
│ ├── global_explainers/ # Permutation, PDP, ALE, SAGE
|
|
@@ -589,6 +640,7 @@ poetry run pytest tests/test_smoothgrad.py::TestSmoothGradBasic -v
|
|
|
589
640
|
- [x] Core framework (BaseExplainer, Explanation, Registry)
|
|
590
641
|
- [x] Perturbation methods: LIME, KernelSHAP, TreeSHAP
|
|
591
642
|
- [x] Gradient methods: Integrated Gradients, DeepLIFT, DeepSHAP, SmoothGrad, Saliency Maps, GradCAM/GradCAM++
|
|
643
|
+
- [x] Concept-based: TCAV (Testing with Concept Activation Vectors)
|
|
592
644
|
- [x] Rule-based: Anchors
|
|
593
645
|
- [x] Counterfactual: DiCE-style
|
|
594
646
|
- [x] Global: Permutation Importance, PDP, ALE, SAGE
|
|
@@ -598,7 +650,6 @@ poetry run pytest tests/test_smoothgrad.py::TestSmoothGradBasic -v
|
|
|
598
650
|
- [x] PyTorch adapter with gradient support
|
|
599
651
|
|
|
600
652
|
### In Progress 🚧
|
|
601
|
-
- [ ] TCAV (Testing with Concept Activation Vectors)
|
|
602
653
|
- [ ] Layer-wise Relevance Propagation (LRP)
|
|
603
654
|
|
|
604
655
|
### Planned 📋
|
|
@@ -620,7 +671,7 @@ If you use Explainiverse in your research, please cite:
|
|
|
620
671
|
author = {Syed, Muntaser},
|
|
621
672
|
year = {2025},
|
|
622
673
|
url = {https://github.com/jemsbhai/explainiverse},
|
|
623
|
-
version = {0.
|
|
674
|
+
version = {0.7.1}
|
|
624
675
|
}
|
|
625
676
|
```
|
|
626
677
|
|
|
@@ -648,5 +699,5 @@ MIT License - see [LICENSE](LICENSE) for details.
|
|
|
648
699
|
|
|
649
700
|
## Acknowledgments
|
|
650
701
|
|
|
651
|
-
Explainiverse builds upon the foundational work of many researchers in the XAI community. We thank the authors of LIME, SHAP, Integrated Gradients, DeepLIFT, GradCAM, Anchors, DiCE, ALE, SAGE, and ProtoDash for their contributions to interpretable machine learning.
|
|
702
|
+
Explainiverse builds upon the foundational work of many researchers in the XAI community. We thank the authors of LIME, SHAP, Integrated Gradients, DeepLIFT, GradCAM, TCAV, Anchors, DiCE, ALE, SAGE, and ProtoDash for their contributions to interpretable machine learning.
|
|
652
703
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
[](https://www.python.org/downloads/)
|
|
5
5
|
[](https://opensource.org/licenses/MIT)
|
|
6
6
|
|
|
7
|
-
**Explainiverse** is a unified, extensible Python framework for Explainable AI (XAI). It provides a standardized interface for **
|
|
7
|
+
**Explainiverse** is a unified, extensible Python framework for Explainable AI (XAI). It provides a standardized interface for **17 state-of-the-art explanation methods** across local, global, gradient-based, concept-based, and example-based paradigms, along with **comprehensive evaluation metrics** for assessing explanation quality.
|
|
8
8
|
|
|
9
9
|
---
|
|
10
10
|
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
| Feature | Description |
|
|
14
14
|
|---------|-------------|
|
|
15
|
-
| **
|
|
15
|
+
| **17 Explainers** | LIME, KernelSHAP, TreeSHAP, Integrated Gradients, DeepLIFT, DeepSHAP, SmoothGrad, Saliency Maps, GradCAM/GradCAM++, TCAV, Anchors, Counterfactual, Permutation Importance, PDP, ALE, SAGE, ProtoDash |
|
|
16
16
|
| **8 Evaluation Metrics** | Faithfulness (PGI, PGU, Comprehensiveness, Sufficiency, Correlation) and Stability (RIS, ROS, Lipschitz) |
|
|
17
17
|
| **Unified API** | Consistent `BaseExplainer` interface with standardized `Explanation` output |
|
|
18
18
|
| **Plugin Registry** | Filter explainers by scope, model type, data type; automatic recommendations |
|
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
| **SmoothGrad** | Gradient | [Smilkov et al., 2017](https://arxiv.org/abs/1706.03825) |
|
|
36
36
|
| **Saliency Maps** | Gradient | [Simonyan et al., 2014](https://arxiv.org/abs/1312.6034) |
|
|
37
37
|
| **GradCAM / GradCAM++** | Gradient (CNN) | [Selvaraju et al., 2017](https://arxiv.org/abs/1610.02391) |
|
|
38
|
+
| **TCAV** | Concept-Based | [Kim et al., 2018](https://arxiv.org/abs/1711.11279) |
|
|
38
39
|
| **Anchors** | Rule-Based | [Ribeiro et al., 2018](https://ojs.aaai.org/index.php/AAAI/article/view/11491) |
|
|
39
40
|
| **Counterfactual** | Contrastive | [Mothilal et al., 2020](https://arxiv.org/abs/1905.07697) |
|
|
40
41
|
| **ProtoDash** | Example-Based | [Gurumoorthy et al., 2019](https://arxiv.org/abs/1707.01212) |
|
|
@@ -111,8 +112,8 @@ adapter = SklearnAdapter(model, class_names=iris.target_names.tolist())
|
|
|
111
112
|
# List all available explainers
|
|
112
113
|
print(default_registry.list_explainers())
|
|
113
114
|
# ['lime', 'shap', 'treeshap', 'integrated_gradients', 'deeplift', 'deepshap',
|
|
114
|
-
# 'smoothgrad', 'gradcam', '
|
|
115
|
-
# 'permutation_importance', 'partial_dependence', 'ale', 'sage']
|
|
115
|
+
# 'smoothgrad', 'saliency', 'gradcam', 'tcav', 'anchors', 'counterfactual',
|
|
116
|
+
# 'protodash', 'permutation_importance', 'partial_dependence', 'ale', 'sage']
|
|
116
117
|
|
|
117
118
|
# Create an explainer via registry
|
|
118
119
|
explainer = default_registry.create(
|
|
@@ -286,6 +287,56 @@ heatmap = explanation.explanation_data["heatmap"]
|
|
|
286
287
|
overlay = explainer.get_overlay(original_image, heatmap, alpha=0.5)
|
|
287
288
|
```
|
|
288
289
|
|
|
290
|
+
### TCAV (Concept-Based Explanations)
|
|
291
|
+
|
|
292
|
+
```python
|
|
293
|
+
from explainiverse.explainers.gradient import TCAVExplainer
|
|
294
|
+
|
|
295
|
+
# For neural network models with concept examples
|
|
296
|
+
adapter = PyTorchAdapter(model, task="classification", class_names=class_names)
|
|
297
|
+
|
|
298
|
+
# Create TCAV explainer targeting a specific layer
|
|
299
|
+
explainer = TCAVExplainer(
|
|
300
|
+
model=adapter,
|
|
301
|
+
layer_name="layer3", # Target layer for concept analysis
|
|
302
|
+
class_names=class_names
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
# Learn a concept from examples (e.g., "striped" pattern)
|
|
306
|
+
explainer.learn_concept(
|
|
307
|
+
concept_name="striped",
|
|
308
|
+
concept_examples=striped_images, # Images with stripes
|
|
309
|
+
negative_examples=random_images, # Random images without stripes
|
|
310
|
+
min_accuracy=0.6 # Minimum CAV classifier accuracy
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
# Compute TCAV score: fraction of inputs where concept positively influences prediction
|
|
314
|
+
tcav_score = explainer.compute_tcav_score(
|
|
315
|
+
test_inputs=test_images,
|
|
316
|
+
target_class=0, # e.g., "zebra"
|
|
317
|
+
concept_name="striped"
|
|
318
|
+
)
|
|
319
|
+
print(f"TCAV score: {tcav_score:.3f}") # >0.5 means concept positively influences class
|
|
320
|
+
|
|
321
|
+
# Statistical significance testing against random concepts
|
|
322
|
+
result = explainer.statistical_significance_test(
|
|
323
|
+
test_inputs=test_images,
|
|
324
|
+
target_class=0,
|
|
325
|
+
concept_name="striped",
|
|
326
|
+
n_random=10,
|
|
327
|
+
negative_examples=random_images
|
|
328
|
+
)
|
|
329
|
+
print(f"p-value: {result['p_value']:.4f}, significant: {result['significant']}")
|
|
330
|
+
|
|
331
|
+
# Full explanation with multiple concepts
|
|
332
|
+
explanation = explainer.explain(
|
|
333
|
+
test_inputs=test_images,
|
|
334
|
+
target_class=0,
|
|
335
|
+
run_significance_test=True
|
|
336
|
+
)
|
|
337
|
+
print(explanation.explanation_data["tcav_scores"])
|
|
338
|
+
```
|
|
339
|
+
|
|
289
340
|
---
|
|
290
341
|
|
|
291
342
|
## Example-Based Explanations
|
|
@@ -520,7 +571,7 @@ explainiverse/
|
|
|
520
571
|
│ └── pytorch_adapter.py # With gradient support
|
|
521
572
|
├── explainers/
|
|
522
573
|
│ ├── attribution/ # LIME, SHAP, TreeSHAP
|
|
523
|
-
│ ├── gradient/ # IG, DeepLIFT, DeepSHAP, SmoothGrad, GradCAM
|
|
574
|
+
│ ├── gradient/ # IG, DeepLIFT, DeepSHAP, SmoothGrad, Saliency, GradCAM, TCAV
|
|
524
575
|
│ ├── rule_based/ # Anchors
|
|
525
576
|
│ ├── counterfactual/ # DiCE-style
|
|
526
577
|
│ ├── global_explainers/ # Permutation, PDP, ALE, SAGE
|
|
@@ -558,6 +609,7 @@ poetry run pytest tests/test_smoothgrad.py::TestSmoothGradBasic -v
|
|
|
558
609
|
- [x] Core framework (BaseExplainer, Explanation, Registry)
|
|
559
610
|
- [x] Perturbation methods: LIME, KernelSHAP, TreeSHAP
|
|
560
611
|
- [x] Gradient methods: Integrated Gradients, DeepLIFT, DeepSHAP, SmoothGrad, Saliency Maps, GradCAM/GradCAM++
|
|
612
|
+
- [x] Concept-based: TCAV (Testing with Concept Activation Vectors)
|
|
561
613
|
- [x] Rule-based: Anchors
|
|
562
614
|
- [x] Counterfactual: DiCE-style
|
|
563
615
|
- [x] Global: Permutation Importance, PDP, ALE, SAGE
|
|
@@ -567,7 +619,6 @@ poetry run pytest tests/test_smoothgrad.py::TestSmoothGradBasic -v
|
|
|
567
619
|
- [x] PyTorch adapter with gradient support
|
|
568
620
|
|
|
569
621
|
### In Progress 🚧
|
|
570
|
-
- [ ] TCAV (Testing with Concept Activation Vectors)
|
|
571
622
|
- [ ] Layer-wise Relevance Propagation (LRP)
|
|
572
623
|
|
|
573
624
|
### Planned 📋
|
|
@@ -589,7 +640,7 @@ If you use Explainiverse in your research, please cite:
|
|
|
589
640
|
author = {Syed, Muntaser},
|
|
590
641
|
year = {2025},
|
|
591
642
|
url = {https://github.com/jemsbhai/explainiverse},
|
|
592
|
-
version = {0.
|
|
643
|
+
version = {0.7.1}
|
|
593
644
|
}
|
|
594
645
|
```
|
|
595
646
|
|
|
@@ -617,4 +668,4 @@ MIT License - see [LICENSE](LICENSE) for details.
|
|
|
617
668
|
|
|
618
669
|
## Acknowledgments
|
|
619
670
|
|
|
620
|
-
Explainiverse builds upon the foundational work of many researchers in the XAI community. We thank the authors of LIME, SHAP, Integrated Gradients, DeepLIFT, GradCAM, Anchors, DiCE, ALE, SAGE, and ProtoDash for their contributions to interpretable machine learning.
|
|
671
|
+
Explainiverse builds upon the foundational work of many researchers in the XAI community. We thank the authors of LIME, SHAP, Integrated Gradients, DeepLIFT, GradCAM, TCAV, Anchors, DiCE, ALE, SAGE, and ProtoDash for their contributions to interpretable machine learning.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "explainiverse"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.7.1"
|
|
4
4
|
description = "Unified, extensible explainability framework supporting LIME, SHAP, Anchors, Counterfactuals, PDP, ALE, SAGE, and more"
|
|
5
5
|
authors = ["Muntaser Syed <jemsbhai@gmail.com>"]
|
|
6
6
|
license = "MIT"
|
|
@@ -25,7 +25,7 @@ Example:
|
|
|
25
25
|
"""
|
|
26
26
|
|
|
27
27
|
import numpy as np
|
|
28
|
-
from typing import List, Optional, Union,
|
|
28
|
+
from typing import List, Optional, Union, Tuple
|
|
29
29
|
|
|
30
30
|
from .base_adapter import BaseModelAdapter
|
|
31
31
|
|
|
@@ -57,6 +57,11 @@ class PyTorchAdapter(BaseModelAdapter):
|
|
|
57
57
|
explainability methods. Handles device management, tensor/numpy
|
|
58
58
|
conversions, and supports both classification and regression tasks.
|
|
59
59
|
|
|
60
|
+
Supports:
|
|
61
|
+
- Multi-class classification (output shape: [batch, n_classes])
|
|
62
|
+
- Binary classification (output shape: [batch, 1] or [batch])
|
|
63
|
+
- Regression (output shape: [batch, n_outputs] or [batch])
|
|
64
|
+
|
|
60
65
|
Attributes:
|
|
61
66
|
model: The PyTorch model (nn.Module)
|
|
62
67
|
task: "classification" or "regression"
|
|
@@ -150,11 +155,27 @@ class PyTorchAdapter(BaseModelAdapter):
|
|
|
150
155
|
def _apply_activation(self, output: "torch.Tensor") -> "torch.Tensor":
|
|
151
156
|
"""Apply output activation function."""
|
|
152
157
|
if self.output_activation == "softmax":
|
|
158
|
+
# Handle different output shapes
|
|
159
|
+
if output.dim() == 1 or (output.dim() == 2 and output.shape[1] == 1):
|
|
160
|
+
# Binary: apply sigmoid instead of softmax
|
|
161
|
+
return torch.sigmoid(output)
|
|
153
162
|
return torch.softmax(output, dim=-1)
|
|
154
163
|
elif self.output_activation == "sigmoid":
|
|
155
164
|
return torch.sigmoid(output)
|
|
156
165
|
return output
|
|
157
166
|
|
|
167
|
+
def _normalize_output_shape(self, output: "torch.Tensor") -> "torch.Tensor":
|
|
168
|
+
"""
|
|
169
|
+
Normalize output to consistent 2D shape (batch, outputs).
|
|
170
|
+
|
|
171
|
+
Handles:
|
|
172
|
+
- (batch,) -> (batch, 1)
|
|
173
|
+
- (batch, n) -> (batch, n)
|
|
174
|
+
"""
|
|
175
|
+
if output.dim() == 1:
|
|
176
|
+
return output.unsqueeze(-1)
|
|
177
|
+
return output
|
|
178
|
+
|
|
158
179
|
def predict(self, data: np.ndarray) -> np.ndarray:
|
|
159
180
|
"""
|
|
160
181
|
Generate predictions for input data.
|
|
@@ -183,16 +204,66 @@ class PyTorchAdapter(BaseModelAdapter):
|
|
|
183
204
|
tensor_batch = self._to_tensor(batch)
|
|
184
205
|
|
|
185
206
|
output = self.model(tensor_batch)
|
|
207
|
+
output = self._normalize_output_shape(output)
|
|
186
208
|
output = self._apply_activation(output)
|
|
187
209
|
outputs.append(self._to_numpy(output))
|
|
188
210
|
|
|
189
211
|
return np.vstack(outputs)
|
|
190
212
|
|
|
213
|
+
def _get_target_scores(
|
|
214
|
+
self,
|
|
215
|
+
output: "torch.Tensor",
|
|
216
|
+
target_class: Optional[Union[int, "torch.Tensor"]] = None
|
|
217
|
+
) -> "torch.Tensor":
|
|
218
|
+
"""
|
|
219
|
+
Extract target scores for gradient computation.
|
|
220
|
+
|
|
221
|
+
Handles both multi-class and binary classification outputs.
|
|
222
|
+
|
|
223
|
+
Args:
|
|
224
|
+
output: Raw model output (logits)
|
|
225
|
+
target_class: Target class index or tensor of indices
|
|
226
|
+
|
|
227
|
+
Returns:
|
|
228
|
+
Target scores tensor for backpropagation
|
|
229
|
+
"""
|
|
230
|
+
batch_size = output.shape[0]
|
|
231
|
+
|
|
232
|
+
# Normalize to 2D
|
|
233
|
+
if output.dim() == 1:
|
|
234
|
+
output = output.unsqueeze(-1)
|
|
235
|
+
|
|
236
|
+
n_outputs = output.shape[1]
|
|
237
|
+
|
|
238
|
+
if self.task == "classification":
|
|
239
|
+
if n_outputs == 1:
|
|
240
|
+
# Binary classification with single logit
|
|
241
|
+
# Score is the logit itself (positive class score)
|
|
242
|
+
return output.squeeze(-1)
|
|
243
|
+
else:
|
|
244
|
+
# Multi-class classification
|
|
245
|
+
if target_class is None:
|
|
246
|
+
target_class = output.argmax(dim=-1)
|
|
247
|
+
elif isinstance(target_class, int):
|
|
248
|
+
target_class = torch.tensor(
|
|
249
|
+
[target_class] * batch_size,
|
|
250
|
+
device=self.device
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
# Gather scores for target class
|
|
254
|
+
return output.gather(1, target_class.view(-1, 1)).squeeze(-1)
|
|
255
|
+
else:
|
|
256
|
+
# Regression: use first output or sum of outputs
|
|
257
|
+
if n_outputs == 1:
|
|
258
|
+
return output.squeeze(-1)
|
|
259
|
+
else:
|
|
260
|
+
return output.sum(dim=-1)
|
|
261
|
+
|
|
191
262
|
def predict_with_gradients(
|
|
192
263
|
self,
|
|
193
264
|
data: np.ndarray,
|
|
194
265
|
target_class: Optional[int] = None
|
|
195
|
-
) ->
|
|
266
|
+
) -> Tuple[np.ndarray, np.ndarray]:
|
|
196
267
|
"""
|
|
197
268
|
Generate predictions and compute gradients w.r.t. inputs.
|
|
198
269
|
|
|
@@ -203,11 +274,17 @@ class PyTorchAdapter(BaseModelAdapter):
|
|
|
203
274
|
data: Input data as numpy array.
|
|
204
275
|
target_class: Class index for gradient computation.
|
|
205
276
|
If None, uses the predicted class.
|
|
277
|
+
For binary classification with single output,
|
|
278
|
+
this is ignored (gradient w.r.t. the single logit).
|
|
206
279
|
|
|
207
280
|
Returns:
|
|
208
281
|
Tuple of (predictions, gradients) as numpy arrays.
|
|
282
|
+
- predictions: (batch, n_classes) probabilities
|
|
283
|
+
- gradients: same shape as input data
|
|
209
284
|
"""
|
|
210
285
|
data = np.array(data)
|
|
286
|
+
original_shape = data.shape
|
|
287
|
+
|
|
211
288
|
if data.ndim == 1:
|
|
212
289
|
data = data.reshape(1, -1)
|
|
213
290
|
|
|
@@ -217,20 +294,13 @@ class PyTorchAdapter(BaseModelAdapter):
|
|
|
217
294
|
|
|
218
295
|
# Forward pass
|
|
219
296
|
output = self.model(tensor_data)
|
|
220
|
-
activated_output = self._apply_activation(output)
|
|
221
297
|
|
|
222
|
-
#
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
# Select target class scores for gradient
|
|
230
|
-
target_scores = output.gather(1, target_class.view(-1, 1)).squeeze()
|
|
231
|
-
else:
|
|
232
|
-
# Regression: gradient w.r.t. output
|
|
233
|
-
target_scores = output.squeeze()
|
|
298
|
+
# Get activated output for return
|
|
299
|
+
output_normalized = self._normalize_output_shape(output)
|
|
300
|
+
activated_output = self._apply_activation(output_normalized)
|
|
301
|
+
|
|
302
|
+
# Get target scores for gradient computation
|
|
303
|
+
target_scores = self._get_target_scores(output, target_class)
|
|
234
304
|
|
|
235
305
|
# Backward pass
|
|
236
306
|
if target_scores.dim() == 0:
|
|
@@ -295,7 +365,7 @@ class PyTorchAdapter(BaseModelAdapter):
|
|
|
295
365
|
data: np.ndarray,
|
|
296
366
|
layer_name: str,
|
|
297
367
|
target_class: Optional[int] = None
|
|
298
|
-
) ->
|
|
368
|
+
) -> Tuple[np.ndarray, np.ndarray]:
|
|
299
369
|
"""
|
|
300
370
|
Get gradients of output w.r.t. a specific layer's activations.
|
|
301
371
|
|
|
@@ -339,15 +409,8 @@ class PyTorchAdapter(BaseModelAdapter):
|
|
|
339
409
|
|
|
340
410
|
output = self.model(tensor_data)
|
|
341
411
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
target_class = output.argmax(dim=-1)
|
|
345
|
-
elif isinstance(target_class, int):
|
|
346
|
-
target_class = torch.tensor([target_class] * data.shape[0], device=self.device)
|
|
347
|
-
|
|
348
|
-
target_scores = output.gather(1, target_class.view(-1, 1)).squeeze()
|
|
349
|
-
else:
|
|
350
|
-
target_scores = output.squeeze()
|
|
412
|
+
# Get target scores using the new method
|
|
413
|
+
target_scores = self._get_target_scores(output, target_class)
|
|
351
414
|
|
|
352
415
|
if target_scores.dim() == 0:
|
|
353
416
|
target_scores.backward()
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# src/explainiverse/core/explanation.py
|
|
2
|
+
"""
|
|
3
|
+
Unified container for explanation results.
|
|
4
|
+
|
|
5
|
+
The Explanation class provides a standardized format for all explainer outputs,
|
|
6
|
+
enabling consistent handling across different explanation methods.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from typing import Dict, List, Optional, Any
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Explanation:
|
|
13
|
+
"""
|
|
14
|
+
Unified container for explanation results.
|
|
15
|
+
|
|
16
|
+
Attributes:
|
|
17
|
+
explainer_name: Name of the explainer that generated this explanation
|
|
18
|
+
target_class: The class/output being explained
|
|
19
|
+
explanation_data: Dictionary containing explanation details
|
|
20
|
+
(e.g., feature_attributions, heatmaps, rules)
|
|
21
|
+
feature_names: Optional list of feature names for index resolution
|
|
22
|
+
metadata: Optional additional metadata about the explanation
|
|
23
|
+
|
|
24
|
+
Example:
|
|
25
|
+
>>> explanation = Explanation(
|
|
26
|
+
... explainer_name="LIME",
|
|
27
|
+
... target_class="cat",
|
|
28
|
+
... explanation_data={"feature_attributions": {"fur": 0.8, "whiskers": 0.6}},
|
|
29
|
+
... feature_names=["fur", "whiskers", "tail", "ears"]
|
|
30
|
+
... )
|
|
31
|
+
>>> print(explanation.get_top_features(k=2))
|
|
32
|
+
[('fur', 0.8), ('whiskers', 0.6)]
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(
|
|
36
|
+
self,
|
|
37
|
+
explainer_name: str,
|
|
38
|
+
target_class: str,
|
|
39
|
+
explanation_data: Dict[str, Any],
|
|
40
|
+
feature_names: Optional[List[str]] = None,
|
|
41
|
+
metadata: Optional[Dict[str, Any]] = None
|
|
42
|
+
):
|
|
43
|
+
"""
|
|
44
|
+
Initialize an Explanation object.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
explainer_name: Name of the explainer (e.g., "LIME", "SHAP")
|
|
48
|
+
target_class: The target class or output being explained
|
|
49
|
+
explanation_data: Dictionary containing the explanation details.
|
|
50
|
+
Common keys include:
|
|
51
|
+
- "feature_attributions": Dict[str, float] mapping feature names to importance
|
|
52
|
+
- "attributions_raw": List[float] of raw attribution values
|
|
53
|
+
- "heatmap": np.ndarray for image explanations
|
|
54
|
+
- "rules": List of rule strings for rule-based explanations
|
|
55
|
+
feature_names: Optional list of feature names. If provided, enables
|
|
56
|
+
index-based lookup in evaluation metrics.
|
|
57
|
+
metadata: Optional additional metadata (e.g., computation time, parameters)
|
|
58
|
+
"""
|
|
59
|
+
self.explainer_name = explainer_name
|
|
60
|
+
self.target_class = target_class
|
|
61
|
+
self.explanation_data = explanation_data
|
|
62
|
+
self.feature_names = list(feature_names) if feature_names is not None else None
|
|
63
|
+
self.metadata = metadata or {}
|
|
64
|
+
|
|
65
|
+
def __repr__(self):
|
|
66
|
+
n_features = len(self.feature_names) if self.feature_names else "N/A"
|
|
67
|
+
return (
|
|
68
|
+
f"Explanation(explainer='{self.explainer_name}', "
|
|
69
|
+
f"target='{self.target_class}', "
|
|
70
|
+
f"keys={list(self.explanation_data.keys())}, "
|
|
71
|
+
f"n_features={n_features})"
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
def get_attributions(self) -> Optional[Dict[str, float]]:
|
|
75
|
+
"""
|
|
76
|
+
Get feature attributions if available.
|
|
77
|
+
|
|
78
|
+
Returns:
|
|
79
|
+
Dictionary mapping feature names to attribution values,
|
|
80
|
+
or None if not available.
|
|
81
|
+
"""
|
|
82
|
+
return self.explanation_data.get("feature_attributions")
|
|
83
|
+
|
|
84
|
+
def get_top_features(self, k: int = 5, absolute: bool = True) -> List[tuple]:
|
|
85
|
+
"""
|
|
86
|
+
Get the top-k most important features.
|
|
87
|
+
|
|
88
|
+
Args:
|
|
89
|
+
k: Number of top features to return
|
|
90
|
+
absolute: If True, rank by absolute value of attribution
|
|
91
|
+
|
|
92
|
+
Returns:
|
|
93
|
+
List of (feature_name, attribution_value) tuples sorted by importance
|
|
94
|
+
"""
|
|
95
|
+
attributions = self.get_attributions()
|
|
96
|
+
if not attributions:
|
|
97
|
+
return []
|
|
98
|
+
|
|
99
|
+
if absolute:
|
|
100
|
+
sorted_items = sorted(
|
|
101
|
+
attributions.items(),
|
|
102
|
+
key=lambda x: abs(x[1]),
|
|
103
|
+
reverse=True
|
|
104
|
+
)
|
|
105
|
+
else:
|
|
106
|
+
sorted_items = sorted(
|
|
107
|
+
attributions.items(),
|
|
108
|
+
key=lambda x: x[1],
|
|
109
|
+
reverse=True
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
return sorted_items[:k]
|
|
113
|
+
|
|
114
|
+
def get_feature_index(self, feature_name: str) -> Optional[int]:
|
|
115
|
+
"""
|
|
116
|
+
Get the index of a feature by name.
|
|
117
|
+
|
|
118
|
+
Args:
|
|
119
|
+
feature_name: Name of the feature
|
|
120
|
+
|
|
121
|
+
Returns:
|
|
122
|
+
Index of the feature, or None if not found or feature_names not set
|
|
123
|
+
"""
|
|
124
|
+
if self.feature_names is None:
|
|
125
|
+
return None
|
|
126
|
+
try:
|
|
127
|
+
return self.feature_names.index(feature_name)
|
|
128
|
+
except ValueError:
|
|
129
|
+
return None
|
|
130
|
+
|
|
131
|
+
def plot(self, plot_type: str = 'bar', **kwargs):
|
|
132
|
+
"""
|
|
133
|
+
Visualize the explanation.
|
|
134
|
+
|
|
135
|
+
Args:
|
|
136
|
+
plot_type: Type of plot ('bar', 'waterfall', 'heatmap')
|
|
137
|
+
**kwargs: Additional arguments passed to the plotting function
|
|
138
|
+
|
|
139
|
+
Note:
|
|
140
|
+
This is a placeholder for future visualization integration.
|
|
141
|
+
"""
|
|
142
|
+
print(
|
|
143
|
+
f"[plot: {plot_type}] Plotting explanation for {self.target_class} "
|
|
144
|
+
f"from {self.explainer_name}."
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
148
|
+
"""
|
|
149
|
+
Convert explanation to a dictionary for serialization.
|
|
150
|
+
|
|
151
|
+
Returns:
|
|
152
|
+
Dictionary representation of the explanation
|
|
153
|
+
"""
|
|
154
|
+
return {
|
|
155
|
+
"explainer_name": self.explainer_name,
|
|
156
|
+
"target_class": self.target_class,
|
|
157
|
+
"explanation_data": self.explanation_data,
|
|
158
|
+
"feature_names": self.feature_names,
|
|
159
|
+
"metadata": self.metadata
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
@classmethod
|
|
163
|
+
def from_dict(cls, data: Dict[str, Any]) -> "Explanation":
|
|
164
|
+
"""
|
|
165
|
+
Create an Explanation from a dictionary.
|
|
166
|
+
|
|
167
|
+
Args:
|
|
168
|
+
data: Dictionary with explanation data
|
|
169
|
+
|
|
170
|
+
Returns:
|
|
171
|
+
Explanation instance
|
|
172
|
+
"""
|
|
173
|
+
return cls(
|
|
174
|
+
explainer_name=data["explainer_name"],
|
|
175
|
+
target_class=data["target_class"],
|
|
176
|
+
explanation_data=data["explanation_data"],
|
|
177
|
+
feature_names=data.get("feature_names"),
|
|
178
|
+
metadata=data.get("metadata", {})
|
|
179
|
+
)
|