hypershap 0.0.3__tar.gz → 0.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hypershap-0.0.3 → hypershap-0.0.4}/CHANGELOG.md +4 -0
- hypershap-0.0.4/PKG-INFO +177 -0
- hypershap-0.0.4/README.md +151 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/pyproject.toml +1 -1
- {hypershap-0.0.3 → hypershap-0.0.4}/src/hypershap/hypershap.py +5 -2
- {hypershap-0.0.3 → hypershap-0.0.4}/src/hypershap/surrogate_model.py +3 -1
- {hypershap-0.0.3 → hypershap-0.0.4}/src/hypershap/task.py +10 -3
- {hypershap-0.0.3 → hypershap-0.0.4}/src/hypershap/utils.py +7 -2
- {hypershap-0.0.3 → hypershap-0.0.4}/tests/fixtures/simple_setup.py +1 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/tests/test_extended_settings.py +8 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/uv.lock +7 -1
- hypershap-0.0.3/PKG-INFO +0 -169
- hypershap-0.0.3/README.md +0 -143
- {hypershap-0.0.3 → hypershap-0.0.4}/.devcontainer/devcontainer.json +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/.devcontainer/postCreateCommand.sh +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/.github/actions/setup-python-env/action.yml +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/.github/workflows/main.yml +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/.github/workflows/on-release-main.yml +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/.github/workflows/validate-codecov-config.yml +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/.gitignore +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/.pre-commit-config.yaml +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/CONTRIBUTING.md +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/LICENSE +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/Makefile +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/codecov.yaml +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/docs/api/games.md +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/docs/api/hypershap.md +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/docs/api/surrogate_model.md +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/docs/api/task.md +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/docs/api/utils.md +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/docs/authors.md +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/docs/cite.md +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/docs/contribute.md +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/docs/getting_started.md +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/docs/help.md +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/docs/index.md +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/docs/license.md +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/examples/ablation-example.ipynb +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/examples/carps-benchmark-example.ipynb +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/examples/explanation-task-example.ipynb +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/examples/optimizerbias-example.ipynb +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/examples/tunability-example.ipynb +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/mkdocs.yml +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/src/hypershap/__init__.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/src/hypershap/games/__init__.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/src/hypershap/games/ablation.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/src/hypershap/games/abstract.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/src/hypershap/games/multi_data.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/src/hypershap/games/optimizerbias.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/src/hypershap/games/tunability.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/tests/__init__.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/tests/conftest.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/tests/fixtures/__init__.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/tests/fixtures/large_setup.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/tests/fixtures/multi_data_setup.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/tests/test_ablation_game.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/tests/test_games.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/tests/test_hypershap.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/tests/test_surrogate_model.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/tests/test_task.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/tests/test_tunability_game.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/tests/test_utils.py +0 -0
- {hypershap-0.0.3 → hypershap-0.0.4}/tox.ini +0 -0
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# v0.0.4
|
|
2
|
+
- Added pseudorandomization
|
|
3
|
+
- Added index-specific approximation
|
|
4
|
+
|
|
1
5
|
# v0.0.3
|
|
2
6
|
- Added multi-baseline ablation game. This game computes ablation paths with respect to multiple baseline configurations and aggregates values for different paths via mean, min, max or variance.
|
|
3
7
|
- Added waterfall plots to the HyperSHAP interface.
|
hypershap-0.0.4/PKG-INFO
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hypershap
|
|
3
|
+
Version: 0.0.4
|
|
4
|
+
Summary: HyperSHAP is a post-hoc explanation method for hyperparameter optimization.
|
|
5
|
+
Project-URL: Homepage, https://automl.github.io/hypershap/
|
|
6
|
+
Project-URL: Repository, https://github.com/automl/hypershap
|
|
7
|
+
Project-URL: Documentation, https://automl.github.io/hypershap/
|
|
8
|
+
Author-email: Marcel Wever <m.wever@ai.uni-hannover.de>
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: python
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Requires-Python: <4.0,>=3.10
|
|
20
|
+
Requires-Dist: matplotlib>=3.10.5
|
|
21
|
+
Requires-Dist: networkx>=3.4.2
|
|
22
|
+
Requires-Dist: numpy>=2.2.6
|
|
23
|
+
Requires-Dist: scikit-learn>=1.7.1
|
|
24
|
+
Requires-Dist: shapiq>=1.2.3
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# HyperSHAP <img src="https://raw.githubusercontent.com/automl/hypershap/main/docs/source/_static/logo/hypershap-logo.png" alt="HyperSHAP Logo" align="right" height="200px"/>
|
|
28
|
+
|
|
29
|
+
[](https://img.shields.io/github/v/release/automl/hypershap)
|
|
30
|
+
[](https://github.com/automl/hypershap/actions/workflows/main.yml?query=branch%3Amain)
|
|
31
|
+
[](https://coveralls.io/github/automl/HyperSHAP?branch=dev)
|
|
32
|
+
[](https://img.shields.io/github/commit-activity/m/automl/hypershap)
|
|
33
|
+
[](https://img.shields.io/github/license/automl/hypershap)
|
|
34
|
+
|
|
35
|
+
HyperSHAP – a game‑theoretic Python library for explaining Hyperparameter Optimization (HPO). It uses Shapley values and interaction indices to provide both local and global insights into how individual hyper‑parameters (and their interactions) affect a model’s performance.
|
|
36
|
+
|
|
37
|
+
- **Github repository**: <https://github.com/automl/HyperSHAP/>
|
|
38
|
+
- **Documentation** <https://automl.github.io/HyperSHAP/>
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
## Table of Contents
|
|
42
|
+
- [Features](#features)
|
|
43
|
+
- [Installation](#installation)
|
|
44
|
+
- [Getting Started](#getting-started)
|
|
45
|
+
- [API Overview](#api-overview)
|
|
46
|
+
- [Example Notebook](#example-notebook)
|
|
47
|
+
- [Citation](#citation)
|
|
48
|
+
- [Contributing](#contributing)
|
|
49
|
+
- [License](#license)
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## Features
|
|
54
|
+
- **Additive Shapley decomposition** of any performance metric across hyper‑parameters.
|
|
55
|
+
- **Interaction analysis** via the Faithful Shapley Interaction Index (FSII).
|
|
56
|
+
- Ready‑made explanation tasks for **Ablation**, **Tunability**, and **Optimizer Bias** studies.
|
|
57
|
+
- Integrated **visualisation** (SI‑graph) for interaction effects.
|
|
58
|
+
- Works with any surrogate model that follows the `ExplanationTask` interface.
|
|
59
|
+
|
|
60
|
+
---
|
|
61
|
+
|
|
62
|
+
## Installation
|
|
63
|
+
|
|
64
|
+
```sh
|
|
65
|
+
$ make install
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Getting Started
|
|
69
|
+
Given an existing setup with a ConfigurationSpace from the [ConfigSpace package](https://github.com/automl/ConfigSpace) and black-box function as follows:
|
|
70
|
+
```Python
|
|
71
|
+
from ConfigSpace import ConfigurationSpace, Configuration
|
|
72
|
+
|
|
73
|
+
# ConfigurationSpace describing the hyperparameter space
|
|
74
|
+
cs = ConfigurationSpace()
|
|
75
|
+
...
|
|
76
|
+
|
|
77
|
+
# A black-box function, evaluating ConfigSpace.Configuration objects
|
|
78
|
+
def blackbox_function(cfg: Configuration) -> float:
|
|
79
|
+
...
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
You can use HyperSHAP as follows:
|
|
83
|
+
```Python
|
|
84
|
+
from hypershap import ExplanationTask, HyperSHAP
|
|
85
|
+
|
|
86
|
+
# Instantiate HyperSHAP
|
|
87
|
+
hypershap = HyperSHAP(ExplanationTask.from_function(config_space=cs,function=blackbox_function))
|
|
88
|
+
# Conduct tunability analysis
|
|
89
|
+
hypershap.tunability(baseline_config=cs.get_default_configuration())
|
|
90
|
+
# Plot results as a Shapley Interaction graph
|
|
91
|
+
hypershap.plot_si_graph()
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
The example demonstrates how to:
|
|
95
|
+
1. Wrap a black-box function in an explanation task.
|
|
96
|
+
2. Use `HyperSHAP` to obtain interaction values for the **tunability** game.
|
|
97
|
+
3. Plot the corresponding SI-graph.
|
|
98
|
+
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
## API Overview
|
|
102
|
+
|
|
103
|
+
| Method | Purpose |
|
|
104
|
+
|---------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
105
|
+
| `HyperSHAP(explanation_task, n_workers, max_hyperparameters_exact, approximation_budget)` | Initialize the explainer with a generic `ExplanationTask`. Optionally, you may activate parallelization by setting n_workers to the number of CPU cores you would like to use for parallelization. `max_hyperparameters_exact` determines until which number of hyperparameters exact Shapley values will be computed and beyond which a budget of `approximation_budget` will be used to approximate them. |
|
|
106
|
+
| `ablation(config_of_interest, baseline_config, index="FSII", order=2)` | Explain the contribution of each hyperparameter value (and interactions) when moving from a baseline to a specific configuration. |
|
|
107
|
+
| `ablation_multibaseline(config_of_interest, baseline_config, aggregation, index="FSII", order=2)` | Explain the contribution of each hyperparameter value (and interactions) when moving from different baselines to a specific configuration. Values are aggregated via a given `aggregation` operator. |
|
|
108
|
+
| `tunability(baseline_config=None, index="FSII", order=2, n_samples=10_000)` | Quantify how much performance can be gained by tuning subsets of hyperparameters. |
|
|
109
|
+
| `sensitivity(baseline_config=None, index="FSII", order=2, n_samples=10_000)` | Quantify how much performance variance can be gained by varying subsets of hyperparameters. |
|
|
110
|
+
| `mistunability(baseline_config=None, index="FSII", order=2, n_samples=10_000)` | Quantify how much performance can be lost due to mistuning a (subsets of) hyperparameter(s). |
|
|
111
|
+
| `optimizer_bias(optimizer_of_interest, optimizer_ensemble, index="FSII", order=2)` | Attribute performance differences to a particular optimizer vs. an ensemble of optimizers. |
|
|
112
|
+
| `plot_si_graph(interaction_values=None, save_path=None)` | Plot the Shapley Interaction (SI) graph; uses the most recent interaction values if none are supplied. |
|
|
113
|
+
| `plot_upset(interaction_values=None, save_path=None)` | Plot interaction values in the form of an upset plot; uses the most recent interaction values if none are supplied. |
|
|
114
|
+
| `plot_force(interaction_values=None, save_path=None)` | Plot interaction values in the form of a force plot; uses the most recent interaction values if none are supplied. |
|
|
115
|
+
| `plot_waterfall(interaction_values=None, save_path=None)` | Plot interaction values in the form of a waterfall plot; uses the most recent interaction values if none are supplied. |
|
|
116
|
+
| `plot_stacked_bar(interaction_values=None, save_path=None)` | Plot summaries of interaction values as stacked bar charts with a bar per interaction order; uses the most recent interaction values if none are supplied. |
|
|
117
|
+
| `get_interaction_values_with_names(interaction_values=None)` | Get interaction values dictionary with names of the hyperparameters instead of IDs. |
|
|
118
|
+
| `ExplanationTask.get_hyperparameter_names()` | Helper to retrieve ordered hyper‑parameter names (used for visualisation). |
|
|
119
|
+
|
|
120
|
+
All methods return an `InteractionValues` object (from **shapiq**) that can be inspected, saved, or passed to the visualisation routine.
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
## Example Notebooks
|
|
125
|
+
Full Jupyter notebooks illustrating all three explanation tasks (ablation, tunability, optimizer bias) are included in the repository under `examples/`. The notebookts walk through:
|
|
126
|
+
|
|
127
|
+
- Building a mockup environment
|
|
128
|
+
- Creating the corresponding explanation task
|
|
129
|
+
- Loading explanation tasks from different setups: data, black-box function, and existing surrogate model.
|
|
130
|
+
- Computing interaction values with HyperSHAP
|
|
131
|
+
- Visualizing results with `plot_si_graph`
|
|
132
|
+
|
|
133
|
+
---
|
|
134
|
+
|
|
135
|
+
## Citation
|
|
136
|
+
If you use HyperSHAP in your research, please cite the original paper:
|
|
137
|
+
|
|
138
|
+
```bibtex
|
|
139
|
+
@article{wever-arxiv25,
|
|
140
|
+
author = {Marcel Wever and
|
|
141
|
+
Maximilian Muschalik and
|
|
142
|
+
Fabian Fumagalli and
|
|
143
|
+
Marius Lindauer},
|
|
144
|
+
title = {HyperSHAP: Shapley Values and Interactions for Hyperparameter Importance},
|
|
145
|
+
journal = {CoRR},
|
|
146
|
+
volume = {abs/2502.01276},
|
|
147
|
+
year = {2025},
|
|
148
|
+
doi = {10.48550/ARXIV.2502.01276},
|
|
149
|
+
}
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
The paper introduces the underlying game-theoretic framework and demonstrates its usefulness for HPO explainability.
|
|
153
|
+
|
|
154
|
+
## Contributing
|
|
155
|
+
|
|
156
|
+
Contributions are welcome! Please follow these steps:
|
|
157
|
+
|
|
158
|
+
1. Fork the repo and create a feature branch (git checkout -b feat/your-feature).
|
|
159
|
+
2. Write tests (the project uses pytest).
|
|
160
|
+
3. Ensure all tests pass (pytest).
|
|
161
|
+
4. Update documentation if you add new functionality.
|
|
162
|
+
5. Submit a Pull Request with a clear description of the changes.
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
See CONTRIBUTING.md for detailed guidelines.
|
|
166
|
+
|
|
167
|
+
---
|
|
168
|
+
|
|
169
|
+
## License
|
|
170
|
+
HyperSHAP is released under the BSD 3-Clause License. See the `LICENSE` file for full terms.
|
|
171
|
+
|
|
172
|
+
---
|
|
173
|
+
|
|
174
|
+
**Enjoy exploring your HPO pipelines with HyperSHAP!** 🎉
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
Repository initiated with [fpgmaas/cookiecutter-uv](https://github.com/fpgmaas/cookiecutter-uv).
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
# HyperSHAP <img src="https://raw.githubusercontent.com/automl/hypershap/main/docs/source/_static/logo/hypershap-logo.png" alt="HyperSHAP Logo" align="right" height="200px"/>
|
|
2
|
+
|
|
3
|
+
[](https://img.shields.io/github/v/release/automl/hypershap)
|
|
4
|
+
[](https://github.com/automl/hypershap/actions/workflows/main.yml?query=branch%3Amain)
|
|
5
|
+
[](https://coveralls.io/github/automl/HyperSHAP?branch=dev)
|
|
6
|
+
[](https://img.shields.io/github/commit-activity/m/automl/hypershap)
|
|
7
|
+
[](https://img.shields.io/github/license/automl/hypershap)
|
|
8
|
+
|
|
9
|
+
HyperSHAP – a game‑theoretic Python library for explaining Hyperparameter Optimization (HPO). It uses Shapley values and interaction indices to provide both local and global insights into how individual hyper‑parameters (and their interactions) affect a model’s performance.
|
|
10
|
+
|
|
11
|
+
- **Github repository**: <https://github.com/automl/HyperSHAP/>
|
|
12
|
+
- **Documentation** <https://automl.github.io/HyperSHAP/>
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
## Table of Contents
|
|
16
|
+
- [Features](#features)
|
|
17
|
+
- [Installation](#installation)
|
|
18
|
+
- [Getting Started](#getting-started)
|
|
19
|
+
- [API Overview](#api-overview)
|
|
20
|
+
- [Example Notebook](#example-notebook)
|
|
21
|
+
- [Citation](#citation)
|
|
22
|
+
- [Contributing](#contributing)
|
|
23
|
+
- [License](#license)
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
## Features
|
|
28
|
+
- **Additive Shapley decomposition** of any performance metric across hyper‑parameters.
|
|
29
|
+
- **Interaction analysis** via the Faithful Shapley Interaction Index (FSII).
|
|
30
|
+
- Ready‑made explanation tasks for **Ablation**, **Tunability**, and **Optimizer Bias** studies.
|
|
31
|
+
- Integrated **visualisation** (SI‑graph) for interaction effects.
|
|
32
|
+
- Works with any surrogate model that follows the `ExplanationTask` interface.
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## Installation
|
|
37
|
+
|
|
38
|
+
```sh
|
|
39
|
+
$ make install
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Getting Started
|
|
43
|
+
Given an existing setup with a ConfigurationSpace from the [ConfigSpace package](https://github.com/automl/ConfigSpace) and black-box function as follows:
|
|
44
|
+
```Python
|
|
45
|
+
from ConfigSpace import ConfigurationSpace, Configuration
|
|
46
|
+
|
|
47
|
+
# ConfigurationSpace describing the hyperparameter space
|
|
48
|
+
cs = ConfigurationSpace()
|
|
49
|
+
...
|
|
50
|
+
|
|
51
|
+
# A black-box function, evaluating ConfigSpace.Configuration objects
|
|
52
|
+
def blackbox_function(cfg: Configuration) -> float:
|
|
53
|
+
...
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
You can use HyperSHAP as follows:
|
|
57
|
+
```Python
|
|
58
|
+
from hypershap import ExplanationTask, HyperSHAP
|
|
59
|
+
|
|
60
|
+
# Instantiate HyperSHAP
|
|
61
|
+
hypershap = HyperSHAP(ExplanationTask.from_function(config_space=cs,function=blackbox_function))
|
|
62
|
+
# Conduct tunability analysis
|
|
63
|
+
hypershap.tunability(baseline_config=cs.get_default_configuration())
|
|
64
|
+
# Plot results as a Shapley Interaction graph
|
|
65
|
+
hypershap.plot_si_graph()
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
The example demonstrates how to:
|
|
69
|
+
1. Wrap a black-box function in an explanation task.
|
|
70
|
+
2. Use `HyperSHAP` to obtain interaction values for the **tunability** game.
|
|
71
|
+
3. Plot the corresponding SI-graph.
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
## API Overview
|
|
76
|
+
|
|
77
|
+
| Method | Purpose |
|
|
78
|
+
|---------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
79
|
+
| `HyperSHAP(explanation_task, n_workers, max_hyperparameters_exact, approximation_budget)` | Initialize the explainer with a generic `ExplanationTask`. Optionally, you may activate parallelization by setting n_workers to the number of CPU cores you would like to use for parallelization. `max_hyperparameters_exact` determines until which number of hyperparameters exact Shapley values will be computed and beyond which a budget of `approximation_budget` will be used to approximate them. |
|
|
80
|
+
| `ablation(config_of_interest, baseline_config, index="FSII", order=2)` | Explain the contribution of each hyperparameter value (and interactions) when moving from a baseline to a specific configuration. |
|
|
81
|
+
| `ablation_multibaseline(config_of_interest, baseline_config, aggregation, index="FSII", order=2)` | Explain the contribution of each hyperparameter value (and interactions) when moving from different baselines to a specific configuration. Values are aggregated via a given `aggregation` operator. |
|
|
82
|
+
| `tunability(baseline_config=None, index="FSII", order=2, n_samples=10_000)` | Quantify how much performance can be gained by tuning subsets of hyperparameters. |
|
|
83
|
+
| `sensitivity(baseline_config=None, index="FSII", order=2, n_samples=10_000)` | Quantify how much performance variance can be gained by varying subsets of hyperparameters. |
|
|
84
|
+
| `mistunability(baseline_config=None, index="FSII", order=2, n_samples=10_000)` | Quantify how much performance can be lost due to mistuning a (subsets of) hyperparameter(s). |
|
|
85
|
+
| `optimizer_bias(optimizer_of_interest, optimizer_ensemble, index="FSII", order=2)` | Attribute performance differences to a particular optimizer vs. an ensemble of optimizers. |
|
|
86
|
+
| `plot_si_graph(interaction_values=None, save_path=None)` | Plot the Shapley Interaction (SI) graph; uses the most recent interaction values if none are supplied. |
|
|
87
|
+
| `plot_upset(interaction_values=None, save_path=None)` | Plot interaction values in the form of an upset plot; uses the most recent interaction values if none are supplied. |
|
|
88
|
+
| `plot_force(interaction_values=None, save_path=None)` | Plot interaction values in the form of a force plot; uses the most recent interaction values if none are supplied. |
|
|
89
|
+
| `plot_waterfall(interaction_values=None, save_path=None)` | Plot interaction values in the form of a waterfall plot; uses the most recent interaction values if none are supplied. |
|
|
90
|
+
| `plot_stacked_bar(interaction_values=None, save_path=None)` | Plot summaries of interaction values as stacked bar charts with a bar per interaction order; uses the most recent interaction values if none are supplied. |
|
|
91
|
+
| `get_interaction_values_with_names(interaction_values=None)` | Get interaction values dictionary with names of the hyperparameters instead of IDs. |
|
|
92
|
+
| `ExplanationTask.get_hyperparameter_names()` | Helper to retrieve ordered hyper‑parameter names (used for visualisation). |
|
|
93
|
+
|
|
94
|
+
All methods return an `InteractionValues` object (from **shapiq**) that can be inspected, saved, or passed to the visualisation routine.
|
|
95
|
+
|
|
96
|
+
---
|
|
97
|
+
|
|
98
|
+
## Example Notebooks
|
|
99
|
+
Full Jupyter notebooks illustrating all three explanation tasks (ablation, tunability, optimizer bias) are included in the repository under `examples/`. The notebookts walk through:
|
|
100
|
+
|
|
101
|
+
- Building a mockup environment
|
|
102
|
+
- Creating the corresponding explanation task
|
|
103
|
+
- Loading explanation tasks from different setups: data, black-box function, and existing surrogate model.
|
|
104
|
+
- Computing interaction values with HyperSHAP
|
|
105
|
+
- Visualizing results with `plot_si_graph`
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
## Citation
|
|
110
|
+
If you use HyperSHAP in your research, please cite the original paper:
|
|
111
|
+
|
|
112
|
+
```bibtex
|
|
113
|
+
@article{wever-arxiv25,
|
|
114
|
+
author = {Marcel Wever and
|
|
115
|
+
Maximilian Muschalik and
|
|
116
|
+
Fabian Fumagalli and
|
|
117
|
+
Marius Lindauer},
|
|
118
|
+
title = {HyperSHAP: Shapley Values and Interactions for Hyperparameter Importance},
|
|
119
|
+
journal = {CoRR},
|
|
120
|
+
volume = {abs/2502.01276},
|
|
121
|
+
year = {2025},
|
|
122
|
+
doi = {10.48550/ARXIV.2502.01276},
|
|
123
|
+
}
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
The paper introduces the underlying game-theoretic framework and demonstrates its usefulness for HPO explainability.
|
|
127
|
+
|
|
128
|
+
## Contributing
|
|
129
|
+
|
|
130
|
+
Contributions are welcome! Please follow these steps:
|
|
131
|
+
|
|
132
|
+
1. Fork the repo and create a feature branch (git checkout -b feat/your-feature).
|
|
133
|
+
2. Write tests (the project uses pytest).
|
|
134
|
+
3. Ensure all tests pass (pytest).
|
|
135
|
+
4. Update documentation if you add new functionality.
|
|
136
|
+
5. Submit a Pull Request with a clear description of the changes.
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
See CONTRIBUTING.md for detailed guidelines.
|
|
140
|
+
|
|
141
|
+
---
|
|
142
|
+
|
|
143
|
+
## License
|
|
144
|
+
HyperSHAP is released under the BSD 3-Clause License. See the `LICENSE` file for full terms.
|
|
145
|
+
|
|
146
|
+
---
|
|
147
|
+
|
|
148
|
+
**Enjoy exploring your HPO pipelines with HyperSHAP!** 🎉
|
|
149
|
+
|
|
150
|
+
---
|
|
151
|
+
Repository initiated with [fpgmaas/cookiecutter-uv](https://github.com/fpgmaas/cookiecutter-uv).
|
|
@@ -20,7 +20,7 @@ import logging
|
|
|
20
20
|
import matplotlib.pyplot as plt
|
|
21
21
|
import networkx as nx
|
|
22
22
|
import numpy as np
|
|
23
|
-
from shapiq import SHAPIQ, ExactComputer, InteractionValues
|
|
23
|
+
from shapiq import SHAPIQ, ExactComputer, InteractionValues, KernelSHAPIQ
|
|
24
24
|
|
|
25
25
|
from hypershap.games import (
|
|
26
26
|
AblationGame,
|
|
@@ -125,7 +125,10 @@ class HyperSHAP:
|
|
|
125
125
|
interaction_values = ec(index=index, order=order)
|
|
126
126
|
else:
|
|
127
127
|
# instantiate kernel
|
|
128
|
-
|
|
128
|
+
if index == "FSII":
|
|
129
|
+
approx = SHAPIQ(n=game.n_players, max_order=2, index=index)
|
|
130
|
+
else:
|
|
131
|
+
approx = KernelSHAPIQ(n=game.n_players, max_order=2, index=index)
|
|
129
132
|
|
|
130
133
|
# approximate interaction values with the given index and order
|
|
131
134
|
interaction_values = approx(budget=self.approximation_budget, game=game)
|
|
@@ -182,6 +182,7 @@ class DataBasedSurrogateModel(ModelBasedSurrogateModel):
|
|
|
182
182
|
config_space: ConfigurationSpace,
|
|
183
183
|
data: list[tuple[Configuration, float]],
|
|
184
184
|
base_model: BaseEstimator | None = None,
|
|
185
|
+
seed: int | None = 0,
|
|
185
186
|
) -> None:
|
|
186
187
|
"""Initialize the DataBasedSurrogateModel with data and an optional base model.
|
|
187
188
|
|
|
@@ -191,13 +192,14 @@ class DataBasedSurrogateModel(ModelBasedSurrogateModel):
|
|
|
191
192
|
is a tuple of (Configuration, float).
|
|
192
193
|
base_model: The base model to be used for fitting the surrogate model.
|
|
193
194
|
If None, a RandomForestRegressor is used.
|
|
195
|
+
seed: The random seed for pseudo-randomization of the surrogate model. Defaults to 0.
|
|
194
196
|
|
|
195
197
|
"""
|
|
196
198
|
train_x = np.array([obs[0].get_array() for obs in data])
|
|
197
199
|
train_y = np.array([obs[1] for obs in data])
|
|
198
200
|
|
|
199
201
|
if base_model is None:
|
|
200
|
-
base_model = RandomForestRegressor()
|
|
202
|
+
base_model = RandomForestRegressor(random_state=seed)
|
|
201
203
|
|
|
202
204
|
pipeline = cast("SklearnRegressorProtocol", base_model)
|
|
203
205
|
pipeline.fit(train_x, train_y)
|
|
@@ -17,6 +17,8 @@ if TYPE_CHECKING:
|
|
|
17
17
|
|
|
18
18
|
from hypershap import ConfigSpaceSearcher
|
|
19
19
|
|
|
20
|
+
from copy import deepcopy
|
|
21
|
+
|
|
20
22
|
from sklearn.ensemble import RandomForestRegressor
|
|
21
23
|
|
|
22
24
|
from hypershap.surrogate_model import DataBasedSurrogateModel, ModelBasedSurrogateModel, SurrogateModel
|
|
@@ -132,6 +134,7 @@ class ExplanationTask:
|
|
|
132
134
|
function: Callable[[Configuration], float],
|
|
133
135
|
n_samples: int = 1_000,
|
|
134
136
|
base_model: BaseEstimator | None = None,
|
|
137
|
+
seed: int | None = 0,
|
|
135
138
|
) -> ExplanationTask:
|
|
136
139
|
"""Create an ExplanationTask from a function that evaluates configurations.
|
|
137
140
|
|
|
@@ -140,17 +143,21 @@ class ExplanationTask:
|
|
|
140
143
|
function: A callable that takes a configuration and returns its performance.
|
|
141
144
|
n_samples: The number of configurations to sample for training the surrogate model. Defaults to 1000.
|
|
142
145
|
base_model: The base model to use for training the surrogate model. Defaults to RandomForestRegressor.
|
|
146
|
+
seed: The seed for the random number generator, it is used to seed a deep copy of the config space.
|
|
143
147
|
|
|
144
148
|
Returns:
|
|
145
149
|
An ExplanationTask instance.
|
|
146
150
|
|
|
147
151
|
"""
|
|
148
|
-
|
|
152
|
+
cs = deepcopy(config_space)
|
|
153
|
+
if seed is not None:
|
|
154
|
+
cs.seed(seed)
|
|
155
|
+
samples: list[Configuration] = cs.sample_configuration(n_samples)
|
|
149
156
|
values: list[float] = [function(config) for config in samples]
|
|
150
157
|
data: list[tuple[Configuration, float]] = list(zip(samples, values, strict=False))
|
|
151
|
-
base_model = base_model if base_model is not None else RandomForestRegressor()
|
|
158
|
+
base_model = base_model if base_model is not None else RandomForestRegressor(random_state=seed)
|
|
152
159
|
|
|
153
|
-
return ExplanationTask.from_data(config_space=
|
|
160
|
+
return ExplanationTask.from_data(config_space=cs, data=data, base_model=base_model)
|
|
154
161
|
|
|
155
162
|
@staticmethod
|
|
156
163
|
def from_function_multidata(
|
|
@@ -6,6 +6,7 @@ This module defines specific error classes for simpler debugging and interfaces
|
|
|
6
6
|
from __future__ import annotations
|
|
7
7
|
|
|
8
8
|
from abc import ABC, abstractmethod
|
|
9
|
+
from copy import deepcopy
|
|
9
10
|
from enum import Enum
|
|
10
11
|
from typing import TYPE_CHECKING
|
|
11
12
|
|
|
@@ -83,6 +84,7 @@ class RandomConfigSpaceSearcher(ConfigSpaceSearcher):
|
|
|
83
84
|
explanation_task: BaselineExplanationTask,
|
|
84
85
|
mode: Aggregation = Aggregation.MAX,
|
|
85
86
|
n_samples: int = 10_000,
|
|
87
|
+
seed: int | None = 0,
|
|
86
88
|
) -> None:
|
|
87
89
|
"""Initialize the random configuration space searcher.
|
|
88
90
|
|
|
@@ -91,11 +93,14 @@ class RandomConfigSpaceSearcher(ConfigSpaceSearcher):
|
|
|
91
93
|
space and surrogate model.
|
|
92
94
|
mode: The aggregation mode for performance values.
|
|
93
95
|
n_samples: The number of configurations to sample.
|
|
96
|
+
seed: The random seed for sampling configurations from the config space.
|
|
94
97
|
|
|
95
98
|
"""
|
|
96
99
|
super().__init__(explanation_task, mode=mode)
|
|
97
|
-
|
|
98
|
-
|
|
100
|
+
cs = deepcopy(explanation_task.config_space)
|
|
101
|
+
if seed is not None:
|
|
102
|
+
cs.seed(seed)
|
|
103
|
+
sampled_configurations = cs.sample_configuration(size=n_samples)
|
|
99
104
|
self.random_sample = np.array([config.get_array() for config in sampled_configurations])
|
|
100
105
|
|
|
101
106
|
# cache coalition values to ensure monotonicity for min/max
|
|
@@ -12,6 +12,7 @@ from hypershap import ExplanationTask
|
|
|
12
12
|
def simple_config_space() -> ConfigurationSpace:
|
|
13
13
|
"""Return a simple config space for testing."""
|
|
14
14
|
config_space = ConfigurationSpace()
|
|
15
|
+
config_space.seed(42)
|
|
15
16
|
config_space.add(UniformFloatHyperparameter("a", 0, 1, 0))
|
|
16
17
|
config_space.add(UniformFloatHyperparameter("b", 0, 1, 0))
|
|
17
18
|
return config_space
|
|
@@ -26,6 +26,14 @@ def test_large_ablation(large_base_et: ExplanationTask) -> None:
|
|
|
26
26
|
hypershap.ablation(comparison, baseline)
|
|
27
27
|
|
|
28
28
|
|
|
29
|
+
def test_large_ablation_kernelshap(large_base_et: ExplanationTask) -> None:
|
|
30
|
+
"""Test HyperSHAP with large config space."""
|
|
31
|
+
baseline = large_base_et.config_space.sample_configuration()
|
|
32
|
+
comparison = large_base_et.config_space.sample_configuration()
|
|
33
|
+
hypershap = HyperSHAP(explanation_task=large_base_et, approximation_budget=2**7)
|
|
34
|
+
hypershap.ablation(comparison, baseline, index="k-SII")
|
|
35
|
+
|
|
36
|
+
|
|
29
37
|
def test_multi_data_ablation(
|
|
30
38
|
multi_data_baseline_config: Configuration,
|
|
31
39
|
multi_data_config_space: ConfigurationSpace,
|
|
@@ -1301,7 +1301,7 @@ wheels = [
|
|
|
1301
1301
|
|
|
1302
1302
|
[[package]]
|
|
1303
1303
|
name = "hypershap"
|
|
1304
|
-
version = "0.0.
|
|
1304
|
+
version = "0.0.4"
|
|
1305
1305
|
source = { editable = "." }
|
|
1306
1306
|
dependencies = [
|
|
1307
1307
|
{ name = "matplotlib" },
|
|
@@ -4184,6 +4184,12 @@ wheels = [
|
|
|
4184
4184
|
{ url = "https://files.pythonhosted.org/packages/1e/48/973da1ee8bc0743519759e74c3615b39acdc3faf00e0a0710f8c856d8c9d/statsmodels-0.14.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a085d47c8ef5387279a991633883d0e700de2b0acc812d7032d165888627bef", size = 10453538, upload-time = "2025-07-07T14:24:06.959Z" },
|
|
4185
4185
|
{ url = "https://files.pythonhosted.org/packages/c7/d6/18903fb707afd31cf1edaec5201964dbdacb2bfae9a22558274647a7c88f/statsmodels-0.14.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f866b2ebb2904b47c342d00def83c526ef2eb1df6a9a3c94ba5fe63d0005aec", size = 10681584, upload-time = "2025-07-07T14:24:21.038Z" },
|
|
4186
4186
|
{ url = "https://files.pythonhosted.org/packages/44/d6/80df1bbbfcdc50bff4152f43274420fa9856d56e234d160d6206eb1f5827/statsmodels-0.14.5-cp313-cp313-win_amd64.whl", hash = "sha256:2a06bca03b7a492f88c8106103ab75f1a5ced25de90103a89f3a287518017939", size = 9604641, upload-time = "2025-07-07T12:08:36.23Z" },
|
|
4187
|
+
{ url = "https://files.pythonhosted.org/packages/fd/6c/0fb40a89d715412160097c6f3387049ed88c9bd866c8838a8852c705ae2f/statsmodels-0.14.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:07c4dad25bbb15864a31b4917a820f6d104bdc24e5ddadcda59027390c3bed9e", size = 10211256, upload-time = "2025-10-30T13:46:58.591Z" },
|
|
4188
|
+
{ url = "https://files.pythonhosted.org/packages/88/4a/e36fe8b19270ab3e80df357da924c6c029cab0fb9a0fbd28aaf49341707d/statsmodels-0.14.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:babb067c852e966c2c933b79dbb5d0240919d861941a2ef6c0e13321c255528d", size = 10110933, upload-time = "2025-10-30T13:47:11.774Z" },
|
|
4189
|
+
{ url = "https://files.pythonhosted.org/packages/8a/bf/1b7e7b1a6c09a88a9c5c9e60622c050dfd08af11c2e6d4a42dbc71b32ee1/statsmodels-0.14.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:110194b137286173cc676d7bad0119a197778de6478fc6cbdc3b33571165ac1e", size = 10253981, upload-time = "2025-10-30T16:32:22.399Z" },
|
|
4190
|
+
{ url = "https://files.pythonhosted.org/packages/b8/d0/f95da95524bdd99613923ca61a3036d1308cee1290e5e8acb89f51736a8c/statsmodels-0.14.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c8a9c384a60c80731b278e7fd18764364c8817f4995b13a175d636f967823d1", size = 10460450, upload-time = "2025-10-30T16:32:44.985Z" },
|
|
4191
|
+
{ url = "https://files.pythonhosted.org/packages/28/bb/59e7be0271be264b7b541baf3973f97747740950bfd5115de731f63da8ab/statsmodels-0.14.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:557df3a870a57248df744fdfcc444ecbc5bdbf1c042b8a8b5d8e3e797830dc2a", size = 10694060, upload-time = "2025-10-30T16:33:07.656Z" },
|
|
4192
|
+
{ url = "https://files.pythonhosted.org/packages/8b/c0/b28d0fd0347ea38d3610052f479e4b922eb33bb8790817f93cd89e6e08ba/statsmodels-0.14.5-cp314-cp314-win_amd64.whl", hash = "sha256:95af7a9c4689d514f4341478b891f867766f3da297f514b8c4adf08f4fa61d03", size = 9648961, upload-time = "2025-10-30T13:47:24.303Z" },
|
|
4187
4193
|
]
|
|
4188
4194
|
|
|
4189
4195
|
[[package]]
|
hypershap-0.0.3/PKG-INFO
DELETED
|
@@ -1,169 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: hypershap
|
|
3
|
-
Version: 0.0.3
|
|
4
|
-
Summary: HyperSHAP is a post-hoc explanation method for hyperparameter optimization.
|
|
5
|
-
Project-URL: Homepage, https://automl.github.io/hypershap/
|
|
6
|
-
Project-URL: Repository, https://github.com/automl/hypershap
|
|
7
|
-
Project-URL: Documentation, https://automl.github.io/hypershap/
|
|
8
|
-
Author-email: Marcel Wever <m.wever@ai.uni-hannover.de>
|
|
9
|
-
License-File: LICENSE
|
|
10
|
-
Keywords: python
|
|
11
|
-
Classifier: Intended Audience :: Developers
|
|
12
|
-
Classifier: Programming Language :: Python
|
|
13
|
-
Classifier: Programming Language :: Python :: 3
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
-
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
-
Requires-Python: <4.0,>=3.10
|
|
20
|
-
Requires-Dist: matplotlib>=3.10.5
|
|
21
|
-
Requires-Dist: networkx>=3.4.2
|
|
22
|
-
Requires-Dist: numpy>=2.2.6
|
|
23
|
-
Requires-Dist: scikit-learn>=1.7.1
|
|
24
|
-
Requires-Dist: shapiq>=1.2.3
|
|
25
|
-
Description-Content-Type: text/markdown
|
|
26
|
-
|
|
27
|
-
# HyperSHAP <img src="https://raw.githubusercontent.com/automl/hypershap/main/docs/source/_static/logo/hypershap-logo.png" alt="HyperSHAP Logo" align="right" height="200px"/>
|
|
28
|
-
|
|
29
|
-
[](https://img.shields.io/github/v/release/automl/hypershap)
|
|
30
|
-
[](https://github.com/automl/hypershap/actions/workflows/main.yml?query=branch%3Amain)
|
|
31
|
-
[](https://coveralls.io/github/automl/HyperSHAP?branch=dev)
|
|
32
|
-
[](https://img.shields.io/github/commit-activity/m/automl/hypershap)
|
|
33
|
-
[](https://img.shields.io/github/license/automl/hypershap)
|
|
34
|
-
|
|
35
|
-
HyperSHAP – a game‑theoretic Python library for explaining Hyperparameter Optimization (HPO). It uses Shapley values and interaction indices to provide both local and global insights into how individual hyper‑parameters (and their interactions) affect a model’s performance.
|
|
36
|
-
|
|
37
|
-
- **Github repository**: <https://github.com/automl/hypershap/>
|
|
38
|
-
- **Documentation** <https://automl.github.io/hypershap/>
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
## Table of Contents
|
|
42
|
-
- [Features](#features)
|
|
43
|
-
- [Installation](#installation)
|
|
44
|
-
- [Getting Started](#getting-started)
|
|
45
|
-
- [API Overview](#api-overview)
|
|
46
|
-
- [Example Notebook](#example-notebook)
|
|
47
|
-
- [Citation](#citation)
|
|
48
|
-
- [Contributing](#contributing)
|
|
49
|
-
- [License](#license)
|
|
50
|
-
|
|
51
|
-
---
|
|
52
|
-
|
|
53
|
-
## Features
|
|
54
|
-
- **Additive Shapley decomposition** of any performance metric across hyper‑parameters.
|
|
55
|
-
- **Interaction analysis** via the Faithful Shapley Interaction Index (FSII).
|
|
56
|
-
- Ready‑made explanation tasks for **Ablation**, **Tunability**, and **Optimizer Bias** studies.
|
|
57
|
-
- Integrated **visualisation** (SI‑graph) for interaction effects.
|
|
58
|
-
- Works with any surrogate model that follows the `ExplanationTask` interface.
|
|
59
|
-
|
|
60
|
-
---
|
|
61
|
-
|
|
62
|
-
## Installation
|
|
63
|
-
|
|
64
|
-
```sh
|
|
65
|
-
$ make install
|
|
66
|
-
```
|
|
67
|
-
|
|
68
|
-
## Getting Started
|
|
69
|
-
Given an existing setup with a ConfigurationSpace from the [ConfigSpace package](https://github.com/automl/ConfigSpace) and black-box function as follows:
|
|
70
|
-
```Python
|
|
71
|
-
from ConfigSpace import ConfigurationSpace, Configuration
|
|
72
|
-
|
|
73
|
-
# ConfigurationSpace describing the hyperparameter space
|
|
74
|
-
cs = ConfigurationSpace()
|
|
75
|
-
...
|
|
76
|
-
|
|
77
|
-
# A black-box function, evaluating ConfigSpace.Configuration objects
|
|
78
|
-
def blackbox_function(cfg: Configuration) -> float:
|
|
79
|
-
...
|
|
80
|
-
```
|
|
81
|
-
|
|
82
|
-
You can use HyperSHAP as follows:
|
|
83
|
-
```Python
|
|
84
|
-
from hypershap import ExplanationTask, HyperSHAP
|
|
85
|
-
|
|
86
|
-
# Instantiate HyperSHAP
|
|
87
|
-
hypershap = HyperSHAP(ExplanationTask.from_function(config_space=cs,function=blackbox_function))
|
|
88
|
-
# Conduct tunability analysis
|
|
89
|
-
hypershap.tunability(baseline_config=cs.get_default_configuration())
|
|
90
|
-
# Plot results as a Shapley Interaction graph
|
|
91
|
-
hypershap.plot_si_graph()
|
|
92
|
-
```
|
|
93
|
-
|
|
94
|
-
The example demonstrates how to:
|
|
95
|
-
1. Wrap a black-box function in an explanation task.
|
|
96
|
-
2. Use `HyperSHAP` to obtain interaction values for the **tunability** game.
|
|
97
|
-
3. Plot the corresponding SI-graph.
|
|
98
|
-
|
|
99
|
-
---
|
|
100
|
-
|
|
101
|
-
## API Overview
|
|
102
|
-
|
|
103
|
-
| Method | Purpose | Key Arguments |
|
|
104
|
-
|--------|-----------------------------------------------------------------------------------------------------------------------------------|---------------|
|
|
105
|
-
| `HyperSHAP(explanation_task)` | Initialize the explainer with a generic `ExplanationTask`. |
|
|
106
|
-
| `ablation(config_of_interest, baseline_config, index="FSII", order=2)` | Explain the contribution of each hyperparameter value (and interactions) when moving from a baseline to a specific configuration. |
|
|
107
|
-
| `tunability(baseline_config=None, index="FSII", order=2, n_samples=10_000)` | Quantify how much performance can be gained by tuning subsets of hyper‑parameters. |
|
|
108
|
-
| `optimizer_bias(optimizer_of_interest, optimizer_ensemble, index="FSII", order=2)` | Attribute performance differences to a particular optimizer vs. an ensemble of optimizers. |
|
|
109
|
-
| `plot_si_graph(interaction_values=None, save_path=None)` | Plot the Shapley Interaction (SI) graph; uses the most recent interaction values if none are supplied. |
|
|
110
|
-
| `ExplanationTask.get_hyperparameter_names()` | Helper to retrieve ordered hyper‑parameter names (used for visualisation). |
|
|
111
|
-
|
|
112
|
-
All methods return an `InteractionValues` object (from **shapiq**) that can be inspected, saved, or passed to the visualisation routine.
|
|
113
|
-
|
|
114
|
-
---
|
|
115
|
-
|
|
116
|
-
## Example Notebooks
|
|
117
|
-
Full Jupyter notebooks illustrating all three explanation tasks (ablation, tunability, optimizer bias) are included in the repository under `examples/`. The notebookts walk through:
|
|
118
|
-
|
|
119
|
-
- Building a mockup environment
|
|
120
|
-
- Creating the corresponding explanation task
|
|
121
|
-
- Loading explanation tasks from different setups: data, black-box function, and existing surrogate model.
|
|
122
|
-
- Computing interaction values with HyperSHAP
|
|
123
|
-
- Visualizing results with `plot_si_graph`
|
|
124
|
-
|
|
125
|
-
---
|
|
126
|
-
|
|
127
|
-
## Citation
|
|
128
|
-
If you use HyperSHAP in your research, please cite the original paper:
|
|
129
|
-
|
|
130
|
-
```bibtex
|
|
131
|
-
@article{wever-arxiv25,
|
|
132
|
-
author = {Marcel Wever and
|
|
133
|
-
Maximilian Muschalik and
|
|
134
|
-
Fabian Fumagalli and
|
|
135
|
-
Marius Lindauer},
|
|
136
|
-
title = {HyperSHAP: Shapley Values and Interactions for Hyperparameter Importance},
|
|
137
|
-
journal = {CoRR},
|
|
138
|
-
volume = {abs/2502.01276},
|
|
139
|
-
year = {2025},
|
|
140
|
-
doi = {10.48550/ARXIV.2502.01276},
|
|
141
|
-
}
|
|
142
|
-
```
|
|
143
|
-
|
|
144
|
-
The paper introduces the underlying game-theoretic framework and demonstrates its usefulness for HPO explainability.
|
|
145
|
-
|
|
146
|
-
## Contributing
|
|
147
|
-
|
|
148
|
-
Contributions are welcome! Please follow these steps:
|
|
149
|
-
|
|
150
|
-
1. Fork the repo and create a feature branch (git checkout -b feat/your-feature).
|
|
151
|
-
2. Write tests (the project uses pytest).
|
|
152
|
-
3. Ensure all tests pass (pytest).
|
|
153
|
-
4. Update documentation if you add new functionality.
|
|
154
|
-
5. Submit a Pull Request with a clear description of the changes.
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
See CONTRIBUTING.md for detailed guidelines.
|
|
158
|
-
|
|
159
|
-
---
|
|
160
|
-
|
|
161
|
-
## License
|
|
162
|
-
HyperSHAP is released under the BSD 3-Clause License. See the `LICENSE` file for full terms.
|
|
163
|
-
|
|
164
|
-
---
|
|
165
|
-
|
|
166
|
-
**Enjoy exploring your HPO pipelines with HyperSHAP!** 🎉
|
|
167
|
-
|
|
168
|
-
---
|
|
169
|
-
Repository initiated with [fpgmaas/cookiecutter-uv](https://github.com/fpgmaas/cookiecutter-uv).
|
hypershap-0.0.3/README.md
DELETED
|
@@ -1,143 +0,0 @@
|
|
|
1
|
-
# HyperSHAP <img src="https://raw.githubusercontent.com/automl/hypershap/main/docs/source/_static/logo/hypershap-logo.png" alt="HyperSHAP Logo" align="right" height="200px"/>
|
|
2
|
-
|
|
3
|
-
[](https://img.shields.io/github/v/release/automl/hypershap)
|
|
4
|
-
[](https://github.com/automl/hypershap/actions/workflows/main.yml?query=branch%3Amain)
|
|
5
|
-
[](https://coveralls.io/github/automl/HyperSHAP?branch=dev)
|
|
6
|
-
[](https://img.shields.io/github/commit-activity/m/automl/hypershap)
|
|
7
|
-
[](https://img.shields.io/github/license/automl/hypershap)
|
|
8
|
-
|
|
9
|
-
HyperSHAP – a game‑theoretic Python library for explaining Hyperparameter Optimization (HPO). It uses Shapley values and interaction indices to provide both local and global insights into how individual hyper‑parameters (and their interactions) affect a model’s performance.
|
|
10
|
-
|
|
11
|
-
- **Github repository**: <https://github.com/automl/hypershap/>
|
|
12
|
-
- **Documentation** <https://automl.github.io/hypershap/>
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
## Table of Contents
|
|
16
|
-
- [Features](#features)
|
|
17
|
-
- [Installation](#installation)
|
|
18
|
-
- [Getting Started](#getting-started)
|
|
19
|
-
- [API Overview](#api-overview)
|
|
20
|
-
- [Example Notebook](#example-notebook)
|
|
21
|
-
- [Citation](#citation)
|
|
22
|
-
- [Contributing](#contributing)
|
|
23
|
-
- [License](#license)
|
|
24
|
-
|
|
25
|
-
---
|
|
26
|
-
|
|
27
|
-
## Features
|
|
28
|
-
- **Additive Shapley decomposition** of any performance metric across hyper‑parameters.
|
|
29
|
-
- **Interaction analysis** via the Faithful Shapley Interaction Index (FSII).
|
|
30
|
-
- Ready‑made explanation tasks for **Ablation**, **Tunability**, and **Optimizer Bias** studies.
|
|
31
|
-
- Integrated **visualisation** (SI‑graph) for interaction effects.
|
|
32
|
-
- Works with any surrogate model that follows the `ExplanationTask` interface.
|
|
33
|
-
|
|
34
|
-
---
|
|
35
|
-
|
|
36
|
-
## Installation
|
|
37
|
-
|
|
38
|
-
```sh
|
|
39
|
-
$ make install
|
|
40
|
-
```
|
|
41
|
-
|
|
42
|
-
## Getting Started
|
|
43
|
-
Given an existing setup with a ConfigurationSpace from the [ConfigSpace package](https://github.com/automl/ConfigSpace) and black-box function as follows:
|
|
44
|
-
```Python
|
|
45
|
-
from ConfigSpace import ConfigurationSpace, Configuration
|
|
46
|
-
|
|
47
|
-
# ConfigurationSpace describing the hyperparameter space
|
|
48
|
-
cs = ConfigurationSpace()
|
|
49
|
-
...
|
|
50
|
-
|
|
51
|
-
# A black-box function, evaluating ConfigSpace.Configuration objects
|
|
52
|
-
def blackbox_function(cfg: Configuration) -> float:
|
|
53
|
-
...
|
|
54
|
-
```
|
|
55
|
-
|
|
56
|
-
You can use HyperSHAP as follows:
|
|
57
|
-
```Python
|
|
58
|
-
from hypershap import ExplanationTask, HyperSHAP
|
|
59
|
-
|
|
60
|
-
# Instantiate HyperSHAP
|
|
61
|
-
hypershap = HyperSHAP(ExplanationTask.from_function(config_space=cs,function=blackbox_function))
|
|
62
|
-
# Conduct tunability analysis
|
|
63
|
-
hypershap.tunability(baseline_config=cs.get_default_configuration())
|
|
64
|
-
# Plot results as a Shapley Interaction graph
|
|
65
|
-
hypershap.plot_si_graph()
|
|
66
|
-
```
|
|
67
|
-
|
|
68
|
-
The example demonstrates how to:
|
|
69
|
-
1. Wrap a black-box function in an explanation task.
|
|
70
|
-
2. Use `HyperSHAP` to obtain interaction values for the **tunability** game.
|
|
71
|
-
3. Plot the corresponding SI-graph.
|
|
72
|
-
|
|
73
|
-
---
|
|
74
|
-
|
|
75
|
-
## API Overview
|
|
76
|
-
|
|
77
|
-
| Method | Purpose | Key Arguments |
|
|
78
|
-
|--------|-----------------------------------------------------------------------------------------------------------------------------------|---------------|
|
|
79
|
-
| `HyperSHAP(explanation_task)` | Initialize the explainer with a generic `ExplanationTask`. |
|
|
80
|
-
| `ablation(config_of_interest, baseline_config, index="FSII", order=2)` | Explain the contribution of each hyperparameter value (and interactions) when moving from a baseline to a specific configuration. |
|
|
81
|
-
| `tunability(baseline_config=None, index="FSII", order=2, n_samples=10_000)` | Quantify how much performance can be gained by tuning subsets of hyper‑parameters. |
|
|
82
|
-
| `optimizer_bias(optimizer_of_interest, optimizer_ensemble, index="FSII", order=2)` | Attribute performance differences to a particular optimizer vs. an ensemble of optimizers. |
|
|
83
|
-
| `plot_si_graph(interaction_values=None, save_path=None)` | Plot the Shapley Interaction (SI) graph; uses the most recent interaction values if none are supplied. |
|
|
84
|
-
| `ExplanationTask.get_hyperparameter_names()` | Helper to retrieve ordered hyper‑parameter names (used for visualisation). |
|
|
85
|
-
|
|
86
|
-
All methods return an `InteractionValues` object (from **shapiq**) that can be inspected, saved, or passed to the visualisation routine.
|
|
87
|
-
|
|
88
|
-
---
|
|
89
|
-
|
|
90
|
-
## Example Notebooks
|
|
91
|
-
Full Jupyter notebooks illustrating all three explanation tasks (ablation, tunability, optimizer bias) are included in the repository under `examples/`. The notebookts walk through:
|
|
92
|
-
|
|
93
|
-
- Building a mockup environment
|
|
94
|
-
- Creating the corresponding explanation task
|
|
95
|
-
- Loading explanation tasks from different setups: data, black-box function, and existing surrogate model.
|
|
96
|
-
- Computing interaction values with HyperSHAP
|
|
97
|
-
- Visualizing results with `plot_si_graph`
|
|
98
|
-
|
|
99
|
-
---
|
|
100
|
-
|
|
101
|
-
## Citation
|
|
102
|
-
If you use HyperSHAP in your research, please cite the original paper:
|
|
103
|
-
|
|
104
|
-
```bibtex
|
|
105
|
-
@article{wever-arxiv25,
|
|
106
|
-
author = {Marcel Wever and
|
|
107
|
-
Maximilian Muschalik and
|
|
108
|
-
Fabian Fumagalli and
|
|
109
|
-
Marius Lindauer},
|
|
110
|
-
title = {HyperSHAP: Shapley Values and Interactions for Hyperparameter Importance},
|
|
111
|
-
journal = {CoRR},
|
|
112
|
-
volume = {abs/2502.01276},
|
|
113
|
-
year = {2025},
|
|
114
|
-
doi = {10.48550/ARXIV.2502.01276},
|
|
115
|
-
}
|
|
116
|
-
```
|
|
117
|
-
|
|
118
|
-
The paper introduces the underlying game-theoretic framework and demonstrates its usefulness for HPO explainability.
|
|
119
|
-
|
|
120
|
-
## Contributing
|
|
121
|
-
|
|
122
|
-
Contributions are welcome! Please follow these steps:
|
|
123
|
-
|
|
124
|
-
1. Fork the repo and create a feature branch (git checkout -b feat/your-feature).
|
|
125
|
-
2. Write tests (the project uses pytest).
|
|
126
|
-
3. Ensure all tests pass (pytest).
|
|
127
|
-
4. Update documentation if you add new functionality.
|
|
128
|
-
5. Submit a Pull Request with a clear description of the changes.
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
See CONTRIBUTING.md for detailed guidelines.
|
|
132
|
-
|
|
133
|
-
---
|
|
134
|
-
|
|
135
|
-
## License
|
|
136
|
-
HyperSHAP is released under the BSD 3-Clause License. See the `LICENSE` file for full terms.
|
|
137
|
-
|
|
138
|
-
---
|
|
139
|
-
|
|
140
|
-
**Enjoy exploring your HPO pipelines with HyperSHAP!** 🎉
|
|
141
|
-
|
|
142
|
-
---
|
|
143
|
-
Repository initiated with [fpgmaas/cookiecutter-uv](https://github.com/fpgmaas/cookiecutter-uv).
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|