hypershap 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hypershap/__init__.py +18 -0
- hypershap/games/__init__.py +18 -0
- hypershap/games/ablation.py +101 -0
- hypershap/games/abstract.py +161 -0
- hypershap/games/optimizerbias.py +115 -0
- hypershap/games/tunability.py +183 -0
- hypershap/hypershap.py +409 -0
- hypershap/surrogate_model.py +235 -0
- hypershap/task.py +265 -0
- hypershap/utils.py +126 -0
- hypershap-0.0.1.dist-info/METADATA +168 -0
- hypershap-0.0.1.dist-info/RECORD +14 -0
- hypershap-0.0.1.dist-info/WHEEL +4 -0
- hypershap-0.0.1.dist-info/licenses/LICENSE +28 -0
hypershap/hypershap.py
ADDED
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
"""HyperSHAP main interface to work on explanation for a given task.
|
|
2
|
+
|
|
3
|
+
This module provides the main interface for working with HyperSHAP to access explanations regarding ablation, tunability,
|
|
4
|
+
sensitivity, and optimizer bias.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from ConfigSpace import Configuration
|
|
13
|
+
|
|
14
|
+
from hypershap.utils import ConfigSpaceSearcher
|
|
15
|
+
|
|
16
|
+
import logging
|
|
17
|
+
|
|
18
|
+
import matplotlib.pyplot as plt
|
|
19
|
+
import networkx as nx
|
|
20
|
+
import numpy as np
|
|
21
|
+
from shapiq import ExactComputer, InteractionValues
|
|
22
|
+
|
|
23
|
+
from hypershap.games import (
|
|
24
|
+
AblationGame,
|
|
25
|
+
AbstractHPIGame,
|
|
26
|
+
MistunabilityGame,
|
|
27
|
+
OptimizerBiasGame,
|
|
28
|
+
SensitivityGame,
|
|
29
|
+
TunabilityGame,
|
|
30
|
+
)
|
|
31
|
+
from hypershap.task import (
|
|
32
|
+
AblationExplanationTask,
|
|
33
|
+
ExplanationTask,
|
|
34
|
+
MistunabilityExplanationTask,
|
|
35
|
+
OptimizerBiasExplanationTask,
|
|
36
|
+
SensitivityExplanationTask,
|
|
37
|
+
TunabilityExplanationTask,
|
|
38
|
+
)
|
|
39
|
+
from hypershap.utils import RandomConfigSpaceSearcher
|
|
40
|
+
|
|
41
|
+
logger = logging.getLogger(__name__)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class NoInteractionValuesError(ValueError):
|
|
45
|
+
"""Exception raised when no interaction values are present for plotting."""
|
|
46
|
+
|
|
47
|
+
def __init__(self) -> None:
|
|
48
|
+
"""Initialize the no interaction values error."""
|
|
49
|
+
super().__init__("No interaction values present for plotting.")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class HyperSHAP:
|
|
53
|
+
"""A class for computing and visualizing HyperSHAP Shapley values and interactions.
|
|
54
|
+
|
|
55
|
+
Attributes:
|
|
56
|
+
explanation_task (ExplanationTask): The task responsible for generating explanations.
|
|
57
|
+
last_interaction_values (InteractionValues | None): The cached interaction values for plotting shortcuts.
|
|
58
|
+
|
|
59
|
+
Methods:
|
|
60
|
+
__init__(explanation_task: ExplanationTask):
|
|
61
|
+
Initializes the HyperSHAP instance with an explanation task.
|
|
62
|
+
|
|
63
|
+
ablation(config_of_interest: Configuration, baseline_config: Configuration, index: str = "FSII", order: int = 2) -> InteractionValues:
|
|
64
|
+
Computes and returns the interaction values for ablation analysis.
|
|
65
|
+
|
|
66
|
+
tunability(baseline_config: Configuration | None, index: str = "FSII", order: int = 2) -> InteractionValues:
|
|
67
|
+
Computes and returns the interaction values for tunability analysis.
|
|
68
|
+
|
|
69
|
+
optimizer_bias(optimizer_of_interest: ConfigSpaceSearcher, optimizer_ensemble: list[ConfigSpaceSearcher], index: str = "FSII", order: int = 2) -> InteractionValues:
|
|
70
|
+
Computes and returns the interaction values for optimizer bias analysis.
|
|
71
|
+
|
|
72
|
+
plot_si_graph(interaction_values: InteractionValues | None = None, save_path: str | None = None):
|
|
73
|
+
Plots the SHAP interaction values as a graph.
|
|
74
|
+
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
def __init__(
|
|
78
|
+
self,
|
|
79
|
+
explanation_task: ExplanationTask,
|
|
80
|
+
n_workers: int | None = None,
|
|
81
|
+
verbose: bool | None = None,
|
|
82
|
+
) -> None:
|
|
83
|
+
"""Initialize the HyperSHAP instance with an explanation task.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
explanation_task (ExplanationTask): The task responsible for generating explanations.
|
|
87
|
+
n_workers: The number of worker threads to use for parallel evaluation
|
|
88
|
+
of coalitions. Defaults to None meaning no parallelization. Using more workers can significantly
|
|
89
|
+
speed up the computation of Shapley values. The maximum number of workers is capped by the number of coalitions.
|
|
90
|
+
verbose: A boolean indicating whether to print verbose messages during
|
|
91
|
+
computation. Defaults to None. When set to True, the method prints
|
|
92
|
+
debugging information and progress updates.
|
|
93
|
+
|
|
94
|
+
"""
|
|
95
|
+
self.explanation_task = explanation_task
|
|
96
|
+
self.last_interaction_values = None
|
|
97
|
+
self.n_workers = n_workers
|
|
98
|
+
self.verbose = verbose
|
|
99
|
+
|
|
100
|
+
def __get_interaction_values(self, game: AbstractHPIGame, index: str = "FSII", order: int = 2) -> InteractionValues:
|
|
101
|
+
# instantiate exact computer if number of hyperparameters is small enough
|
|
102
|
+
ec = ExactComputer(n_players=game.get_num_hyperparameters(), game=game) # pyright: ignore
|
|
103
|
+
|
|
104
|
+
# compute interaction values with the given index and order
|
|
105
|
+
interaction_values = ec(index=index, order=order)
|
|
106
|
+
|
|
107
|
+
# cache current interaction values for plotting shortcuts
|
|
108
|
+
self.last_interaction_values = interaction_values
|
|
109
|
+
|
|
110
|
+
return interaction_values
|
|
111
|
+
|
|
112
|
+
def ablation(
|
|
113
|
+
self,
|
|
114
|
+
config_of_interest: Configuration,
|
|
115
|
+
baseline_config: Configuration,
|
|
116
|
+
index: str = "FSII",
|
|
117
|
+
order: int = 2,
|
|
118
|
+
) -> InteractionValues:
|
|
119
|
+
"""Compute and return the interaction values for ablation analysis.
|
|
120
|
+
|
|
121
|
+
Args:
|
|
122
|
+
config_of_interest (Configuration): The configuration of interest.
|
|
123
|
+
baseline_config (Configuration): The baseline configuration.
|
|
124
|
+
index (str, optional): The index to use for computing interaction values. Defaults to "FSII".
|
|
125
|
+
order (int, optional): The order of the interaction values. Defaults to 2.
|
|
126
|
+
|
|
127
|
+
Returns:
|
|
128
|
+
InteractionValues: The computed interaction values.
|
|
129
|
+
|
|
130
|
+
"""
|
|
131
|
+
# setup explanation task
|
|
132
|
+
ablation_task: AblationExplanationTask = AblationExplanationTask(
|
|
133
|
+
config_space=self.explanation_task.config_space,
|
|
134
|
+
surrogate_model=self.explanation_task.surrogate_model,
|
|
135
|
+
baseline_config=baseline_config,
|
|
136
|
+
config_of_interest=config_of_interest,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
# setup ablation game and get interaction values
|
|
140
|
+
ag = AblationGame(
|
|
141
|
+
explanation_task=ablation_task,
|
|
142
|
+
n_workers=self.n_workers,
|
|
143
|
+
verbose=self.verbose,
|
|
144
|
+
)
|
|
145
|
+
return self.__get_interaction_values(game=ag, index=index, order=order)
|
|
146
|
+
|
|
147
|
+
def tunability(
|
|
148
|
+
self,
|
|
149
|
+
baseline_config: Configuration | None = None,
|
|
150
|
+
index: str = "FSII",
|
|
151
|
+
order: int = 2,
|
|
152
|
+
n_samples: int = 10_000,
|
|
153
|
+
) -> InteractionValues:
|
|
154
|
+
"""Compute and return the interaction values for tunability analysis.
|
|
155
|
+
|
|
156
|
+
Args:
|
|
157
|
+
baseline_config (Configuration | None, optional): The baseline configuration. Defaults to None.
|
|
158
|
+
index (str, optional): The index to use for computing interaction values. Defaults to "FSII".
|
|
159
|
+
order (int, optional): The order of the interaction values. Defaults to 2.
|
|
160
|
+
n_samples (int, optional): The number of samples to use for simulating HPO. Defaults to 10_000.
|
|
161
|
+
|
|
162
|
+
Returns:
|
|
163
|
+
InteractionValues: The computed interaction values.
|
|
164
|
+
|
|
165
|
+
"""
|
|
166
|
+
if baseline_config is None:
|
|
167
|
+
baseline_config = self.explanation_task.config_space.get_default_configuration()
|
|
168
|
+
|
|
169
|
+
# setup explanation task
|
|
170
|
+
tunability_task: TunabilityExplanationTask = TunabilityExplanationTask(
|
|
171
|
+
config_space=self.explanation_task.config_space,
|
|
172
|
+
surrogate_model=self.explanation_task.surrogate_model,
|
|
173
|
+
baseline_config=baseline_config,
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
# setup tunability game and get interaction values
|
|
177
|
+
tg = TunabilityGame(
|
|
178
|
+
explanation_task=tunability_task,
|
|
179
|
+
cs_searcher=RandomConfigSpaceSearcher(
|
|
180
|
+
explanation_task=tunability_task,
|
|
181
|
+
n_samples=n_samples,
|
|
182
|
+
mode="max",
|
|
183
|
+
),
|
|
184
|
+
n_workers=self.n_workers,
|
|
185
|
+
verbose=self.verbose,
|
|
186
|
+
)
|
|
187
|
+
return self.__get_interaction_values(game=tg, index=index, order=order)
|
|
188
|
+
|
|
189
|
+
def sensitivity(
|
|
190
|
+
self,
|
|
191
|
+
baseline_config: Configuration | None = None,
|
|
192
|
+
index: str = "FSII",
|
|
193
|
+
order: int = 2,
|
|
194
|
+
n_samples: int = 10_000,
|
|
195
|
+
) -> InteractionValues:
|
|
196
|
+
"""Compute and return the interaction values for sensitivity analysis.
|
|
197
|
+
|
|
198
|
+
Args:
|
|
199
|
+
baseline_config (Configuration | None, optional): The baseline configuration. Defaults to None.
|
|
200
|
+
index (str, optional): The index to use for computing interaction values. Defaults to "FSII".
|
|
201
|
+
order (int, optional): The order of the interaction values. Defaults to 2.
|
|
202
|
+
n_samples (int, optional): The number of samples to use for simulating HPO. Defaults to 10_000.
|
|
203
|
+
|
|
204
|
+
Returns:
|
|
205
|
+
InteractionValues: The computed interaction values.
|
|
206
|
+
|
|
207
|
+
"""
|
|
208
|
+
if baseline_config is None:
|
|
209
|
+
baseline_config = self.explanation_task.config_space.get_default_configuration()
|
|
210
|
+
|
|
211
|
+
# setup explanation task
|
|
212
|
+
sensitivity_task: SensitivityExplanationTask = SensitivityExplanationTask(
|
|
213
|
+
config_space=self.explanation_task.config_space,
|
|
214
|
+
surrogate_model=self.explanation_task.surrogate_model,
|
|
215
|
+
baseline_config=baseline_config,
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
# setup tunability game and get interaction values
|
|
219
|
+
tg = SensitivityGame(
|
|
220
|
+
explanation_task=sensitivity_task,
|
|
221
|
+
cs_searcher=RandomConfigSpaceSearcher(
|
|
222
|
+
explanation_task=sensitivity_task,
|
|
223
|
+
n_samples=n_samples,
|
|
224
|
+
mode="var",
|
|
225
|
+
),
|
|
226
|
+
n_workers=self.n_workers,
|
|
227
|
+
verbose=self.verbose,
|
|
228
|
+
)
|
|
229
|
+
return self.__get_interaction_values(game=tg, index=index, order=order)
|
|
230
|
+
|
|
231
|
+
def mistunability(
|
|
232
|
+
self,
|
|
233
|
+
baseline_config: Configuration | None = None,
|
|
234
|
+
index: str = "FSII",
|
|
235
|
+
order: int = 2,
|
|
236
|
+
n_samples: int = 10_000,
|
|
237
|
+
) -> InteractionValues:
|
|
238
|
+
"""Compute and return the interaction values for mistunability analysis.
|
|
239
|
+
|
|
240
|
+
Args:
|
|
241
|
+
baseline_config (Configuration | None, optional): The baseline configuration. Defaults to None.
|
|
242
|
+
index (str, optional): The index to use for computing interaction values. Defaults to "FSII".
|
|
243
|
+
order (int, optional): The order of the interaction values. Defaults to 2.
|
|
244
|
+
n_samples (int, optional): The number of samples to use for simulating HPO. Defaults to 10_000.
|
|
245
|
+
|
|
246
|
+
Returns:
|
|
247
|
+
InteractionValues: The computed interaction values.
|
|
248
|
+
|
|
249
|
+
"""
|
|
250
|
+
if baseline_config is None:
|
|
251
|
+
baseline_config = self.explanation_task.config_space.get_default_configuration()
|
|
252
|
+
|
|
253
|
+
# setup explanation task
|
|
254
|
+
mistunability_task: MistunabilityExplanationTask = MistunabilityExplanationTask(
|
|
255
|
+
config_space=self.explanation_task.config_space,
|
|
256
|
+
surrogate_model=self.explanation_task.surrogate_model,
|
|
257
|
+
baseline_config=baseline_config,
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
# setup tunability game and get interaction values
|
|
261
|
+
tg = MistunabilityGame(
|
|
262
|
+
explanation_task=mistunability_task,
|
|
263
|
+
cs_searcher=RandomConfigSpaceSearcher(
|
|
264
|
+
explanation_task=mistunability_task,
|
|
265
|
+
n_samples=n_samples,
|
|
266
|
+
mode="min",
|
|
267
|
+
),
|
|
268
|
+
n_workers=self.n_workers,
|
|
269
|
+
verbose=self.verbose,
|
|
270
|
+
)
|
|
271
|
+
return self.__get_interaction_values(game=tg, index=index, order=order)
|
|
272
|
+
|
|
273
|
+
def optimizer_bias(
|
|
274
|
+
self,
|
|
275
|
+
optimizer_of_interest: ConfigSpaceSearcher,
|
|
276
|
+
optimizer_ensemble: list[ConfigSpaceSearcher],
|
|
277
|
+
index: str = "FSII",
|
|
278
|
+
order: int = 2,
|
|
279
|
+
) -> InteractionValues:
|
|
280
|
+
"""Compute and return the interaction values for optimizer bias analysis.
|
|
281
|
+
|
|
282
|
+
Args:
|
|
283
|
+
optimizer_of_interest (ConfigSpaceSearcher): The optimizer of interest.
|
|
284
|
+
optimizer_ensemble (list[ConfigSpaceSearcher]): The ensemble of optimizers.
|
|
285
|
+
index (str, optional): The index to use for computing interaction values. Defaults to "FSII".
|
|
286
|
+
order (int, optional): The order of the interaction values. Defaults to 2.
|
|
287
|
+
|
|
288
|
+
Returns:
|
|
289
|
+
InteractionValues: The computed interaction values.
|
|
290
|
+
|
|
291
|
+
"""
|
|
292
|
+
# setup explanation task
|
|
293
|
+
optimizer_bias_task: OptimizerBiasExplanationTask = OptimizerBiasExplanationTask(
|
|
294
|
+
config_space=self.explanation_task.config_space,
|
|
295
|
+
surrogate_model=self.explanation_task.surrogate_model,
|
|
296
|
+
optimizer_of_interest=optimizer_of_interest,
|
|
297
|
+
optimizer_ensemble=optimizer_ensemble,
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
# setup optimizer bias game and get interaction values
|
|
301
|
+
og = OptimizerBiasGame(explanation_task=optimizer_bias_task, n_workers=self.n_workers, verbose=self.verbose)
|
|
302
|
+
return self.__get_interaction_values(game=og, index=index, order=order)
|
|
303
|
+
|
|
304
|
+
def plot_si_graph(self, interaction_values: InteractionValues | None = None, save_path: str | None = None) -> None:
|
|
305
|
+
"""Plot the SHAP interaction values as a graph.
|
|
306
|
+
|
|
307
|
+
Args:
|
|
308
|
+
interaction_values (InteractionValues | None, optional): The interaction values to plot. Defaults to None.
|
|
309
|
+
save_path (str | None, optional): The path to save the plot. Defaults to None.
|
|
310
|
+
|
|
311
|
+
"""
|
|
312
|
+
if interaction_values is None and self.last_interaction_values is None:
|
|
313
|
+
raise NoInteractionValuesError
|
|
314
|
+
|
|
315
|
+
# if given interaction values use those, else use cached interaction values
|
|
316
|
+
iv = interaction_values if interaction_values is not None else self.last_interaction_values
|
|
317
|
+
|
|
318
|
+
if not isinstance(iv, InteractionValues):
|
|
319
|
+
raise TypeError
|
|
320
|
+
|
|
321
|
+
hyperparameter_names = self.explanation_task.get_hyperparameter_names()
|
|
322
|
+
|
|
323
|
+
def get_circular_layout(n_players: int) -> dict:
|
|
324
|
+
original_graph, graph_nodes = nx.Graph(), []
|
|
325
|
+
for i in range(n_players):
|
|
326
|
+
original_graph.add_node(i, label=i)
|
|
327
|
+
graph_nodes.append(i)
|
|
328
|
+
return nx.circular_layout(original_graph)
|
|
329
|
+
|
|
330
|
+
pos = get_circular_layout(n_players=self.explanation_task.get_num_hyperparameters())
|
|
331
|
+
iv.plot_si_graph(
|
|
332
|
+
show=False,
|
|
333
|
+
size_factor=3.0,
|
|
334
|
+
feature_names=hyperparameter_names,
|
|
335
|
+
pos=pos,
|
|
336
|
+
n_interactions=1_000,
|
|
337
|
+
compactness=1e50,
|
|
338
|
+
)
|
|
339
|
+
plt.tight_layout()
|
|
340
|
+
|
|
341
|
+
if save_path is not None:
|
|
342
|
+
plt.savefig(save_path)
|
|
343
|
+
logger.info("Saved SI graph to %s", save_path)
|
|
344
|
+
|
|
345
|
+
plt.show()
|
|
346
|
+
|
|
347
|
+
def plot_upset(self, interaction_values: InteractionValues | None = None, save_path: str | None = None) -> None:
|
|
348
|
+
"""Plot the SHAP interaction values as an upset plot graph.
|
|
349
|
+
|
|
350
|
+
Args:
|
|
351
|
+
interaction_values (InteractionValues | None, optional): The interaction values to plot. Defaults to None.
|
|
352
|
+
save_path (str | None, optional): The path to save the plot. Defaults to None.
|
|
353
|
+
|
|
354
|
+
"""
|
|
355
|
+
if interaction_values is None and self.last_interaction_values is None:
|
|
356
|
+
raise NoInteractionValuesError
|
|
357
|
+
|
|
358
|
+
# if given interaction values use those, else use cached interaction values
|
|
359
|
+
iv = interaction_values if interaction_values is not None else self.last_interaction_values
|
|
360
|
+
|
|
361
|
+
if not isinstance(iv, InteractionValues):
|
|
362
|
+
raise TypeError
|
|
363
|
+
|
|
364
|
+
hyperparameter_names = self.explanation_task.get_hyperparameter_names()
|
|
365
|
+
|
|
366
|
+
fig = iv.plot_upset(feature_names=hyperparameter_names, show=False)
|
|
367
|
+
|
|
368
|
+
if fig is None:
|
|
369
|
+
raise TypeError
|
|
370
|
+
|
|
371
|
+
ax = fig.get_axes()[0]
|
|
372
|
+
ax.set_ylabel("Performance Gain")
|
|
373
|
+
# also add "parameter" to the y-axis label
|
|
374
|
+
ax = fig.get_axes()[1]
|
|
375
|
+
ax.set_ylabel("Hyperparameter")
|
|
376
|
+
|
|
377
|
+
plt.tight_layout()
|
|
378
|
+
|
|
379
|
+
if save_path is not None:
|
|
380
|
+
plt.savefig(save_path)
|
|
381
|
+
|
|
382
|
+
plt.show()
|
|
383
|
+
|
|
384
|
+
def plot_force(self, interaction_values: InteractionValues | None = None, save_path: str | None = None) -> None:
|
|
385
|
+
"""Plot the SHAP interaction values as a forceplot graph.
|
|
386
|
+
|
|
387
|
+
Args:
|
|
388
|
+
interaction_values: Interaction values to plot. Defaults to None.
|
|
389
|
+
save_path: The path to save the plot. Defaults to None.
|
|
390
|
+
|
|
391
|
+
"""
|
|
392
|
+
if interaction_values is None and self.last_interaction_values is None:
|
|
393
|
+
raise NoInteractionValuesError
|
|
394
|
+
|
|
395
|
+
# if given interaction values use those, else use cached interaction values
|
|
396
|
+
iv = interaction_values if interaction_values is not None else self.last_interaction_values
|
|
397
|
+
|
|
398
|
+
if not isinstance(iv, InteractionValues):
|
|
399
|
+
raise TypeError
|
|
400
|
+
|
|
401
|
+
hyperparameter_names = self.explanation_task.get_hyperparameter_names()
|
|
402
|
+
|
|
403
|
+
iv.plot_force(feature_names=np.array(hyperparameter_names), show=False)
|
|
404
|
+
plt.tight_layout()
|
|
405
|
+
|
|
406
|
+
if save_path is not None:
|
|
407
|
+
plt.savefig(save_path)
|
|
408
|
+
|
|
409
|
+
plt.show()
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""The surrogate module defines the basic classes for surrogate models.
|
|
2
|
+
|
|
3
|
+
It provides methods for training and evaluating a model that approximates
|
|
4
|
+
the relationship between input hyperparameters and performance.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import TYPE_CHECKING, Any, Protocol, cast
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from ConfigSpace import Configuration, ConfigurationSpace
|
|
13
|
+
from sklearn.base import BaseEstimator
|
|
14
|
+
|
|
15
|
+
from abc import ABC, abstractmethod
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
from sklearn.ensemble import RandomForestRegressor
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class SklearnRegressorProtocol(Protocol):
|
|
22
|
+
"""Defines the interface for scikit-learn-like regression models.
|
|
23
|
+
|
|
24
|
+
This protocol specifies the required methods for a class to be considered
|
|
25
|
+
a compatible scikit-learn regression model. It mandates the presence of `fit`,
|
|
26
|
+
`predict`, and `score` methods, mirroring the structure of many
|
|
27
|
+
scikit-learn estimators.
|
|
28
|
+
|
|
29
|
+
Attributes:
|
|
30
|
+
fit (callable): A method that fits the model to the provided data.
|
|
31
|
+
It should accept training data (X) and target variables (y)
|
|
32
|
+
as arguments, and any optional fit parameters. It should return
|
|
33
|
+
the fitted model instance itself (allowing for chaining).
|
|
34
|
+
predict (callable): A method that generates predictions for a given input dataset.
|
|
35
|
+
It accepts input data (X) and returns predictions as a NumPy array.
|
|
36
|
+
score (callable): A method that evaluates the model's performance on a given dataset.
|
|
37
|
+
It accepts input data (X) and corresponding target variables (y)
|
|
38
|
+
and returns a scalar performance score (e.g., R-squared, MSE).
|
|
39
|
+
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def fit(self, X: Any, y: Any, **fit_params: Any) -> None:
|
|
43
|
+
"""Fit the regression model to the provided training data.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
X (np.ndarray): The training data features.
|
|
47
|
+
y (np.ndarray): The training data target variables.
|
|
48
|
+
**fit_params (Any): Optional keyword arguments passed to the fit method.
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
'SklearnRegressorProtocol': The fitted regression model instance itself, allowing for method chaining.
|
|
52
|
+
|
|
53
|
+
"""
|
|
54
|
+
...
|
|
55
|
+
|
|
56
|
+
def predict(self, X: Any) -> np.ndarray:
|
|
57
|
+
"""Generate predictions for a given input dataset.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
X (np.ndarray): The input data features for prediction.
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
np.ndarray: The predicted target values as a NumPy array.
|
|
64
|
+
|
|
65
|
+
"""
|
|
66
|
+
...
|
|
67
|
+
|
|
68
|
+
def score(self, X: Any, y: Any) -> float:
|
|
69
|
+
"""Evaluate the model's performance on a given dataset.
|
|
70
|
+
|
|
71
|
+
Args:
|
|
72
|
+
X (np.ndarray): The input data features.
|
|
73
|
+
y (np.ndarray): The corresponding target variables.
|
|
74
|
+
|
|
75
|
+
Returns:
|
|
76
|
+
float: A scalar performance score, representing the model's accuracy on the dataset.
|
|
77
|
+
|
|
78
|
+
"""
|
|
79
|
+
...
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class SurrogateModel(ABC):
|
|
83
|
+
"""An abstract class for defining the interface of surrogate models.
|
|
84
|
+
|
|
85
|
+
This class defines the basic methods that all surrogate models should implement,
|
|
86
|
+
allowing for a consistent interface for evaluating different models.
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
def __init__(self, config_space: ConfigurationSpace) -> None:
|
|
90
|
+
"""Initialize the SurrogateModel with a configuration space.
|
|
91
|
+
|
|
92
|
+
Args:
|
|
93
|
+
config_space: The configuration space for the surrogate model.
|
|
94
|
+
|
|
95
|
+
"""
|
|
96
|
+
self.config_space = config_space
|
|
97
|
+
|
|
98
|
+
def evaluate_config(self, config: Configuration) -> float:
|
|
99
|
+
"""Evaluate a single configuration using the surrogate model.
|
|
100
|
+
|
|
101
|
+
Args:
|
|
102
|
+
config: The configuration to evaluate.
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
The predicted performance for the given configuration.
|
|
106
|
+
|
|
107
|
+
"""
|
|
108
|
+
res = self.evaluate(np.array(config.get_array()))
|
|
109
|
+
if not isinstance(res, float):
|
|
110
|
+
raise TypeError
|
|
111
|
+
return res
|
|
112
|
+
|
|
113
|
+
def evaluate_config_batch(self, config_batch: list[Configuration]) -> list[float]:
|
|
114
|
+
"""Evaluate a batch of configurations using the surrogate model.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
config_batch: A list of configurations to evaluate.
|
|
118
|
+
|
|
119
|
+
Returns:
|
|
120
|
+
A list of predicted performances for the given configurations.
|
|
121
|
+
|
|
122
|
+
"""
|
|
123
|
+
res = self.evaluate(np.array([config.get_array() for config in config_batch]))
|
|
124
|
+
if not isinstance(res, list):
|
|
125
|
+
raise TypeError
|
|
126
|
+
return res
|
|
127
|
+
|
|
128
|
+
@abstractmethod
|
|
129
|
+
def evaluate(self, config_array: np.ndarray) -> float | list[float]:
|
|
130
|
+
"""Evaluate a configuration (or batch of configurations) represented as a numpy array.
|
|
131
|
+
|
|
132
|
+
Args:
|
|
133
|
+
config_array: A numpy array representing the configuration(s).
|
|
134
|
+
|
|
135
|
+
Returns:
|
|
136
|
+
The predicted performance(s).
|
|
137
|
+
|
|
138
|
+
"""
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class ModelBasedSurrogateModel(SurrogateModel):
|
|
142
|
+
"""A surrogate model based on a pre-trained machine learning model."""
|
|
143
|
+
|
|
144
|
+
def __init__(self, config_space: ConfigurationSpace, base_model: BaseEstimator) -> None:
|
|
145
|
+
"""Initialize the ModelBasedSurrogateModel with a configuration space and a base model.
|
|
146
|
+
|
|
147
|
+
Args:
|
|
148
|
+
config_space: The configuration space.
|
|
149
|
+
base_model: The base machine learning model.
|
|
150
|
+
|
|
151
|
+
"""
|
|
152
|
+
super().__init__(config_space)
|
|
153
|
+
self.base_model = base_model
|
|
154
|
+
|
|
155
|
+
def evaluate_config(self, config: Configuration) -> float:
|
|
156
|
+
"""Evaluate a single configuration.
|
|
157
|
+
|
|
158
|
+
Args:
|
|
159
|
+
config: The configuration to evaluate.
|
|
160
|
+
|
|
161
|
+
Returns:
|
|
162
|
+
The predicted performance for the given configuration.
|
|
163
|
+
|
|
164
|
+
"""
|
|
165
|
+
res = self.evaluate(config.get_array())
|
|
166
|
+
if not isinstance(res, float):
|
|
167
|
+
raise TypeError
|
|
168
|
+
return res
|
|
169
|
+
|
|
170
|
+
def evaluate_config_batch(self, config_batch: list[Configuration]) -> list[float]:
|
|
171
|
+
"""Evaluate a batch of configurations.
|
|
172
|
+
|
|
173
|
+
Args:
|
|
174
|
+
config_batch: A list of configurations to evaluate.
|
|
175
|
+
|
|
176
|
+
Returns:
|
|
177
|
+
A list of predicted performances for the given configurations.
|
|
178
|
+
|
|
179
|
+
"""
|
|
180
|
+
res = self.evaluate(np.array([config.get_array() for config in config_batch]))
|
|
181
|
+
if not isinstance(res, list):
|
|
182
|
+
raise TypeError
|
|
183
|
+
return res
|
|
184
|
+
|
|
185
|
+
def evaluate(self, config_array: np.ndarray) -> float | list[float]:
|
|
186
|
+
"""Evaluate a configuration (or batch of configurations).
|
|
187
|
+
|
|
188
|
+
Args:
|
|
189
|
+
config_array: A numpy array representing the configuration(s).
|
|
190
|
+
|
|
191
|
+
Returns:
|
|
192
|
+
The predicted performance(s).
|
|
193
|
+
|
|
194
|
+
"""
|
|
195
|
+
if config_array.ndim == 1:
|
|
196
|
+
config_array = config_array.reshape(1, -1)
|
|
197
|
+
|
|
198
|
+
base_model = cast("SklearnRegressorProtocol", self.base_model)
|
|
199
|
+
predictions = base_model.predict(config_array)
|
|
200
|
+
|
|
201
|
+
if predictions.shape == (1,): # Check for a 1-element array (scalar)
|
|
202
|
+
return float(predictions[0]) # Convert to a Python float
|
|
203
|
+
|
|
204
|
+
return predictions.tolist() # Convert to a Python list
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
class DataBasedSurrogateModel(ModelBasedSurrogateModel):
|
|
208
|
+
"""A surrogate model trained on a dataset of configurations and their performance."""
|
|
209
|
+
|
|
210
|
+
def __init__(
|
|
211
|
+
self,
|
|
212
|
+
config_space: ConfigurationSpace,
|
|
213
|
+
data: list[tuple[Configuration, float]],
|
|
214
|
+
base_model: BaseEstimator | None = None,
|
|
215
|
+
) -> None:
|
|
216
|
+
"""Initialize the DataBasedSurrogateModel with data and an optional base model.
|
|
217
|
+
|
|
218
|
+
Args:
|
|
219
|
+
config_space: The configuration space.
|
|
220
|
+
data: The data to be used for fitting the surrogate model. Each element
|
|
221
|
+
is a tuple of (Configuration, float).
|
|
222
|
+
base_model: The base model to be used for fitting the surrogate model.
|
|
223
|
+
If None, a RandomForestRegressor is used.
|
|
224
|
+
|
|
225
|
+
"""
|
|
226
|
+
train_x = np.array([obs[0].get_array() for obs in data])
|
|
227
|
+
train_y = np.array([obs[1] for obs in data])
|
|
228
|
+
|
|
229
|
+
if base_model is None:
|
|
230
|
+
base_model = RandomForestRegressor()
|
|
231
|
+
|
|
232
|
+
pipeline = cast("SklearnRegressorProtocol", base_model)
|
|
233
|
+
pipeline.fit(train_x, train_y)
|
|
234
|
+
|
|
235
|
+
super().__init__(config_space, base_model)
|