tabpackage 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tabpackage/__init__.py +18 -0
- tabpackage/_training.py +3380 -0
- tabpackage/ensemble.py +263 -0
- tabpackage/metrics.py +253 -0
- tabpackage/nn.py +2110 -0
- tabpackage/optim.py +943 -0
- tabpackage/sklearn/__init__.py +18 -0
- tabpackage/sklearn/_config.py +561 -0
- tabpackage/sklearn/_data.py +307 -0
- tabpackage/sklearn/_estimators.py +1210 -0
- tabpackage/utils/__init__.py +1 -0
- tabpackage/utils/torch.py +166 -0
- tabpackage-0.0.1.dist-info/METADATA +398 -0
- tabpackage-0.0.1.dist-info/RECORD +15 -0
- tabpackage-0.0.1.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""The Scikit-learn-compatible part of TabPackage."""
|
|
2
|
+
|
|
3
|
+
from tabpackage.sklearn._config import ( # noqa: F401
|
|
4
|
+
Ensembles,
|
|
5
|
+
GreedyEnsemble,
|
|
6
|
+
HyperparameterBounds,
|
|
7
|
+
IntUniformDistribution,
|
|
8
|
+
LogUniformDistribution,
|
|
9
|
+
OptionalDistribution,
|
|
10
|
+
OptionalHyperparameterBounds,
|
|
11
|
+
PerModelHyperparameters,
|
|
12
|
+
TopKEnsemble,
|
|
13
|
+
UniformDistribution,
|
|
14
|
+
)
|
|
15
|
+
from tabpackage.sklearn._estimators import ( # noqa: F401
|
|
16
|
+
TabPackClassifier,
|
|
17
|
+
TabPackRegressor,
|
|
18
|
+
)
|
|
@@ -0,0 +1,561 @@
|
|
|
1
|
+
import math
|
|
2
|
+
import random
|
|
3
|
+
from functools import partial
|
|
4
|
+
from typing import Any, Generic, Literal, Self, TypeAlias, TypedDict, TypeVar
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
import torch
|
|
8
|
+
from sklearn.base import BaseEstimator
|
|
9
|
+
from sklearn.utils.metaestimators import _BaseComposition
|
|
10
|
+
from torch import Tensor
|
|
11
|
+
|
|
12
|
+
import tabpackage
|
|
13
|
+
from tabpackage import ensemble, metrics
|
|
14
|
+
|
|
15
|
+
KWArgs: TypeAlias = dict[str, Any]
|
|
16
|
+
DatasetName: TypeAlias = str
|
|
17
|
+
MetricName: TypeAlias = Literal['accuracy', 'log_loss', 'roc_auc', 'mse', 'mae', 'r2']
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# >>> Per-model hyperparameters
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Distribution(BaseEstimator):
|
|
24
|
+
def _get_bounds(self) -> tuple[float, float]:
|
|
25
|
+
raise NotImplementedError
|
|
26
|
+
|
|
27
|
+
def sample(self, rng: random.Random) -> float:
|
|
28
|
+
"""Draw a value using a given random generator."""
|
|
29
|
+
raise NotImplementedError
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class IntUniformDistribution(Distribution):
|
|
33
|
+
"""A uniform distribution over integers within inclusive bounds."""
|
|
34
|
+
|
|
35
|
+
def __init__(self, min: int, max: int) -> None:
|
|
36
|
+
self.min = min
|
|
37
|
+
self.max = max
|
|
38
|
+
|
|
39
|
+
def _get_bounds(self) -> tuple[int, int]:
|
|
40
|
+
if self.min > self.max:
|
|
41
|
+
raise ValueError('`min` must not exceed `max`')
|
|
42
|
+
|
|
43
|
+
return self.min, self.max
|
|
44
|
+
|
|
45
|
+
def sample(self, rng: random.Random) -> int:
|
|
46
|
+
self._get_bounds()
|
|
47
|
+
return rng.randint(self.min, self.max)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class UniformDistribution(Distribution):
|
|
51
|
+
"""A uniform distribution between finite bounds."""
|
|
52
|
+
|
|
53
|
+
def __init__(self, min: float, max: float) -> None:
|
|
54
|
+
self.min = min
|
|
55
|
+
self.max = max
|
|
56
|
+
|
|
57
|
+
def _get_bounds(self) -> tuple[float, float]:
|
|
58
|
+
if not math.isfinite(self.min) or not math.isfinite(self.max):
|
|
59
|
+
raise ValueError('`min` and `max` must be finite')
|
|
60
|
+
if self.min > self.max:
|
|
61
|
+
raise ValueError('`min` must not exceed `max`')
|
|
62
|
+
|
|
63
|
+
return self.min, self.max
|
|
64
|
+
|
|
65
|
+
def sample(self, rng: random.Random) -> float:
|
|
66
|
+
self._get_bounds()
|
|
67
|
+
return rng.uniform(self.min, self.max)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class LogUniformDistribution(UniformDistribution):
|
|
71
|
+
"""A log-uniform distribution between positive finite bounds."""
|
|
72
|
+
|
|
73
|
+
def _get_bounds(self) -> tuple[float, float]:
|
|
74
|
+
bounds = super()._get_bounds()
|
|
75
|
+
if self.min <= 0.0:
|
|
76
|
+
raise ValueError('`min` must be positive')
|
|
77
|
+
|
|
78
|
+
return bounds
|
|
79
|
+
|
|
80
|
+
def sample(self, rng: random.Random) -> float:
|
|
81
|
+
self._get_bounds()
|
|
82
|
+
return math.exp(rng.uniform(math.log(self.min), math.log(self.max)))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
_DistributionT = TypeVar('_DistributionT', bound=Distribution)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class OptionalDistribution(Distribution, Generic[_DistributionT]):
|
|
89
|
+
"""A mixture of a given distribution (probability p) and zero."""
|
|
90
|
+
|
|
91
|
+
def __init__(self, distribution: _DistributionT, p: float = 0.5) -> None:
|
|
92
|
+
self.distribution = distribution
|
|
93
|
+
self.p = p
|
|
94
|
+
|
|
95
|
+
def _get_bounds(self) -> tuple[float, float]:
|
|
96
|
+
if not math.isfinite(self.p) or self.p < 0.0 or self.p > 1.0:
|
|
97
|
+
raise ValueError('`p` must be finite and between 0 and 1, inclusive')
|
|
98
|
+
|
|
99
|
+
min_value, max_value = self.distribution._get_bounds()
|
|
100
|
+
return min(0.0, min_value), max(0.0, max_value)
|
|
101
|
+
|
|
102
|
+
def sample(self, rng: random.Random) -> float:
|
|
103
|
+
self._get_bounds()
|
|
104
|
+
return self.distribution.sample(rng) if rng.random() < self.p else 0.0
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
_HyperparameterT = TypeVar('_HyperparameterT', int, float)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class HyperparameterBounds(TypedDict, Generic[_HyperparameterT]):
|
|
111
|
+
"""The lower and upper bounds for sampling a hyperparameter."""
|
|
112
|
+
|
|
113
|
+
min: _HyperparameterT
|
|
114
|
+
max: _HyperparameterT
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class OptionalHyperparameterBounds(HyperparameterBounds[float]):
|
|
118
|
+
"""Hyperparameter sampling bounds for an optimal hyperparameter."""
|
|
119
|
+
|
|
120
|
+
optional: bool
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class PerModelHyperparameters(TypedDict, total=False):
|
|
124
|
+
"""Hyperparameters for one model, overriding the shared hyperparameters."""
|
|
125
|
+
|
|
126
|
+
n_blocks: int
|
|
127
|
+
d_block: int
|
|
128
|
+
dropout: float
|
|
129
|
+
numerical_embeddings_d: int
|
|
130
|
+
numerical_embeddings_init_scale: float
|
|
131
|
+
adamw_lr: float
|
|
132
|
+
adamw_beta1: float
|
|
133
|
+
adamw_beta2: float
|
|
134
|
+
adamw_eps: float
|
|
135
|
+
muon_lr: float
|
|
136
|
+
muon_momentum: float
|
|
137
|
+
weight_decay: float
|
|
138
|
+
max_epochs: int | None
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def validate_hyperparameter(name: str, value: float | Distribution) -> None:
|
|
142
|
+
if not isinstance(value, Distribution):
|
|
143
|
+
if not math.isfinite(value):
|
|
144
|
+
raise ValueError(f'`{name}` must be finite')
|
|
145
|
+
|
|
146
|
+
if name in ('numerical_embeddings_init_scale', 'adamw_lr', 'muon_lr'):
|
|
147
|
+
if value <= 0.0:
|
|
148
|
+
raise ValueError(f'`{name}` must be positive')
|
|
149
|
+
elif name == 'dropout' and value >= 1.0:
|
|
150
|
+
raise ValueError('`dropout` must be less than 1')
|
|
151
|
+
|
|
152
|
+
return
|
|
153
|
+
|
|
154
|
+
min_value, max_value = value._get_bounds()
|
|
155
|
+
if not math.isfinite(min_value) or not math.isfinite(max_value):
|
|
156
|
+
raise ValueError(f'The distribution bounds for `{name}` must be finite')
|
|
157
|
+
|
|
158
|
+
if name in ('dropout', 'adamw_beta1', 'adamw_beta2', 'muon_momentum'):
|
|
159
|
+
if min_value < 0.0 or min_value > max_value or max_value >= 1.0:
|
|
160
|
+
raise ValueError(
|
|
161
|
+
f'The distribution bounds for `{name}` must be ordered and lie in [0,'
|
|
162
|
+
' 1)'
|
|
163
|
+
)
|
|
164
|
+
elif name == 'weight_decay':
|
|
165
|
+
if min_value < 0.0:
|
|
166
|
+
raise ValueError(
|
|
167
|
+
'The lower distribution bound for `weight_decay` must be non-negative'
|
|
168
|
+
)
|
|
169
|
+
else:
|
|
170
|
+
if min_value <= 0:
|
|
171
|
+
raise ValueError(
|
|
172
|
+
f'The lower distribution bound for `{name}` must be positive'
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
# >>> Evaluation
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _accuracy_pack(*, y_true: Tensor, y_pred: Tensor) -> Tensor:
|
|
180
|
+
return metrics.accuracy_pack(y_true=y_true, y_pred=y_pred.argmax(dim=-1))
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _log_loss_pack(*, y_true: Tensor, y_pred: Tensor) -> Tensor:
|
|
184
|
+
return metrics.multiclass_cross_entropy_pack(y_true=y_true, y_proba=y_pred)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _roc_auc_pack(*, y_true: Tensor, y_pred: Tensor) -> Tensor:
|
|
188
|
+
return metrics.binary_roc_auc_pack(y_true=y_true, y_score=y_pred[..., 1])
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _resolve_metric(
|
|
192
|
+
metric: str, d_out: int
|
|
193
|
+
) -> tuple[tabpackage.MetricFn, tabpackage.MetricDirection]:
|
|
194
|
+
allowed_metrics = (
|
|
195
|
+
('accuracy', 'log_loss', 'roc_auc') if d_out > 1 else ('mse', 'mae', 'r2')
|
|
196
|
+
)
|
|
197
|
+
if metric not in allowed_metrics:
|
|
198
|
+
raise ValueError(f'Unsupported metric for this task: {metric!r}')
|
|
199
|
+
if metric == 'roc_auc' and d_out != 2:
|
|
200
|
+
raise ValueError('ROC AUC requires binary classification')
|
|
201
|
+
|
|
202
|
+
metric_fn = {
|
|
203
|
+
'accuracy': _accuracy_pack,
|
|
204
|
+
'log_loss': _log_loss_pack,
|
|
205
|
+
'roc_auc': _roc_auc_pack,
|
|
206
|
+
'mse': metrics.mse_pack,
|
|
207
|
+
'mae': metrics.mae_pack,
|
|
208
|
+
'r2': metrics.r2_pack,
|
|
209
|
+
}[metric]
|
|
210
|
+
metric_direction = (
|
|
211
|
+
tabpackage.MetricDirection.MAXIMIZE
|
|
212
|
+
if metric in ('accuracy', 'roc_auc', 'r2')
|
|
213
|
+
else tabpackage.MetricDirection.MINIMIZE
|
|
214
|
+
)
|
|
215
|
+
return metric_fn, metric_direction
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def compute_scalar_metric(
|
|
219
|
+
metric_fn: tabpackage.MetricFn, *, y_true: Tensor, y_pred: Tensor
|
|
220
|
+
) -> Tensor:
|
|
221
|
+
return metric_fn(y_true=y_true, y_pred=y_pred.unsqueeze(0))[0]
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _validate_metric_data(
|
|
225
|
+
metric: str, dataset: DatasetName, labels: dict[DatasetName, np.ndarray]
|
|
226
|
+
) -> None:
|
|
227
|
+
if dataset not in labels:
|
|
228
|
+
raise ValueError(f'Unknown dataset: {dataset!r}')
|
|
229
|
+
|
|
230
|
+
if metric == 'roc_auc' and len(np.unique(labels[dataset])) != 2:
|
|
231
|
+
raise ValueError(f'ROC AUC requires both classes in dataset {dataset!r}')
|
|
232
|
+
elif metric == 'r2' and len(labels[dataset]) < 2:
|
|
233
|
+
raise ValueError(f'R2 requires at least two rows in dataset {dataset!r}')
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def resolve_criterion(
|
|
237
|
+
dataset: DatasetName | None,
|
|
238
|
+
metric: str | None,
|
|
239
|
+
*,
|
|
240
|
+
default_dataset: DatasetName | None,
|
|
241
|
+
default_metric: str | None,
|
|
242
|
+
d_out: int,
|
|
243
|
+
) -> tabpackage.EvalCriterionConfig | None:
|
|
244
|
+
metric = default_metric if metric is None else metric
|
|
245
|
+
if metric is None or metric == 'none':
|
|
246
|
+
return None
|
|
247
|
+
|
|
248
|
+
dataset = default_dataset if dataset is None else dataset
|
|
249
|
+
if dataset is None:
|
|
250
|
+
raise ValueError('An enabled criterion requires an explicit dataset')
|
|
251
|
+
|
|
252
|
+
_, metric_direction = _resolve_metric(metric, d_out)
|
|
253
|
+
return tabpackage.EvalCriterionConfig(
|
|
254
|
+
dataset=dataset, metric=metric, metric_direction=metric_direction
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def resolve_eval_metrics(
|
|
259
|
+
dataset_names: list[DatasetName] | None,
|
|
260
|
+
metric_names: list[MetricName] | None,
|
|
261
|
+
*,
|
|
262
|
+
eval_criterion: tabpackage.EvalCriterionConfig | None,
|
|
263
|
+
d_out: int,
|
|
264
|
+
labels: dict[DatasetName, np.ndarray],
|
|
265
|
+
) -> dict[DatasetName, dict[str, tabpackage.MetricFn]]:
|
|
266
|
+
if eval_criterion is None:
|
|
267
|
+
if (dataset_names is None) != (metric_names is None):
|
|
268
|
+
raise ValueError(
|
|
269
|
+
'Reporting without a criterion requires both datasets and metrics'
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
dataset_names = [] if dataset_names is None else dataset_names
|
|
273
|
+
resolved_metric_names = [] if metric_names is None else metric_names
|
|
274
|
+
else:
|
|
275
|
+
dataset_names = (
|
|
276
|
+
[eval_criterion.dataset] if dataset_names is None else dataset_names
|
|
277
|
+
)
|
|
278
|
+
resolved_metric_names = (
|
|
279
|
+
[eval_criterion.metric] if metric_names is None else metric_names
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
for dataset in dataset_names:
|
|
283
|
+
if dataset not in labels:
|
|
284
|
+
raise ValueError(f'Unknown dataset: {dataset!r}')
|
|
285
|
+
|
|
286
|
+
metric_fns = {
|
|
287
|
+
name: _resolve_metric(name, d_out)[0] for name in resolved_metric_names
|
|
288
|
+
}
|
|
289
|
+
eval_metrics = {
|
|
290
|
+
dataset: metric_fns.copy() for dataset in dataset_names if metric_fns
|
|
291
|
+
}
|
|
292
|
+
if eval_criterion is not None:
|
|
293
|
+
eval_metrics.setdefault(eval_criterion.dataset, {})[eval_criterion.metric] = (
|
|
294
|
+
_resolve_metric(eval_criterion.metric, d_out)[0]
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
for dataset, dataset_metrics in eval_metrics.items():
|
|
298
|
+
for metric in dataset_metrics:
|
|
299
|
+
_validate_metric_data(metric, dataset, labels)
|
|
300
|
+
|
|
301
|
+
return eval_metrics
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
# >>> Devices and precision
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def resolve_device(device: str | torch.device | Literal['auto']) -> torch.device:
|
|
308
|
+
if device == 'auto':
|
|
309
|
+
device = (
|
|
310
|
+
'cuda'
|
|
311
|
+
if torch.cuda.is_available()
|
|
312
|
+
else 'mps'
|
|
313
|
+
if torch.backends.mps.is_available()
|
|
314
|
+
else 'cpu'
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
device = torch.device(device)
|
|
318
|
+
if device.type == 'cuda':
|
|
319
|
+
if not torch.cuda.is_available():
|
|
320
|
+
raise RuntimeError('CUDA is not available')
|
|
321
|
+
|
|
322
|
+
index = torch.cuda.current_device() if device.index is None else device.index
|
|
323
|
+
if index >= torch.cuda.device_count():
|
|
324
|
+
raise ValueError('The CUDA device index is out of range')
|
|
325
|
+
|
|
326
|
+
return torch.device('cuda', index)
|
|
327
|
+
if device.type == 'mps':
|
|
328
|
+
if not torch.backends.mps.is_available():
|
|
329
|
+
raise RuntimeError('MPS is not available')
|
|
330
|
+
if device.index not in (None, 0):
|
|
331
|
+
raise ValueError('The MPS device index must be unspecified or zero')
|
|
332
|
+
|
|
333
|
+
return torch.device('mps')
|
|
334
|
+
if device.type != 'cpu':
|
|
335
|
+
raise ValueError('The device type is not supported')
|
|
336
|
+
|
|
337
|
+
return torch.device('cpu')
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def resolve_autocast_dtype(dtype: Literal['bfloat16'] | None) -> torch.dtype | None:
|
|
341
|
+
return {None: None, 'bfloat16': torch.bfloat16}[dtype]
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
# >>> Ensembles
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
class _Ensemble(BaseEstimator):
|
|
348
|
+
online: bool
|
|
349
|
+
early_stopping_patience: int | None
|
|
350
|
+
algorithm_metric: MetricName | None
|
|
351
|
+
algorithm_dataset: DatasetName | None
|
|
352
|
+
algorithm_device: str | torch.device | Literal['auto']
|
|
353
|
+
|
|
354
|
+
def _make_algorithm_config(self) -> KWArgs:
|
|
355
|
+
raise NotImplementedError
|
|
356
|
+
|
|
357
|
+
def _make_config(
|
|
358
|
+
self,
|
|
359
|
+
*,
|
|
360
|
+
d_out: int,
|
|
361
|
+
eval_criterion: tabpackage.EvalCriterionConfig | None,
|
|
362
|
+
default_algorithm_dataset: DatasetName | None,
|
|
363
|
+
labels: dict[DatasetName, np.ndarray],
|
|
364
|
+
device: torch.device,
|
|
365
|
+
) -> tabpackage.EnsembleConfig:
|
|
366
|
+
if self.algorithm_metric is None and eval_criterion is None:
|
|
367
|
+
raise ValueError(
|
|
368
|
+
'Without the evaluation criterion, `algorithm_metric` must be specified'
|
|
369
|
+
)
|
|
370
|
+
if (
|
|
371
|
+
self.algorithm_dataset is None
|
|
372
|
+
and eval_criterion is None
|
|
373
|
+
and default_algorithm_dataset is None
|
|
374
|
+
):
|
|
375
|
+
raise ValueError(
|
|
376
|
+
'Without the evaluation criterion and with extra datasets passed as'
|
|
377
|
+
' dictionaries, `algorithm_dataset` must be specified'
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
if self.algorithm_metric is None:
|
|
381
|
+
assert eval_criterion is not None
|
|
382
|
+
metric = eval_criterion.metric
|
|
383
|
+
else:
|
|
384
|
+
metric = self.algorithm_metric
|
|
385
|
+
|
|
386
|
+
dataset = (
|
|
387
|
+
self.algorithm_dataset
|
|
388
|
+
if self.algorithm_dataset is not None
|
|
389
|
+
else (
|
|
390
|
+
eval_criterion.dataset
|
|
391
|
+
if eval_criterion is not None
|
|
392
|
+
else default_algorithm_dataset
|
|
393
|
+
)
|
|
394
|
+
)
|
|
395
|
+
assert dataset is not None
|
|
396
|
+
|
|
397
|
+
_validate_metric_data(metric, dataset, labels)
|
|
398
|
+
metric_fn, metric_direction = _resolve_metric(metric, d_out)
|
|
399
|
+
|
|
400
|
+
return tabpackage.EnsembleConfig(
|
|
401
|
+
algorithm=self._make_algorithm_config(),
|
|
402
|
+
online=self.online,
|
|
403
|
+
prediction_transform=(
|
|
404
|
+
partial(torch.clamp_max, max=1.0) if d_out > 1 else None
|
|
405
|
+
),
|
|
406
|
+
early_stopping_patience=(
|
|
407
|
+
None if eval_criterion is None else self.early_stopping_patience
|
|
408
|
+
),
|
|
409
|
+
algorithm_metric=metric_fn,
|
|
410
|
+
algorithm_metric_direction=metric_direction,
|
|
411
|
+
algorithm_dataset=dataset,
|
|
412
|
+
algorithm_device=(
|
|
413
|
+
device
|
|
414
|
+
if self.algorithm_device == 'auto'
|
|
415
|
+
else resolve_device(self.algorithm_device)
|
|
416
|
+
),
|
|
417
|
+
)
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
class GreedyEnsemble(_Ensemble):
|
|
421
|
+
"""Configuration of a greedy ensemble."""
|
|
422
|
+
|
|
423
|
+
def __init__(
|
|
424
|
+
self,
|
|
425
|
+
*,
|
|
426
|
+
init_top_k: int = 1,
|
|
427
|
+
max_ensemble_size: int | None = 32,
|
|
428
|
+
with_replacement: bool = False,
|
|
429
|
+
online: bool = True,
|
|
430
|
+
early_stopping_patience: int | None = 32,
|
|
431
|
+
algorithm_metric: MetricName | None = None,
|
|
432
|
+
algorithm_dataset: DatasetName | None = None,
|
|
433
|
+
algorithm_device: str | torch.device | Literal['auto'] = 'auto',
|
|
434
|
+
) -> None:
|
|
435
|
+
"""Greedy ensemble configuration.
|
|
436
|
+
|
|
437
|
+
Args:
|
|
438
|
+
init_top_k: the number of top-performing models to start with.
|
|
439
|
+
max_ensemble_size: the maximum number of ensemble members.
|
|
440
|
+
with_replacement: whether a model can be selected more than once.
|
|
441
|
+
online: whether to consider models that are still training.
|
|
442
|
+
early_stopping_patience: the number of updates without improvement
|
|
443
|
+
tolerated before finalizing the ensemble.
|
|
444
|
+
algorithm_metric: the metric used to select ensemble members.
|
|
445
|
+
algorithm_dataset: the dataset used to select ensemble members.
|
|
446
|
+
algorithm_device: the device used to build the ensemble.
|
|
447
|
+
"""
|
|
448
|
+
self.init_top_k = init_top_k
|
|
449
|
+
self.max_ensemble_size = max_ensemble_size
|
|
450
|
+
self.with_replacement = with_replacement
|
|
451
|
+
self.online = online
|
|
452
|
+
self.early_stopping_patience = early_stopping_patience
|
|
453
|
+
self.algorithm_metric = algorithm_metric
|
|
454
|
+
self.algorithm_dataset = algorithm_dataset
|
|
455
|
+
self.algorithm_device = algorithm_device
|
|
456
|
+
|
|
457
|
+
def _make_algorithm_config(self) -> KWArgs:
|
|
458
|
+
return {
|
|
459
|
+
'type': ensemble.greedy_ensemble,
|
|
460
|
+
'init_top_k': self.init_top_k,
|
|
461
|
+
'max_ensemble_size': self.max_ensemble_size,
|
|
462
|
+
'with_replacement': self.with_replacement,
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
class TopKEnsemble(_Ensemble):
|
|
467
|
+
"""Configuration of an equally weighted ensemble of the top-k performers."""
|
|
468
|
+
|
|
469
|
+
def __init__(
|
|
470
|
+
self,
|
|
471
|
+
*,
|
|
472
|
+
k: int,
|
|
473
|
+
online: bool = True,
|
|
474
|
+
early_stopping_patience: int | None = 32,
|
|
475
|
+
algorithm_metric: MetricName | None = None,
|
|
476
|
+
algorithm_dataset: DatasetName | None = None,
|
|
477
|
+
algorithm_device: str | torch.device | Literal['auto'] = 'auto',
|
|
478
|
+
) -> None:
|
|
479
|
+
"""Top-K ensemble configuration.
|
|
480
|
+
|
|
481
|
+
Args:
|
|
482
|
+
k: the number of models to select.
|
|
483
|
+
online: whether to consider models that are still training.
|
|
484
|
+
early_stopping_patience: the number of updates without improvement
|
|
485
|
+
tolerated before finalizing the ensemble.
|
|
486
|
+
algorithm_metric: the metric used to rank models.
|
|
487
|
+
algorithm_dataset: the dataset used to rank models.
|
|
488
|
+
algorithm_device: the device used to build the ensemble.
|
|
489
|
+
"""
|
|
490
|
+
self.k = k
|
|
491
|
+
self.online = online
|
|
492
|
+
self.early_stopping_patience = early_stopping_patience
|
|
493
|
+
self.algorithm_metric = algorithm_metric
|
|
494
|
+
self.algorithm_dataset = algorithm_dataset
|
|
495
|
+
self.algorithm_device = algorithm_device
|
|
496
|
+
|
|
497
|
+
def _make_algorithm_config(self) -> KWArgs:
|
|
498
|
+
return {'type': ensemble.topk_ensemble, 'k': self.k}
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
class Ensembles(_BaseComposition):
|
|
502
|
+
"""A collection of named ensemble configurations."""
|
|
503
|
+
|
|
504
|
+
def __init__(
|
|
505
|
+
self, members: dict[str, GreedyEnsemble | TopKEnsemble | KWArgs]
|
|
506
|
+
) -> None:
|
|
507
|
+
"""Named ensemble configurations.
|
|
508
|
+
|
|
509
|
+
Each ensemble configuration is accessible through `get_params` and `set_params`
|
|
510
|
+
through names such as `my_ensemble__early_stopping_patience`.
|
|
511
|
+
"""
|
|
512
|
+
self.members = members
|
|
513
|
+
|
|
514
|
+
@property
|
|
515
|
+
def _ensemble__items(
|
|
516
|
+
self,
|
|
517
|
+
) -> list[tuple[str, GreedyEnsemble | TopKEnsemble | KWArgs]]:
|
|
518
|
+
return list(self.members.items())
|
|
519
|
+
|
|
520
|
+
@_ensemble__items.setter
|
|
521
|
+
def _ensemble__items(
|
|
522
|
+
self, items: list[tuple[str, GreedyEnsemble | TopKEnsemble | KWArgs]]
|
|
523
|
+
) -> None:
|
|
524
|
+
self.members = dict(items)
|
|
525
|
+
|
|
526
|
+
def get_params(self, deep: bool = True) -> KWArgs:
|
|
527
|
+
return self._get_params('_ensemble__items', deep=deep)
|
|
528
|
+
|
|
529
|
+
def set_params(self, **params) -> Self:
|
|
530
|
+
if 'members' in params:
|
|
531
|
+
self.members = params.pop('members')
|
|
532
|
+
return self._set_params('_ensemble__items', **params)
|
|
533
|
+
|
|
534
|
+
def make_config(
|
|
535
|
+
self,
|
|
536
|
+
*,
|
|
537
|
+
d_out: int,
|
|
538
|
+
eval_criterion: tabpackage.EvalCriterionConfig | None,
|
|
539
|
+
default_algorithm_dataset: DatasetName | None,
|
|
540
|
+
labels: dict[DatasetName, np.ndarray],
|
|
541
|
+
device: torch.device,
|
|
542
|
+
) -> dict[str, tabpackage.EnsembleConfig]:
|
|
543
|
+
self._validate_names(list(self.members))
|
|
544
|
+
|
|
545
|
+
configs = {}
|
|
546
|
+
for name, member in self.members.items():
|
|
547
|
+
if isinstance(member, dict):
|
|
548
|
+
kwargs = member.copy()
|
|
549
|
+
ensemble_type = {'greedy': GreedyEnsemble, 'topk': TopKEnsemble}[
|
|
550
|
+
kwargs.pop('type', 'greedy')
|
|
551
|
+
]
|
|
552
|
+
member = ensemble_type(**kwargs)
|
|
553
|
+
|
|
554
|
+
configs[name] = member._make_config(
|
|
555
|
+
d_out=d_out,
|
|
556
|
+
eval_criterion=eval_criterion,
|
|
557
|
+
default_algorithm_dataset=default_algorithm_dataset,
|
|
558
|
+
labels=labels,
|
|
559
|
+
device=device,
|
|
560
|
+
)
|
|
561
|
+
return configs
|