tabpackage 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tabpackage/__init__.py +18 -0
- tabpackage/_training.py +3380 -0
- tabpackage/ensemble.py +263 -0
- tabpackage/metrics.py +253 -0
- tabpackage/nn.py +2110 -0
- tabpackage/optim.py +943 -0
- tabpackage/sklearn/__init__.py +18 -0
- tabpackage/sklearn/_config.py +561 -0
- tabpackage/sklearn/_data.py +307 -0
- tabpackage/sklearn/_estimators.py +1210 -0
- tabpackage/utils/__init__.py +1 -0
- tabpackage/utils/torch.py +166 -0
- tabpackage-0.0.1.dist-info/METADATA +398 -0
- tabpackage-0.0.1.dist-info/RECORD +15 -0
- tabpackage-0.0.1.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,1210 @@
|
|
|
1
|
+
import copy
|
|
2
|
+
import dataclasses
|
|
3
|
+
import random
|
|
4
|
+
import warnings
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from functools import partial
|
|
7
|
+
from typing import Any, Literal, Self
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
import torch
|
|
11
|
+
from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin
|
|
12
|
+
from sklearn.preprocessing import OrdinalEncoder
|
|
13
|
+
from sklearn.utils import check_random_state
|
|
14
|
+
from sklearn.utils.validation import check_is_fitted
|
|
15
|
+
from torch import Tensor
|
|
16
|
+
|
|
17
|
+
import tabpackage
|
|
18
|
+
from tabpackage import _training, nn, optim
|
|
19
|
+
from tabpackage.sklearn import _config, _data
|
|
20
|
+
from tabpackage.sklearn._config import (
|
|
21
|
+
DatasetName,
|
|
22
|
+
Ensembles,
|
|
23
|
+
GreedyEnsemble,
|
|
24
|
+
HyperparameterBounds,
|
|
25
|
+
IntUniformDistribution,
|
|
26
|
+
KWArgs,
|
|
27
|
+
LogUniformDistribution,
|
|
28
|
+
MetricName,
|
|
29
|
+
OptionalDistribution,
|
|
30
|
+
OptionalHyperparameterBounds,
|
|
31
|
+
PerModelHyperparameters,
|
|
32
|
+
TopKEnsemble,
|
|
33
|
+
UniformDistribution,
|
|
34
|
+
)
|
|
35
|
+
from tabpackage.sklearn._data import XLike, YLike, YPredLike
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _cross_entropy(input: Tensor, target: Tensor) -> Tensor:
|
|
39
|
+
return torch.nn.functional.cross_entropy(
|
|
40
|
+
input.transpose(1, 2), target, reduction='none'
|
|
41
|
+
).mean(1)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _mse_loss(input: Tensor, target: Tensor) -> Tensor:
|
|
45
|
+
return (input.squeeze(-1) - target).square().mean(1)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclasses.dataclass(frozen=True, kw_only=True)
|
|
49
|
+
class _PredictorKey:
|
|
50
|
+
ensemble_name: str
|
|
51
|
+
autocast_dtype: torch.dtype | None
|
|
52
|
+
backbone_forward_impl: str
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclasses.dataclass(frozen=True, kw_only=True)
|
|
56
|
+
class _RefitEnsemble:
|
|
57
|
+
name: str
|
|
58
|
+
member_weights: Tensor | None
|
|
59
|
+
prediction_transform: tabpackage.PredictionTransform | None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _make_predictor(
|
|
63
|
+
tabpack: tabpackage.TrainedTabPack,
|
|
64
|
+
predictor_key: _PredictorKey,
|
|
65
|
+
*,
|
|
66
|
+
refit_ensemble: _RefitEnsemble | None = None,
|
|
67
|
+
device: torch.device,
|
|
68
|
+
) -> tabpackage.TabPackPredictor:
|
|
69
|
+
if refit_ensemble is None:
|
|
70
|
+
predictor = tabpack.make_predictor(
|
|
71
|
+
ensemble_names=[predictor_key.ensemble_name],
|
|
72
|
+
autocast_dtype=predictor_key.autocast_dtype,
|
|
73
|
+
)
|
|
74
|
+
else:
|
|
75
|
+
assert predictor_key.ensemble_name == refit_ensemble.name
|
|
76
|
+
|
|
77
|
+
# Completion order can differ from the recipe and weight order.
|
|
78
|
+
model_ids = tabpack.load_finished_model_ids().sort().values
|
|
79
|
+
predictor = tabpackage.TabPackPredictor(
|
|
80
|
+
model_pack=tabpack._make_model_pack(
|
|
81
|
+
model_ids=model_ids,
|
|
82
|
+
model_steps=tabpack.load_model_steps(model_ids),
|
|
83
|
+
),
|
|
84
|
+
autocast_dtype=predictor_key.autocast_dtype,
|
|
85
|
+
prediction_transform=tabpack.load_model_prediction_transform(),
|
|
86
|
+
ensembles={
|
|
87
|
+
predictor_key.ensemble_name: tabpackage.TabPackPredictor.EnsembleHead(
|
|
88
|
+
member_pack_idx=torch.arange(len(model_ids)),
|
|
89
|
+
member_weights=refit_ensemble.member_weights,
|
|
90
|
+
prediction_transform=refit_ensemble.prediction_transform,
|
|
91
|
+
)
|
|
92
|
+
},
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
predictor.to(device)
|
|
96
|
+
predictor.model_pack.backbone.set_forward_impl(predictor_key.backbone_forward_impl)
|
|
97
|
+
return predictor
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# >>> Estimators
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class _TabPackEstimator(BaseEstimator):
|
|
104
|
+
_classification: bool
|
|
105
|
+
|
|
106
|
+
def __init__(
|
|
107
|
+
self,
|
|
108
|
+
*,
|
|
109
|
+
# Models
|
|
110
|
+
n_models: int | None = None,
|
|
111
|
+
n_blocks: (
|
|
112
|
+
int | IntUniformDistribution | HyperparameterBounds[int] | None
|
|
113
|
+
) = None,
|
|
114
|
+
d_block: int | IntUniformDistribution | HyperparameterBounds[int] | None = None,
|
|
115
|
+
dropout: (
|
|
116
|
+
float
|
|
117
|
+
| OptionalDistribution[UniformDistribution]
|
|
118
|
+
| OptionalHyperparameterBounds
|
|
119
|
+
| None
|
|
120
|
+
) = None,
|
|
121
|
+
numerical_embeddings: Literal['none', 'linear-relu', 'cosine'] = 'cosine',
|
|
122
|
+
numerical_embeddings_d: (
|
|
123
|
+
int | IntUniformDistribution | HyperparameterBounds[int] | None
|
|
124
|
+
) = None,
|
|
125
|
+
numerical_embeddings_init_scale: (
|
|
126
|
+
float | LogUniformDistribution | HyperparameterBounds[float] | None
|
|
127
|
+
) = None,
|
|
128
|
+
# Training
|
|
129
|
+
optimizer: Literal['adamw', 'muon-adamw'] = 'muon-adamw',
|
|
130
|
+
adamw_lr: (
|
|
131
|
+
float | LogUniformDistribution | HyperparameterBounds[float] | None
|
|
132
|
+
) = None,
|
|
133
|
+
adamw_beta1: (
|
|
134
|
+
float | UniformDistribution | HyperparameterBounds[float] | None
|
|
135
|
+
) = None,
|
|
136
|
+
adamw_beta2: (
|
|
137
|
+
float | UniformDistribution | HyperparameterBounds[float] | None
|
|
138
|
+
) = None,
|
|
139
|
+
adamw_eps: (
|
|
140
|
+
float | LogUniformDistribution | HyperparameterBounds[float] | None
|
|
141
|
+
) = None,
|
|
142
|
+
muon_lr: (
|
|
143
|
+
float | LogUniformDistribution | HyperparameterBounds[float] | None
|
|
144
|
+
) = None,
|
|
145
|
+
muon_momentum: (
|
|
146
|
+
float | UniformDistribution | HyperparameterBounds[float] | None
|
|
147
|
+
) = None,
|
|
148
|
+
muon_newton_schulz_steps: int | None = None,
|
|
149
|
+
muon_nesterov: bool | None = None,
|
|
150
|
+
muon_skip_zero_updates: bool | None = None,
|
|
151
|
+
weight_decay: (
|
|
152
|
+
float | LogUniformDistribution | HyperparameterBounds[float] | None
|
|
153
|
+
) = None,
|
|
154
|
+
batch_size: int | None = None,
|
|
155
|
+
max_epochs: (
|
|
156
|
+
int | IntUniformDistribution | HyperparameterBounds[int] | None
|
|
157
|
+
) = None,
|
|
158
|
+
early_stopping_patience: int | None = 16,
|
|
159
|
+
# Per-model hyperparameters
|
|
160
|
+
per_model_hyperparameters: list[PerModelHyperparameters] | None = None,
|
|
161
|
+
# Evaluation
|
|
162
|
+
eval_datasets: list[DatasetName] | None = None,
|
|
163
|
+
eval_metrics: list[MetricName] | None = None,
|
|
164
|
+
eval_criterion_dataset: DatasetName | None = None,
|
|
165
|
+
eval_criterion_metric: MetricName | Literal['none'] | None = None,
|
|
166
|
+
eval_batch_size: int | None = None,
|
|
167
|
+
# Ensembles
|
|
168
|
+
ensembles: (
|
|
169
|
+
Ensembles | dict[str, GreedyEnsemble | TopKEnsemble | KWArgs] | None
|
|
170
|
+
) = None,
|
|
171
|
+
ensemble_eval_datasets: list[DatasetName] | None = None,
|
|
172
|
+
ensemble_eval_metrics: list[MetricName] | None = None,
|
|
173
|
+
ensemble_eval_criterion_dataset: DatasetName | None = None,
|
|
174
|
+
ensemble_eval_criterion_metric: MetricName | Literal['none'] | None = None,
|
|
175
|
+
stop_when_all_ensembles_are_finalized: bool = True,
|
|
176
|
+
# Reproducibility
|
|
177
|
+
random_state: int | np.random.RandomState | None = None,
|
|
178
|
+
# Efficiency
|
|
179
|
+
max_pack_size: int | None = None,
|
|
180
|
+
device: str | torch.device | Literal['auto'] = 'auto',
|
|
181
|
+
aux_device: str | torch.device | Literal['auto'] = 'auto',
|
|
182
|
+
move_datasets_to_device: bool | list[DatasetName] = False,
|
|
183
|
+
autocast_dtype: Literal['bfloat16'] | None = None,
|
|
184
|
+
compile_model: bool = False,
|
|
185
|
+
compile_model_numerical_embeddings: bool = False,
|
|
186
|
+
compile_model_first_block: bool = False,
|
|
187
|
+
compile_optimizer_muon_newton_schulz: bool = False,
|
|
188
|
+
backbone_forward_impl: Literal['indexed', 'dense'] = 'dense',
|
|
189
|
+
# Printing
|
|
190
|
+
print_progress: bool = True,
|
|
191
|
+
show_progress_bar: bool | None = None,
|
|
192
|
+
# Prediction
|
|
193
|
+
prediction_ensemble: str | None = None,
|
|
194
|
+
prediction_batch_size: int | None = None,
|
|
195
|
+
prediction_autocast_dtype: Literal['bfloat16'] | None = None,
|
|
196
|
+
prediction_backbone_forward_impl: Literal['indexed', 'dense'] = 'dense',
|
|
197
|
+
) -> None:
|
|
198
|
+
"""TabPack.
|
|
199
|
+
|
|
200
|
+
Note:
|
|
201
|
+
In the docs and code, the MLPs are usually referred to as "models".
|
|
202
|
+
|
|
203
|
+
TabPack trains many MLPs with different hyperparameters in parallel and
|
|
204
|
+
builds ensembles as they train. Hyperparameters can be fixed or sampled
|
|
205
|
+
for each MLP. Use distributions or sampling bounds to define a search
|
|
206
|
+
space, or `per_model_hyperparameters` to specify the models directly.
|
|
207
|
+
|
|
208
|
+
The fitted TabPack provides access to training results through `tabpack_`.
|
|
209
|
+
After fitting, consider using `TabPack.refit` to refit a selected ensemble on
|
|
210
|
+
the full training data for potentially better performance.
|
|
211
|
+
|
|
212
|
+
Args:
|
|
213
|
+
n_models: the total number of models to train.
|
|
214
|
+
n_blocks: the number of backbone blocks per model.
|
|
215
|
+
d_block: the width of each backbone block.
|
|
216
|
+
dropout: the dropout rate.
|
|
217
|
+
numerical_embeddings: the embedding method for numerical features.
|
|
218
|
+
numerical_embeddings_d: the embedding size per numerical feature.
|
|
219
|
+
numerical_embeddings_init_scale: the initialization scale for cosine
|
|
220
|
+
embeddings.
|
|
221
|
+
optimizer: the optimization method.
|
|
222
|
+
adamw_lr: the AdamW learning rate.
|
|
223
|
+
adamw_beta1: the decay rate for AdamW's first moment estimate.
|
|
224
|
+
adamw_beta2: the decay rate for AdamW's second moment estimate.
|
|
225
|
+
adamw_eps: the term used for numerical stability in AdamW.
|
|
226
|
+
muon_lr: the Muon learning rate.
|
|
227
|
+
muon_momentum: the Muon momentum.
|
|
228
|
+
muon_newton_schulz_steps: the number of Newton-Schulz iterations in Muon.
|
|
229
|
+
muon_nesterov: whether Muon uses Nesterov momentum.
|
|
230
|
+
muon_skip_zero_updates: whether Muon skips zero updates.
|
|
231
|
+
weight_decay: the weight decay strength.
|
|
232
|
+
batch_size: the number of training examples per model per step.
|
|
233
|
+
max_epochs: the maximum number of training epochs per model.
|
|
234
|
+
early_stopping_patience: the number of epochs without improvement
|
|
235
|
+
tolerated before stopping each model.
|
|
236
|
+
per_model_hyperparameters: hyperparameters for each model, overriding
|
|
237
|
+
the shared values.
|
|
238
|
+
eval_datasets: the datasets used to evaluate individual models.
|
|
239
|
+
eval_metrics: the metrics used to evaluate individual models.
|
|
240
|
+
eval_criterion_dataset: the dataset used to select each model's best
|
|
241
|
+
checkpoint and guide early stopping.
|
|
242
|
+
eval_criterion_metric: the metric used to select each model's best
|
|
243
|
+
checkpoint and guide early stopping.
|
|
244
|
+
eval_batch_size: the initial number of examples per evaluation batch.
|
|
245
|
+
ensembles: the named ensembles to build during training.
|
|
246
|
+
ensemble_eval_datasets: the datasets used to evaluate ensembles.
|
|
247
|
+
ensemble_eval_metrics: the metrics used to evaluate ensembles.
|
|
248
|
+
ensemble_eval_criterion_dataset: the dataset used to select the best
|
|
249
|
+
version of each ensemble and guide its early stopping.
|
|
250
|
+
ensemble_eval_criterion_metric: the metric used to select the best
|
|
251
|
+
version of each ensemble and guide its early stopping.
|
|
252
|
+
stop_when_all_ensembles_are_finalized: whether to end training once all
|
|
253
|
+
ensembles are finalized, even if some models are still training.
|
|
254
|
+
random_state: controls randomness in hyperparameter sampling and training.
|
|
255
|
+
max_pack_size: the maximum number of models trained in parallel.
|
|
256
|
+
device: the device used for training and prediction.
|
|
257
|
+
aux_device: the device used to compute evaluation metrics and store
|
|
258
|
+
predictions and checkpoints.
|
|
259
|
+
move_datasets_to_device: whether to keep all datasets on the training
|
|
260
|
+
device, or the names of specific datasets to keep there.
|
|
261
|
+
autocast_dtype: the precision used for mixed-precision training.
|
|
262
|
+
compile_model: whether to compile the whole model.
|
|
263
|
+
compile_model_numerical_embeddings: whether to compile the numerical
|
|
264
|
+
embeddings.
|
|
265
|
+
compile_model_first_block: whether to compile the first backbone block.
|
|
266
|
+
compile_optimizer_muon_newton_schulz: whether to compile Muon's
|
|
267
|
+
Newton-Schulz computation.
|
|
268
|
+
backbone_forward_impl: the method used to evaluate the backbone.
|
|
269
|
+
print_progress: whether to print a training summary after each epoch.
|
|
270
|
+
show_progress_bar: whether to show training progress within each epoch.
|
|
271
|
+
prediction_ensemble: the ensemble used for prediction. Required when
|
|
272
|
+
training multiple ensembles.
|
|
273
|
+
prediction_batch_size: the number of examples per prediction batch.
|
|
274
|
+
prediction_autocast_dtype: the precision used for mixed-precision
|
|
275
|
+
prediction.
|
|
276
|
+
prediction_backbone_forward_impl: the method used to evaluate the
|
|
277
|
+
backbone during prediction.
|
|
278
|
+
"""
|
|
279
|
+
self.n_models = n_models
|
|
280
|
+
self.n_blocks = n_blocks
|
|
281
|
+
self.d_block = d_block
|
|
282
|
+
self.dropout = dropout
|
|
283
|
+
self.numerical_embeddings = numerical_embeddings
|
|
284
|
+
self.numerical_embeddings_d = numerical_embeddings_d
|
|
285
|
+
self.numerical_embeddings_init_scale = numerical_embeddings_init_scale
|
|
286
|
+
|
|
287
|
+
self.optimizer = optimizer
|
|
288
|
+
self.adamw_lr = adamw_lr
|
|
289
|
+
self.adamw_beta1 = adamw_beta1
|
|
290
|
+
self.adamw_beta2 = adamw_beta2
|
|
291
|
+
self.adamw_eps = adamw_eps
|
|
292
|
+
self.muon_lr = muon_lr
|
|
293
|
+
self.muon_momentum = muon_momentum
|
|
294
|
+
self.muon_newton_schulz_steps = muon_newton_schulz_steps
|
|
295
|
+
self.muon_nesterov = muon_nesterov
|
|
296
|
+
self.muon_skip_zero_updates = muon_skip_zero_updates
|
|
297
|
+
self.weight_decay = weight_decay
|
|
298
|
+
self.batch_size = batch_size
|
|
299
|
+
self.max_epochs = max_epochs
|
|
300
|
+
self.early_stopping_patience = early_stopping_patience
|
|
301
|
+
|
|
302
|
+
self.per_model_hyperparameters = per_model_hyperparameters
|
|
303
|
+
|
|
304
|
+
self.eval_datasets = eval_datasets
|
|
305
|
+
self.eval_metrics = eval_metrics
|
|
306
|
+
self.eval_criterion_dataset = eval_criterion_dataset
|
|
307
|
+
self.eval_criterion_metric = eval_criterion_metric
|
|
308
|
+
self.eval_batch_size = eval_batch_size
|
|
309
|
+
|
|
310
|
+
self.ensembles = ensembles
|
|
311
|
+
self.ensemble_eval_datasets = ensemble_eval_datasets
|
|
312
|
+
self.ensemble_eval_metrics = ensemble_eval_metrics
|
|
313
|
+
self.ensemble_eval_criterion_dataset = ensemble_eval_criterion_dataset
|
|
314
|
+
self.ensemble_eval_criterion_metric = ensemble_eval_criterion_metric
|
|
315
|
+
self.stop_when_all_ensembles_are_finalized = (
|
|
316
|
+
stop_when_all_ensembles_are_finalized
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
self.random_state = random_state
|
|
320
|
+
|
|
321
|
+
self.max_pack_size = max_pack_size
|
|
322
|
+
self.device = device
|
|
323
|
+
self.aux_device = aux_device
|
|
324
|
+
self.move_datasets_to_device = move_datasets_to_device
|
|
325
|
+
self.autocast_dtype = autocast_dtype
|
|
326
|
+
self.compile_model = compile_model
|
|
327
|
+
self.compile_model_numerical_embeddings = compile_model_numerical_embeddings
|
|
328
|
+
self.compile_model_first_block = compile_model_first_block
|
|
329
|
+
self.compile_optimizer_muon_newton_schulz = compile_optimizer_muon_newton_schulz
|
|
330
|
+
self.backbone_forward_impl = backbone_forward_impl
|
|
331
|
+
|
|
332
|
+
self.print_progress = print_progress
|
|
333
|
+
self.show_progress_bar = show_progress_bar
|
|
334
|
+
|
|
335
|
+
self.prediction_ensemble = prediction_ensemble
|
|
336
|
+
self.prediction_batch_size = prediction_batch_size
|
|
337
|
+
self.prediction_autocast_dtype = prediction_autocast_dtype
|
|
338
|
+
self.prediction_backbone_forward_impl = prediction_backbone_forward_impl
|
|
339
|
+
|
|
340
|
+
def __sklearn_is_fitted__(self) -> bool:
|
|
341
|
+
return getattr(self, '_is_fitted', False)
|
|
342
|
+
|
|
343
|
+
def _make_hyperparameter_configs(
|
|
344
|
+
self,
|
|
345
|
+
*,
|
|
346
|
+
n_numerical_features: int,
|
|
347
|
+
d_out: int,
|
|
348
|
+
rng: random.Random,
|
|
349
|
+
) -> tuple[KWArgs, KWArgs, list[KWArgs]]:
|
|
350
|
+
unused_hyperparameters = set()
|
|
351
|
+
if self.optimizer == 'adamw':
|
|
352
|
+
unused_hyperparameters.update(
|
|
353
|
+
(
|
|
354
|
+
'muon_lr',
|
|
355
|
+
'muon_momentum',
|
|
356
|
+
'muon_newton_schulz_steps',
|
|
357
|
+
'muon_nesterov',
|
|
358
|
+
'muon_skip_zero_updates',
|
|
359
|
+
)
|
|
360
|
+
)
|
|
361
|
+
elif (
|
|
362
|
+
self.muon_newton_schulz_steps is not None
|
|
363
|
+
and self.muon_newton_schulz_steps < 0
|
|
364
|
+
):
|
|
365
|
+
raise ValueError('`muon_newton_schulz_steps` must be non-negative')
|
|
366
|
+
|
|
367
|
+
use_numerical_embeddings = (
|
|
368
|
+
n_numerical_features > 0 and self.numerical_embeddings != 'none'
|
|
369
|
+
)
|
|
370
|
+
if not use_numerical_embeddings:
|
|
371
|
+
unused_hyperparameters.add('numerical_embeddings_d')
|
|
372
|
+
if not use_numerical_embeddings or self.numerical_embeddings != 'cosine':
|
|
373
|
+
unused_hyperparameters.add('numerical_embeddings_init_scale')
|
|
374
|
+
|
|
375
|
+
for name in unused_hyperparameters:
|
|
376
|
+
if getattr(self, name) is not None:
|
|
377
|
+
raise ValueError(
|
|
378
|
+
f'`{name}` does not apply to the current model and optimizer'
|
|
379
|
+
' configuration; leave it as `None`'
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
model_config: KWArgs = {
|
|
383
|
+
'd_out': d_out,
|
|
384
|
+
'forward_impl': self.backbone_forward_impl,
|
|
385
|
+
}
|
|
386
|
+
if use_numerical_embeddings:
|
|
387
|
+
model_config['numerical_embeddings'] = {
|
|
388
|
+
'type': (
|
|
389
|
+
nn.CosineEmbeddingsPack
|
|
390
|
+
if self.numerical_embeddings == 'cosine'
|
|
391
|
+
else nn.LinearReLUEmbeddingsPack
|
|
392
|
+
),
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
optimizer_config: KWArgs = {
|
|
396
|
+
'type': (
|
|
397
|
+
optim.AdamWPack if self.optimizer == 'adamw' else optim.MuonAdamWPack
|
|
398
|
+
)
|
|
399
|
+
}
|
|
400
|
+
for name in (
|
|
401
|
+
'muon_newton_schulz_steps',
|
|
402
|
+
'muon_nesterov',
|
|
403
|
+
'muon_skip_zero_updates',
|
|
404
|
+
):
|
|
405
|
+
value = getattr(self, name)
|
|
406
|
+
if value is not None:
|
|
407
|
+
optimizer_config[name] = value
|
|
408
|
+
|
|
409
|
+
# >>> Explicit per-model configuration
|
|
410
|
+
|
|
411
|
+
if self.per_model_hyperparameters is not None:
|
|
412
|
+
hyperparameter_names = PerModelHyperparameters.__annotations__.keys()
|
|
413
|
+
shared_hyperparameters = {
|
|
414
|
+
name: getattr(self, name)
|
|
415
|
+
for name in hyperparameter_names
|
|
416
|
+
if getattr(self, name) is not None
|
|
417
|
+
}
|
|
418
|
+
for name, value in shared_hyperparameters.items():
|
|
419
|
+
if isinstance(value, (_config.Distribution, dict)):
|
|
420
|
+
raise TypeError(
|
|
421
|
+
f'`{name}` must be a scalar when `per_model_hyperparameters` is'
|
|
422
|
+
' provided'
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
per_model_hyperparameter_names = {
|
|
426
|
+
name for entry in self.per_model_hyperparameters for name in entry
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
required_hyperparameters = {
|
|
430
|
+
'n_blocks',
|
|
431
|
+
'dropout',
|
|
432
|
+
'adamw_lr',
|
|
433
|
+
'weight_decay',
|
|
434
|
+
}
|
|
435
|
+
if self.optimizer == 'muon-adamw':
|
|
436
|
+
required_hyperparameters.add('muon_lr')
|
|
437
|
+
if use_numerical_embeddings:
|
|
438
|
+
required_hyperparameters.add('numerical_embeddings_d')
|
|
439
|
+
if self.numerical_embeddings == 'cosine':
|
|
440
|
+
required_hyperparameters.add('numerical_embeddings_init_scale')
|
|
441
|
+
missing_hyperparameters = (
|
|
442
|
+
required_hyperparameters
|
|
443
|
+
- shared_hyperparameters.keys()
|
|
444
|
+
- per_model_hyperparameter_names
|
|
445
|
+
)
|
|
446
|
+
if missing_hyperparameters:
|
|
447
|
+
raise ValueError(
|
|
448
|
+
'Missing explicit hyperparameters:'
|
|
449
|
+
f' {sorted(missing_hyperparameters)}'
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
if self.d_block is None and 'd_block' not in per_model_hyperparameter_names:
|
|
453
|
+
shared_hyperparameters['d_block'] = 384
|
|
454
|
+
|
|
455
|
+
def translate(values: Mapping[str, Any], config: KWArgs) -> None:
|
|
456
|
+
for name, value in values.items():
|
|
457
|
+
if name in unused_hyperparameters:
|
|
458
|
+
raise ValueError(
|
|
459
|
+
f'`{name}` does not apply to the current model and'
|
|
460
|
+
' optimizer configuration; omit it from'
|
|
461
|
+
' `per_model_hyperparameters`'
|
|
462
|
+
)
|
|
463
|
+
if value is not None:
|
|
464
|
+
_config.validate_hyperparameter(name, value)
|
|
465
|
+
|
|
466
|
+
if name in ('n_blocks', 'd_block', 'dropout'):
|
|
467
|
+
config['model'][name] = value
|
|
468
|
+
elif name in (
|
|
469
|
+
'numerical_embeddings_d',
|
|
470
|
+
'numerical_embeddings_init_scale',
|
|
471
|
+
):
|
|
472
|
+
embedding_key = (
|
|
473
|
+
'd_embedding'
|
|
474
|
+
if name == 'numerical_embeddings_d'
|
|
475
|
+
else 'init_scale'
|
|
476
|
+
)
|
|
477
|
+
config['model'].setdefault('numerical_embeddings', {})[
|
|
478
|
+
embedding_key
|
|
479
|
+
] = value
|
|
480
|
+
elif name == 'max_epochs':
|
|
481
|
+
config[name] = value
|
|
482
|
+
else:
|
|
483
|
+
optimizer_key = (
|
|
484
|
+
name.removeprefix('adamw_')
|
|
485
|
+
if self.optimizer == 'adamw'
|
|
486
|
+
else name
|
|
487
|
+
)
|
|
488
|
+
config['optimizer'][optimizer_key] = value
|
|
489
|
+
|
|
490
|
+
translate(
|
|
491
|
+
shared_hyperparameters,
|
|
492
|
+
{'model': model_config, 'optimizer': optimizer_config},
|
|
493
|
+
)
|
|
494
|
+
|
|
495
|
+
explicit_hyperparameters = []
|
|
496
|
+
for entry in self.per_model_hyperparameters:
|
|
497
|
+
config: KWArgs = {'model': {}, 'optimizer': {}}
|
|
498
|
+
translate(entry, config)
|
|
499
|
+
explicit_hyperparameters.append(config)
|
|
500
|
+
|
|
501
|
+
return model_config, optimizer_config, explicit_hyperparameters
|
|
502
|
+
|
|
503
|
+
# >>> Hyperparameter resolution
|
|
504
|
+
|
|
505
|
+
default_hyperparameters = {
|
|
506
|
+
'n_blocks': IntUniformDistribution(1, 3),
|
|
507
|
+
'd_block': 384,
|
|
508
|
+
'dropout': OptionalDistribution(UniformDistribution(0.0, 0.5)),
|
|
509
|
+
'adamw_lr': LogUniformDistribution(1e-4, 5e-3),
|
|
510
|
+
'weight_decay': LogUniformDistribution(1e-3, 1.0),
|
|
511
|
+
'adamw_beta1': None,
|
|
512
|
+
'adamw_beta2': None,
|
|
513
|
+
'adamw_eps': None,
|
|
514
|
+
'max_epochs': None,
|
|
515
|
+
}
|
|
516
|
+
if use_numerical_embeddings:
|
|
517
|
+
default_hyperparameters['numerical_embeddings_d'] = IntUniformDistribution(
|
|
518
|
+
8, 32
|
|
519
|
+
)
|
|
520
|
+
if self.numerical_embeddings == 'cosine':
|
|
521
|
+
default_hyperparameters['numerical_embeddings_init_scale'] = (
|
|
522
|
+
LogUniformDistribution(0.01, 10.0)
|
|
523
|
+
)
|
|
524
|
+
if self.optimizer == 'muon-adamw':
|
|
525
|
+
default_hyperparameters['muon_lr'] = LogUniformDistribution(1e-3, 1e-1)
|
|
526
|
+
default_hyperparameters['muon_momentum'] = None
|
|
527
|
+
|
|
528
|
+
resolved_hyperparameters = {}
|
|
529
|
+
for name, default in default_hyperparameters.items():
|
|
530
|
+
value = getattr(self, name)
|
|
531
|
+
value = default if value is None else value
|
|
532
|
+
if value is None:
|
|
533
|
+
continue
|
|
534
|
+
|
|
535
|
+
if isinstance(value, dict):
|
|
536
|
+
bounds = value.copy()
|
|
537
|
+
if name == 'dropout':
|
|
538
|
+
optional = bounds.pop('optional')
|
|
539
|
+
value = UniformDistribution(**bounds)
|
|
540
|
+
value = OptionalDistribution(value) if optional else value
|
|
541
|
+
elif name in (
|
|
542
|
+
'n_blocks',
|
|
543
|
+
'd_block',
|
|
544
|
+
'numerical_embeddings_d',
|
|
545
|
+
'max_epochs',
|
|
546
|
+
):
|
|
547
|
+
value = IntUniformDistribution(**bounds)
|
|
548
|
+
elif name in ('adamw_beta1', 'adamw_beta2', 'muon_momentum'):
|
|
549
|
+
value = UniformDistribution(**bounds)
|
|
550
|
+
else:
|
|
551
|
+
value = LogUniformDistribution(**bounds)
|
|
552
|
+
|
|
553
|
+
_config.validate_hyperparameter(name, value)
|
|
554
|
+
resolved_hyperparameters[name] = value
|
|
555
|
+
|
|
556
|
+
def sample(name: str) -> int | float:
|
|
557
|
+
value = resolved_hyperparameters[name]
|
|
558
|
+
return (
|
|
559
|
+
value.sample(rng) if isinstance(value, _config.Distribution) else value
|
|
560
|
+
)
|
|
561
|
+
|
|
562
|
+
# >>> Optimizer
|
|
563
|
+
|
|
564
|
+
per_model_optimizer_hyperparameters = {}
|
|
565
|
+
for name in (
|
|
566
|
+
'adamw_lr',
|
|
567
|
+
'weight_decay',
|
|
568
|
+
'muon_lr',
|
|
569
|
+
'adamw_beta1',
|
|
570
|
+
'adamw_beta2',
|
|
571
|
+
'adamw_eps',
|
|
572
|
+
'muon_momentum',
|
|
573
|
+
):
|
|
574
|
+
if name not in resolved_hyperparameters:
|
|
575
|
+
continue
|
|
576
|
+
|
|
577
|
+
optimizer_key = (
|
|
578
|
+
name.removeprefix('adamw_') if self.optimizer == 'adamw' else name
|
|
579
|
+
)
|
|
580
|
+
if name in ('adamw_lr', 'weight_decay', 'muon_lr') or isinstance(
|
|
581
|
+
resolved_hyperparameters[name], _config.Distribution
|
|
582
|
+
):
|
|
583
|
+
per_model_optimizer_hyperparameters[optimizer_key] = name
|
|
584
|
+
else:
|
|
585
|
+
optimizer_config[optimizer_key] = resolved_hyperparameters[name]
|
|
586
|
+
|
|
587
|
+
# >>> Per-model sampling
|
|
588
|
+
|
|
589
|
+
per_model_hyperparameters = []
|
|
590
|
+
for _ in range(64 if self.n_models is None else self.n_models):
|
|
591
|
+
model_hyperparameters = {}
|
|
592
|
+
if use_numerical_embeddings:
|
|
593
|
+
model_hyperparameters['numerical_embeddings'] = {
|
|
594
|
+
'd_embedding': sample('numerical_embeddings_d'),
|
|
595
|
+
}
|
|
596
|
+
if self.numerical_embeddings == 'cosine':
|
|
597
|
+
model_hyperparameters['numerical_embeddings']['init_scale'] = (
|
|
598
|
+
sample('numerical_embeddings_init_scale')
|
|
599
|
+
)
|
|
600
|
+
|
|
601
|
+
model_hyperparameters['n_blocks'] = sample('n_blocks')
|
|
602
|
+
if isinstance(resolved_hyperparameters['d_block'], _config.Distribution):
|
|
603
|
+
model_hyperparameters['d_block'] = sample('d_block')
|
|
604
|
+
model_hyperparameters['dropout'] = sample('dropout')
|
|
605
|
+
|
|
606
|
+
hyperparameters = {
|
|
607
|
+
'model': model_hyperparameters,
|
|
608
|
+
'optimizer': {
|
|
609
|
+
key: sample(name)
|
|
610
|
+
for key, name in per_model_optimizer_hyperparameters.items()
|
|
611
|
+
},
|
|
612
|
+
}
|
|
613
|
+
if isinstance(
|
|
614
|
+
resolved_hyperparameters.get('max_epochs'), _config.Distribution
|
|
615
|
+
):
|
|
616
|
+
hyperparameters['max_epochs'] = sample('max_epochs')
|
|
617
|
+
per_model_hyperparameters.append(hyperparameters)
|
|
618
|
+
|
|
619
|
+
if not isinstance(resolved_hyperparameters['d_block'], _config.Distribution):
|
|
620
|
+
model_config['d_block'] = resolved_hyperparameters['d_block']
|
|
621
|
+
|
|
622
|
+
return model_config, optimizer_config, per_model_hyperparameters
|
|
623
|
+
|
|
624
|
+
def _make_train_kwargs(
|
|
625
|
+
self,
|
|
626
|
+
*,
|
|
627
|
+
n_numerical_features: int,
|
|
628
|
+
d_out: int,
|
|
629
|
+
rng: random.Random,
|
|
630
|
+
default_dataset: DatasetName | None,
|
|
631
|
+
labels: dict[DatasetName, np.ndarray],
|
|
632
|
+
device: torch.device,
|
|
633
|
+
) -> KWArgs:
|
|
634
|
+
batch_size = self.batch_size
|
|
635
|
+
if batch_size is None:
|
|
636
|
+
n_train = len(labels['train'])
|
|
637
|
+
batch_size = (
|
|
638
|
+
32
|
|
639
|
+
if n_train < 2048
|
|
640
|
+
else 64
|
|
641
|
+
if n_train < 4096
|
|
642
|
+
else 128
|
|
643
|
+
if n_train < 8192
|
|
644
|
+
else 256
|
|
645
|
+
if n_train < 32768
|
|
646
|
+
else 512
|
|
647
|
+
if n_train < 131072
|
|
648
|
+
else 1024
|
|
649
|
+
)
|
|
650
|
+
|
|
651
|
+
eval_batch_size = self.eval_batch_size
|
|
652
|
+
eval_batch_size = (
|
|
653
|
+
(4096 if device.type == 'cuda' else 512)
|
|
654
|
+
if eval_batch_size is None
|
|
655
|
+
else eval_batch_size
|
|
656
|
+
)
|
|
657
|
+
|
|
658
|
+
aux_device = (
|
|
659
|
+
(device if device.type == 'mps' else torch.device('cpu'))
|
|
660
|
+
if self.aux_device == 'auto'
|
|
661
|
+
else _config.resolve_device(self.aux_device)
|
|
662
|
+
)
|
|
663
|
+
|
|
664
|
+
eval_criterion = _config.resolve_criterion(
|
|
665
|
+
self.eval_criterion_dataset,
|
|
666
|
+
self.eval_criterion_metric,
|
|
667
|
+
default_dataset=default_dataset,
|
|
668
|
+
default_metric='accuracy' if self._classification else 'mse',
|
|
669
|
+
d_out=d_out,
|
|
670
|
+
)
|
|
671
|
+
eval_metrics = _config.resolve_eval_metrics(
|
|
672
|
+
self.eval_datasets,
|
|
673
|
+
self.eval_metrics,
|
|
674
|
+
eval_criterion=eval_criterion,
|
|
675
|
+
d_out=d_out,
|
|
676
|
+
labels=labels,
|
|
677
|
+
)
|
|
678
|
+
|
|
679
|
+
ensembles = self.ensembles
|
|
680
|
+
ensembles = (
|
|
681
|
+
Ensembles({'default': GreedyEnsemble()})
|
|
682
|
+
if ensembles is None
|
|
683
|
+
else (Ensembles(ensembles) if isinstance(ensembles, dict) else ensembles)
|
|
684
|
+
)
|
|
685
|
+
|
|
686
|
+
ensemble_eval_criterion = None
|
|
687
|
+
ensemble_eval_metrics = {}
|
|
688
|
+
if ensembles.members:
|
|
689
|
+
ensemble_eval_criterion = _config.resolve_criterion(
|
|
690
|
+
self.ensemble_eval_criterion_dataset,
|
|
691
|
+
self.ensemble_eval_criterion_metric,
|
|
692
|
+
default_dataset=(
|
|
693
|
+
default_dataset
|
|
694
|
+
if eval_criterion is None
|
|
695
|
+
else eval_criterion.dataset
|
|
696
|
+
),
|
|
697
|
+
default_metric=(
|
|
698
|
+
None if eval_criterion is None else eval_criterion.metric
|
|
699
|
+
),
|
|
700
|
+
d_out=d_out,
|
|
701
|
+
)
|
|
702
|
+
ensemble_eval_metrics = _config.resolve_eval_metrics(
|
|
703
|
+
self.ensemble_eval_datasets,
|
|
704
|
+
self.ensemble_eval_metrics,
|
|
705
|
+
eval_criterion=ensemble_eval_criterion,
|
|
706
|
+
d_out=d_out,
|
|
707
|
+
labels=labels,
|
|
708
|
+
)
|
|
709
|
+
|
|
710
|
+
ensemble_configs = ensembles.make_config(
|
|
711
|
+
d_out=d_out,
|
|
712
|
+
eval_criterion=ensemble_eval_criterion,
|
|
713
|
+
default_algorithm_dataset=default_dataset,
|
|
714
|
+
labels=labels,
|
|
715
|
+
device=device,
|
|
716
|
+
)
|
|
717
|
+
|
|
718
|
+
model_config, optimizer_config, per_model_hyperparameters = (
|
|
719
|
+
self._make_hyperparameter_configs(
|
|
720
|
+
n_numerical_features=n_numerical_features,
|
|
721
|
+
d_out=d_out,
|
|
722
|
+
rng=rng,
|
|
723
|
+
)
|
|
724
|
+
)
|
|
725
|
+
|
|
726
|
+
max_epochs = self.max_epochs
|
|
727
|
+
max_epochs = (
|
|
728
|
+
None
|
|
729
|
+
if isinstance(max_epochs, (IntUniformDistribution, dict))
|
|
730
|
+
else max_epochs
|
|
731
|
+
)
|
|
732
|
+
|
|
733
|
+
train_kwargs = {
|
|
734
|
+
'model': model_config,
|
|
735
|
+
'prediction_transform': (
|
|
736
|
+
partial(torch.softmax, dim=-1)
|
|
737
|
+
if self._classification
|
|
738
|
+
else partial(torch.squeeze, dim=-1)
|
|
739
|
+
),
|
|
740
|
+
'loss_fn': _cross_entropy if self._classification else _mse_loss,
|
|
741
|
+
'optimizer': optimizer_config,
|
|
742
|
+
'batch_size': batch_size,
|
|
743
|
+
'max_epochs': max_epochs,
|
|
744
|
+
'per_model_hyperparameters': per_model_hyperparameters,
|
|
745
|
+
'eval_metrics': eval_metrics,
|
|
746
|
+
'eval_criterion': eval_criterion,
|
|
747
|
+
'eval_batch_size': eval_batch_size,
|
|
748
|
+
'ensembles': ensemble_configs,
|
|
749
|
+
'max_pack_size': self.max_pack_size,
|
|
750
|
+
'device': device,
|
|
751
|
+
'aux_device': aux_device,
|
|
752
|
+
'autocast_dtype': self.autocast_dtype,
|
|
753
|
+
'compile_model': self.compile_model,
|
|
754
|
+
'compile_model_numerical_embeddings': (
|
|
755
|
+
self.compile_model_numerical_embeddings
|
|
756
|
+
),
|
|
757
|
+
'compile_model_first_block': self.compile_model_first_block,
|
|
758
|
+
'compile_optimizer_muon_newton_schulz': (
|
|
759
|
+
self.compile_optimizer_muon_newton_schulz
|
|
760
|
+
),
|
|
761
|
+
'print_progress': self.print_progress,
|
|
762
|
+
'show_progress_bar': self.show_progress_bar,
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
if self.n_models is not None:
|
|
766
|
+
train_kwargs['n_models'] = self.n_models
|
|
767
|
+
if eval_criterion is not None:
|
|
768
|
+
train_kwargs['early_stopping_patience'] = self.early_stopping_patience
|
|
769
|
+
if ensemble_configs:
|
|
770
|
+
train_kwargs.update(
|
|
771
|
+
ensemble_eval_metrics={
|
|
772
|
+
dataset: {
|
|
773
|
+
name: partial(_config.compute_scalar_metric, metric_fn)
|
|
774
|
+
for name, metric_fn in dataset_metrics.items()
|
|
775
|
+
}
|
|
776
|
+
for dataset, dataset_metrics in ensemble_eval_metrics.items()
|
|
777
|
+
},
|
|
778
|
+
ensemble_eval_criterion=ensemble_eval_criterion,
|
|
779
|
+
stop_when_all_ensembles_are_finalized=self.stop_when_all_ensembles_are_finalized,
|
|
780
|
+
)
|
|
781
|
+
|
|
782
|
+
return train_kwargs
|
|
783
|
+
|
|
784
|
+
def _resolve_prediction_ensemble(self, ensemble_names: list[str]) -> str | None:
|
|
785
|
+
if self.prediction_ensemble is None:
|
|
786
|
+
if len(ensemble_names) > 1:
|
|
787
|
+
raise ValueError(
|
|
788
|
+
'Set prediction_ensemble when using multiple ensembles'
|
|
789
|
+
)
|
|
790
|
+
return ensemble_names[0] if ensemble_names else None
|
|
791
|
+
|
|
792
|
+
if self.prediction_ensemble not in ensemble_names:
|
|
793
|
+
raise ValueError('Unknown prediction ensemble')
|
|
794
|
+
return self.prediction_ensemble
|
|
795
|
+
|
|
796
|
+
def _get_predictor_key(self, ensemble_name: str) -> _PredictorKey:
|
|
797
|
+
return _PredictorKey(
|
|
798
|
+
ensemble_name=ensemble_name,
|
|
799
|
+
autocast_dtype=_config.resolve_autocast_dtype(
|
|
800
|
+
self.prediction_autocast_dtype
|
|
801
|
+
),
|
|
802
|
+
backbone_forward_impl=self.prediction_backbone_forward_impl,
|
|
803
|
+
)
|
|
804
|
+
|
|
805
|
+
def _set_fitted_data(
|
|
806
|
+
self,
|
|
807
|
+
feature_schema: _data.FeatureSchema,
|
|
808
|
+
label_schema: _data.LabelSchema,
|
|
809
|
+
feature_encoder: OrdinalEncoder | None,
|
|
810
|
+
) -> None:
|
|
811
|
+
self._feature_schema = feature_schema
|
|
812
|
+
self._label_schema = label_schema
|
|
813
|
+
self._feature_encoder = feature_encoder
|
|
814
|
+
|
|
815
|
+
self.n_features_in_ = feature_schema.n_features
|
|
816
|
+
if feature_schema.columns is None:
|
|
817
|
+
if hasattr(self, 'feature_names_in_'):
|
|
818
|
+
del self.feature_names_in_
|
|
819
|
+
else:
|
|
820
|
+
self.feature_names_in_ = np.array(feature_schema.columns, dtype=object)
|
|
821
|
+
|
|
822
|
+
if label_schema.classes is not None:
|
|
823
|
+
self.classes_ = label_schema.classes
|
|
824
|
+
|
|
825
|
+
def fit(
|
|
826
|
+
self,
|
|
827
|
+
X: XLike,
|
|
828
|
+
y: YLike,
|
|
829
|
+
*,
|
|
830
|
+
extra_set: tuple[XLike, YLike] | dict[DatasetName, tuple[XLike, YLike]],
|
|
831
|
+
) -> Self:
|
|
832
|
+
"""Train models and build ensembles using the extra data for validation.
|
|
833
|
+
|
|
834
|
+
Args:
|
|
835
|
+
X: the training features.
|
|
836
|
+
y: the training labels or regression targets.
|
|
837
|
+
extra_set: extra data as an `(X, y)` pair or a collection of named
|
|
838
|
+
pairs. Use these datasets to evaluate models and build ensembles.
|
|
839
|
+
When passing named datasets, choose the validation dataset through
|
|
840
|
+
`eval_criterion_dataset`.
|
|
841
|
+
|
|
842
|
+
Returns:
|
|
843
|
+
The fitted estimator.
|
|
844
|
+
"""
|
|
845
|
+
# >>> Data validation
|
|
846
|
+
|
|
847
|
+
feature_schema = _data.FeatureSchema.from_data(X)
|
|
848
|
+
label_schema = _data.LabelSchema.from_data(
|
|
849
|
+
y, classification=self._classification
|
|
850
|
+
)
|
|
851
|
+
if len(X) != len(y):
|
|
852
|
+
raise ValueError('Features and labels must have the same number of rows')
|
|
853
|
+
|
|
854
|
+
default_dataset = None if isinstance(extra_set, dict) else 'extra'
|
|
855
|
+
extra_datasets = (
|
|
856
|
+
extra_set if isinstance(extra_set, dict) else {'extra': extra_set}
|
|
857
|
+
)
|
|
858
|
+
if 'train' in extra_datasets:
|
|
859
|
+
raise ValueError('The dataset name "train" is reserved')
|
|
860
|
+
|
|
861
|
+
for X_extra, y_extra in extra_datasets.values():
|
|
862
|
+
if _data.FeatureSchema.from_data(X_extra) != feature_schema:
|
|
863
|
+
raise ValueError(
|
|
864
|
+
'Extra datasets must have the same feature schema as the training'
|
|
865
|
+
' data'
|
|
866
|
+
)
|
|
867
|
+
label_schema.validate(y_extra)
|
|
868
|
+
if len(X_extra) != len(y_extra):
|
|
869
|
+
raise ValueError(
|
|
870
|
+
'Features and labels must have the same number of rows'
|
|
871
|
+
)
|
|
872
|
+
|
|
873
|
+
# >>> Encoding
|
|
874
|
+
|
|
875
|
+
feature_encoder = feature_schema.make_encoder(X)
|
|
876
|
+
arrays = {
|
|
877
|
+
name: {
|
|
878
|
+
**feature_schema.encode(features, feature_encoder),
|
|
879
|
+
'y': label_schema.encode(target),
|
|
880
|
+
}
|
|
881
|
+
for name, (features, target) in {'train': (X, y), **extra_datasets}.items()
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
# >>> Training configuration
|
|
885
|
+
|
|
886
|
+
device = _config.resolve_device(self.device)
|
|
887
|
+
base_seed = int(
|
|
888
|
+
check_random_state(self.random_state).randint(0, 2**32 - 2, dtype=np.int64)
|
|
889
|
+
)
|
|
890
|
+
rng = random.Random(base_seed)
|
|
891
|
+
train_kwargs = self._make_train_kwargs(
|
|
892
|
+
n_numerical_features=feature_schema.n_numerical_features,
|
|
893
|
+
d_out=1 if label_schema.classes is None else len(label_schema.classes),
|
|
894
|
+
rng=random.Random(_training.get_next_seed(rng)),
|
|
895
|
+
default_dataset=default_dataset,
|
|
896
|
+
labels={name: data['y'] for name, data in arrays.items()},
|
|
897
|
+
device=device,
|
|
898
|
+
)
|
|
899
|
+
train_kwargs['model']['categorical_cardinalities'] = (
|
|
900
|
+
[]
|
|
901
|
+
if feature_encoder is None
|
|
902
|
+
else [len(categories) for categories in feature_encoder.categories_]
|
|
903
|
+
)
|
|
904
|
+
prediction_ensemble = self._resolve_prediction_ensemble(
|
|
905
|
+
list(train_kwargs['ensembles'])
|
|
906
|
+
)
|
|
907
|
+
predictor_key = (
|
|
908
|
+
None
|
|
909
|
+
if prediction_ensemble is None
|
|
910
|
+
else self._get_predictor_key(prediction_ensemble)
|
|
911
|
+
)
|
|
912
|
+
|
|
913
|
+
# >>> Dataset placement
|
|
914
|
+
|
|
915
|
+
move_datasets_to_device = self.move_datasets_to_device
|
|
916
|
+
datasets_on_device = (
|
|
917
|
+
set(arrays)
|
|
918
|
+
if move_datasets_to_device is True
|
|
919
|
+
else (
|
|
920
|
+
set()
|
|
921
|
+
if move_datasets_to_device is False
|
|
922
|
+
else set(move_datasets_to_device)
|
|
923
|
+
)
|
|
924
|
+
)
|
|
925
|
+
if not datasets_on_device <= arrays.keys():
|
|
926
|
+
raise ValueError('Unknown dataset in move_datasets_to_device')
|
|
927
|
+
|
|
928
|
+
datasets = {
|
|
929
|
+
name: _data.make_dataset(
|
|
930
|
+
data,
|
|
931
|
+
device=(
|
|
932
|
+
device if name in datasets_on_device else train_kwargs['aux_device']
|
|
933
|
+
),
|
|
934
|
+
)
|
|
935
|
+
for name, data in arrays.items()
|
|
936
|
+
}
|
|
937
|
+
del arrays
|
|
938
|
+
|
|
939
|
+
# >>> Training
|
|
940
|
+
|
|
941
|
+
fit_train_kwargs = copy.deepcopy(train_kwargs)
|
|
942
|
+
train_device = _config.resolve_device(datasets['train']['y'].device)
|
|
943
|
+
tabpack = tabpackage.train(
|
|
944
|
+
**train_kwargs,
|
|
945
|
+
datasets=datasets,
|
|
946
|
+
seed=_training.get_next_seed(rng),
|
|
947
|
+
)
|
|
948
|
+
predictor = (
|
|
949
|
+
None
|
|
950
|
+
if predictor_key is None
|
|
951
|
+
else _make_predictor(tabpack, predictor_key, device=device)
|
|
952
|
+
)
|
|
953
|
+
|
|
954
|
+
# >>> Fitted state
|
|
955
|
+
|
|
956
|
+
# Preserve prior fitted state if training or predictor construction fails.
|
|
957
|
+
self._set_fitted_data(feature_schema, label_schema, feature_encoder)
|
|
958
|
+
self.tabpack_ = tabpack
|
|
959
|
+
self._fitted_device = device
|
|
960
|
+
self._predictor = predictor
|
|
961
|
+
self._predictor_key = predictor_key
|
|
962
|
+
self._default_prediction_batch_size = train_kwargs['eval_batch_size']
|
|
963
|
+
self._fit_train_kwargs = fit_train_kwargs
|
|
964
|
+
self._refit_seed_rng = rng
|
|
965
|
+
self._fit_train_device = train_device
|
|
966
|
+
self._refit_ensemble: _RefitEnsemble | None = None
|
|
967
|
+
self._is_fitted = True
|
|
968
|
+
|
|
969
|
+
return self
|
|
970
|
+
|
|
971
|
+
def refit(
|
|
972
|
+
self,
|
|
973
|
+
X: XLike,
|
|
974
|
+
y: YLike,
|
|
975
|
+
*,
|
|
976
|
+
ensemble: str | None = None,
|
|
977
|
+
max_n_models: int | None = None,
|
|
978
|
+
n_repeats: int | None = None,
|
|
979
|
+
) -> Self:
|
|
980
|
+
"""Retrain an ensemble from scratch on full training data."""
|
|
981
|
+
check_is_fitted(self)
|
|
982
|
+
if self._refit_ensemble is not None:
|
|
983
|
+
raise RuntimeError('Refit requires a fresh fit')
|
|
984
|
+
|
|
985
|
+
feature_schema = _data.FeatureSchema.from_data(X)
|
|
986
|
+
label_schema = _data.LabelSchema.from_data(
|
|
987
|
+
y, classification=self._classification
|
|
988
|
+
)
|
|
989
|
+
if len(X) != len(y):
|
|
990
|
+
raise ValueError('Features and labels must have the same number of rows')
|
|
991
|
+
|
|
992
|
+
if self._classification:
|
|
993
|
+
assert label_schema.classes is not None
|
|
994
|
+
assert self._label_schema.classes is not None
|
|
995
|
+
if not np.array_equal(label_schema.classes, self._label_schema.classes):
|
|
996
|
+
raise ValueError('Refit requires the same classification labels')
|
|
997
|
+
|
|
998
|
+
ensemble_names = self.tabpack_.get_ensemble_names()
|
|
999
|
+
if ensemble is None:
|
|
1000
|
+
if len(ensemble_names) != 1:
|
|
1001
|
+
raise ValueError(
|
|
1002
|
+
'Omitting ensemble requires exactly one fitted ensemble'
|
|
1003
|
+
)
|
|
1004
|
+
ensemble = ensemble_names[0]
|
|
1005
|
+
else:
|
|
1006
|
+
if ensemble not in ensemble_names:
|
|
1007
|
+
raise ValueError('Unknown refit ensemble')
|
|
1008
|
+
|
|
1009
|
+
if max_n_models is not None and n_repeats is not None:
|
|
1010
|
+
raise ValueError('Specify either max_n_models or n_repeats, not both')
|
|
1011
|
+
|
|
1012
|
+
# >>> Checkpoint recipes
|
|
1013
|
+
|
|
1014
|
+
member_ids = self.tabpack_.load_ensemble_member_ids(ensemble)
|
|
1015
|
+
member_weights = self.tabpack_.load_ensemble_member_weights(ensemble)
|
|
1016
|
+
n_repeats = (
|
|
1017
|
+
n_repeats if max_n_models is None else max_n_models // len(member_ids)
|
|
1018
|
+
)
|
|
1019
|
+
n_repeats = 1 if n_repeats is None else n_repeats
|
|
1020
|
+
if n_repeats <= 0:
|
|
1021
|
+
raise ValueError('Refit must include at least one complete ensemble')
|
|
1022
|
+
|
|
1023
|
+
n_steps_per_epoch = self.tabpack_.get_n_steps_per_epoch()
|
|
1024
|
+
member_steps = self.tabpack_.load_ensemble_member_steps(ensemble).tolist()
|
|
1025
|
+
assert all(step % n_steps_per_epoch == 0 for step in member_steps)
|
|
1026
|
+
member_epochs = [step // n_steps_per_epoch for step in member_steps]
|
|
1027
|
+
|
|
1028
|
+
per_model_hyperparameters = self.tabpack_.load_per_model_hyperparameters(
|
|
1029
|
+
member_ids.tolist()
|
|
1030
|
+
)
|
|
1031
|
+
assert per_model_hyperparameters is not None
|
|
1032
|
+
for hyperparameters, epochs in zip(
|
|
1033
|
+
per_model_hyperparameters, member_epochs, strict=True
|
|
1034
|
+
):
|
|
1035
|
+
hyperparameters['max_epochs'] = epochs
|
|
1036
|
+
if feature_schema.n_numerical_features == 0:
|
|
1037
|
+
hyperparameters.get('model', {}).pop('numerical_embeddings', None)
|
|
1038
|
+
|
|
1039
|
+
refit_ensemble = _RefitEnsemble(
|
|
1040
|
+
name=ensemble,
|
|
1041
|
+
member_weights=(
|
|
1042
|
+
None
|
|
1043
|
+
if member_weights is None
|
|
1044
|
+
else member_weights.repeat(n_repeats) / n_repeats
|
|
1045
|
+
),
|
|
1046
|
+
prediction_transform=self.tabpack_.load_ensemble_prediction_transform(
|
|
1047
|
+
ensemble
|
|
1048
|
+
),
|
|
1049
|
+
)
|
|
1050
|
+
|
|
1051
|
+
# >>> Training configuration and datasets
|
|
1052
|
+
|
|
1053
|
+
feature_encoder = feature_schema.make_encoder(X)
|
|
1054
|
+
|
|
1055
|
+
train_kwargs = copy.deepcopy(self._fit_train_kwargs)
|
|
1056
|
+
train_kwargs['model']['categorical_cardinalities'] = (
|
|
1057
|
+
[]
|
|
1058
|
+
if feature_encoder is None
|
|
1059
|
+
else [len(categories) for categories in feature_encoder.categories_]
|
|
1060
|
+
)
|
|
1061
|
+
|
|
1062
|
+
if feature_schema.n_numerical_features == 0:
|
|
1063
|
+
train_kwargs['model'].pop('numerical_embeddings', None)
|
|
1064
|
+
|
|
1065
|
+
for name in (
|
|
1066
|
+
'n_models',
|
|
1067
|
+
'early_stopping_patience',
|
|
1068
|
+
'ensemble_eval_metrics',
|
|
1069
|
+
'ensemble_eval_criterion',
|
|
1070
|
+
'stop_when_all_ensembles_are_finalized',
|
|
1071
|
+
):
|
|
1072
|
+
train_kwargs.pop(name, None)
|
|
1073
|
+
train_kwargs.update(
|
|
1074
|
+
max_epochs=None,
|
|
1075
|
+
per_model_hyperparameters=[
|
|
1076
|
+
copy.deepcopy(hyperparameters)
|
|
1077
|
+
for _ in range(n_repeats)
|
|
1078
|
+
for hyperparameters in per_model_hyperparameters
|
|
1079
|
+
],
|
|
1080
|
+
eval_metrics={},
|
|
1081
|
+
eval_criterion=None,
|
|
1082
|
+
ensembles={},
|
|
1083
|
+
)
|
|
1084
|
+
predictor_key = self._get_predictor_key(ensemble)
|
|
1085
|
+
datasets = {
|
|
1086
|
+
'train': _data.make_dataset(
|
|
1087
|
+
{
|
|
1088
|
+
**feature_schema.encode(X, feature_encoder),
|
|
1089
|
+
'y': label_schema.encode(y),
|
|
1090
|
+
},
|
|
1091
|
+
device=self._fit_train_device,
|
|
1092
|
+
)
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
# >>> Training
|
|
1096
|
+
|
|
1097
|
+
rng = copy.deepcopy(self._refit_seed_rng)
|
|
1098
|
+
with warnings.catch_warnings():
|
|
1099
|
+
# Refit can contain fewer models than the original training run.
|
|
1100
|
+
warnings.filterwarnings(
|
|
1101
|
+
'ignore',
|
|
1102
|
+
message=(
|
|
1103
|
+
r'^The provided `max_pack_size=\d+` is no less than'
|
|
1104
|
+
r' the number of models \d+ and therefore will have'
|
|
1105
|
+
r' no effect\. Omit `max_pack_size` to avoid this warning$'
|
|
1106
|
+
),
|
|
1107
|
+
category=UserWarning,
|
|
1108
|
+
)
|
|
1109
|
+
tabpack = tabpackage.train(
|
|
1110
|
+
**train_kwargs,
|
|
1111
|
+
datasets=datasets,
|
|
1112
|
+
seed=_training.get_next_seed(rng),
|
|
1113
|
+
)
|
|
1114
|
+
predictor = _make_predictor(
|
|
1115
|
+
tabpack,
|
|
1116
|
+
predictor_key,
|
|
1117
|
+
refit_ensemble=refit_ensemble,
|
|
1118
|
+
device=train_kwargs['device'],
|
|
1119
|
+
)
|
|
1120
|
+
|
|
1121
|
+
# >>> Fitted state
|
|
1122
|
+
|
|
1123
|
+
self._set_fitted_data(feature_schema, label_schema, feature_encoder)
|
|
1124
|
+
self.tabpack_ = tabpack
|
|
1125
|
+
self._fitted_device = train_kwargs['device']
|
|
1126
|
+
self._predictor = predictor
|
|
1127
|
+
self._predictor_key = predictor_key
|
|
1128
|
+
self._default_prediction_batch_size = train_kwargs['eval_batch_size']
|
|
1129
|
+
self._refit_ensemble = refit_ensemble
|
|
1130
|
+
self.prediction_ensemble = ensemble
|
|
1131
|
+
|
|
1132
|
+
return self
|
|
1133
|
+
|
|
1134
|
+
def _predict(self, X: XLike) -> YPredLike:
|
|
1135
|
+
check_is_fitted(self)
|
|
1136
|
+
if _data.FeatureSchema.from_data(X) != self._feature_schema:
|
|
1137
|
+
raise ValueError(
|
|
1138
|
+
'Prediction data must have the same feature schema as the training data'
|
|
1139
|
+
)
|
|
1140
|
+
|
|
1141
|
+
prediction_ensemble = self._resolve_prediction_ensemble(
|
|
1142
|
+
self.tabpack_.get_ensemble_names()
|
|
1143
|
+
if self._refit_ensemble is None
|
|
1144
|
+
else [self._refit_ensemble.name]
|
|
1145
|
+
)
|
|
1146
|
+
if prediction_ensemble is None:
|
|
1147
|
+
raise RuntimeError('No ensembles were trained; prediction is unavailable')
|
|
1148
|
+
|
|
1149
|
+
batch_size = self.prediction_batch_size
|
|
1150
|
+
batch_size = (
|
|
1151
|
+
self._default_prediction_batch_size if batch_size is None else batch_size
|
|
1152
|
+
)
|
|
1153
|
+
if batch_size <= 0:
|
|
1154
|
+
raise ValueError('The prediction batch size must be positive')
|
|
1155
|
+
|
|
1156
|
+
predictor_key = self._get_predictor_key(prediction_ensemble)
|
|
1157
|
+
if predictor_key != self._predictor_key:
|
|
1158
|
+
predictor = _make_predictor(
|
|
1159
|
+
self.tabpack_,
|
|
1160
|
+
predictor_key,
|
|
1161
|
+
refit_ensemble=self._refit_ensemble,
|
|
1162
|
+
device=self._fitted_device,
|
|
1163
|
+
)
|
|
1164
|
+
self._predictor = predictor
|
|
1165
|
+
self._predictor_key = predictor_key
|
|
1166
|
+
|
|
1167
|
+
dataset = _data.make_dataset(
|
|
1168
|
+
self._feature_schema.encode(X, self._feature_encoder),
|
|
1169
|
+
device=torch.device('cpu'),
|
|
1170
|
+
)
|
|
1171
|
+
|
|
1172
|
+
assert self._predictor is not None
|
|
1173
|
+
prediction_batches = []
|
|
1174
|
+
with torch.inference_mode():
|
|
1175
|
+
for start in range(0, len(X), batch_size):
|
|
1176
|
+
batch = {
|
|
1177
|
+
name: values[start : start + batch_size].to(self._fitted_device)
|
|
1178
|
+
for name, values in dataset.items()
|
|
1179
|
+
}
|
|
1180
|
+
_, predictions = self._predictor(batch.get('x_num'), batch.get('x_cat'))
|
|
1181
|
+
prediction_batches.append(
|
|
1182
|
+
predictions[prediction_ensemble].cpu().numpy()
|
|
1183
|
+
)
|
|
1184
|
+
|
|
1185
|
+
return np.concatenate(prediction_batches)
|
|
1186
|
+
|
|
1187
|
+
|
|
1188
|
+
class TabPackClassifier(ClassifierMixin, _TabPackEstimator):
|
|
1189
|
+
"""A TabPack classifier."""
|
|
1190
|
+
|
|
1191
|
+
_classification = True
|
|
1192
|
+
|
|
1193
|
+
def predict_proba(self, X: XLike) -> YPredLike:
|
|
1194
|
+
"""Predict class probabilities, with columns in the order of `classes_`."""
|
|
1195
|
+
return self._predict(X)
|
|
1196
|
+
|
|
1197
|
+
def predict(self, X: XLike) -> YPredLike:
|
|
1198
|
+
"""Predict class labels."""
|
|
1199
|
+
probabilities = self.predict_proba(X)
|
|
1200
|
+
return self.classes_[probabilities.argmax(axis=1)]
|
|
1201
|
+
|
|
1202
|
+
|
|
1203
|
+
class TabPackRegressor(RegressorMixin, _TabPackEstimator):
|
|
1204
|
+
"""A TabPack regressor."""
|
|
1205
|
+
|
|
1206
|
+
_classification = False
|
|
1207
|
+
|
|
1208
|
+
def predict(self, X: XLike) -> YPredLike:
|
|
1209
|
+
"""Predict regression targets."""
|
|
1210
|
+
return self._predict(X)
|