tabpackage 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1210 @@
1
+ import copy
2
+ import dataclasses
3
+ import random
4
+ import warnings
5
+ from collections.abc import Mapping
6
+ from functools import partial
7
+ from typing import Any, Literal, Self
8
+
9
+ import numpy as np
10
+ import torch
11
+ from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin
12
+ from sklearn.preprocessing import OrdinalEncoder
13
+ from sklearn.utils import check_random_state
14
+ from sklearn.utils.validation import check_is_fitted
15
+ from torch import Tensor
16
+
17
+ import tabpackage
18
+ from tabpackage import _training, nn, optim
19
+ from tabpackage.sklearn import _config, _data
20
+ from tabpackage.sklearn._config import (
21
+ DatasetName,
22
+ Ensembles,
23
+ GreedyEnsemble,
24
+ HyperparameterBounds,
25
+ IntUniformDistribution,
26
+ KWArgs,
27
+ LogUniformDistribution,
28
+ MetricName,
29
+ OptionalDistribution,
30
+ OptionalHyperparameterBounds,
31
+ PerModelHyperparameters,
32
+ TopKEnsemble,
33
+ UniformDistribution,
34
+ )
35
+ from tabpackage.sklearn._data import XLike, YLike, YPredLike
36
+
37
+
38
+ def _cross_entropy(input: Tensor, target: Tensor) -> Tensor:
39
+ return torch.nn.functional.cross_entropy(
40
+ input.transpose(1, 2), target, reduction='none'
41
+ ).mean(1)
42
+
43
+
44
+ def _mse_loss(input: Tensor, target: Tensor) -> Tensor:
45
+ return (input.squeeze(-1) - target).square().mean(1)
46
+
47
+
48
+ @dataclasses.dataclass(frozen=True, kw_only=True)
49
+ class _PredictorKey:
50
+ ensemble_name: str
51
+ autocast_dtype: torch.dtype | None
52
+ backbone_forward_impl: str
53
+
54
+
55
+ @dataclasses.dataclass(frozen=True, kw_only=True)
56
+ class _RefitEnsemble:
57
+ name: str
58
+ member_weights: Tensor | None
59
+ prediction_transform: tabpackage.PredictionTransform | None
60
+
61
+
62
+ def _make_predictor(
63
+ tabpack: tabpackage.TrainedTabPack,
64
+ predictor_key: _PredictorKey,
65
+ *,
66
+ refit_ensemble: _RefitEnsemble | None = None,
67
+ device: torch.device,
68
+ ) -> tabpackage.TabPackPredictor:
69
+ if refit_ensemble is None:
70
+ predictor = tabpack.make_predictor(
71
+ ensemble_names=[predictor_key.ensemble_name],
72
+ autocast_dtype=predictor_key.autocast_dtype,
73
+ )
74
+ else:
75
+ assert predictor_key.ensemble_name == refit_ensemble.name
76
+
77
+ # Completion order can differ from the recipe and weight order.
78
+ model_ids = tabpack.load_finished_model_ids().sort().values
79
+ predictor = tabpackage.TabPackPredictor(
80
+ model_pack=tabpack._make_model_pack(
81
+ model_ids=model_ids,
82
+ model_steps=tabpack.load_model_steps(model_ids),
83
+ ),
84
+ autocast_dtype=predictor_key.autocast_dtype,
85
+ prediction_transform=tabpack.load_model_prediction_transform(),
86
+ ensembles={
87
+ predictor_key.ensemble_name: tabpackage.TabPackPredictor.EnsembleHead(
88
+ member_pack_idx=torch.arange(len(model_ids)),
89
+ member_weights=refit_ensemble.member_weights,
90
+ prediction_transform=refit_ensemble.prediction_transform,
91
+ )
92
+ },
93
+ )
94
+
95
+ predictor.to(device)
96
+ predictor.model_pack.backbone.set_forward_impl(predictor_key.backbone_forward_impl)
97
+ return predictor
98
+
99
+
100
+ # >>> Estimators
101
+
102
+
103
+ class _TabPackEstimator(BaseEstimator):
104
+ _classification: bool
105
+
106
+ def __init__(
107
+ self,
108
+ *,
109
+ # Models
110
+ n_models: int | None = None,
111
+ n_blocks: (
112
+ int | IntUniformDistribution | HyperparameterBounds[int] | None
113
+ ) = None,
114
+ d_block: int | IntUniformDistribution | HyperparameterBounds[int] | None = None,
115
+ dropout: (
116
+ float
117
+ | OptionalDistribution[UniformDistribution]
118
+ | OptionalHyperparameterBounds
119
+ | None
120
+ ) = None,
121
+ numerical_embeddings: Literal['none', 'linear-relu', 'cosine'] = 'cosine',
122
+ numerical_embeddings_d: (
123
+ int | IntUniformDistribution | HyperparameterBounds[int] | None
124
+ ) = None,
125
+ numerical_embeddings_init_scale: (
126
+ float | LogUniformDistribution | HyperparameterBounds[float] | None
127
+ ) = None,
128
+ # Training
129
+ optimizer: Literal['adamw', 'muon-adamw'] = 'muon-adamw',
130
+ adamw_lr: (
131
+ float | LogUniformDistribution | HyperparameterBounds[float] | None
132
+ ) = None,
133
+ adamw_beta1: (
134
+ float | UniformDistribution | HyperparameterBounds[float] | None
135
+ ) = None,
136
+ adamw_beta2: (
137
+ float | UniformDistribution | HyperparameterBounds[float] | None
138
+ ) = None,
139
+ adamw_eps: (
140
+ float | LogUniformDistribution | HyperparameterBounds[float] | None
141
+ ) = None,
142
+ muon_lr: (
143
+ float | LogUniformDistribution | HyperparameterBounds[float] | None
144
+ ) = None,
145
+ muon_momentum: (
146
+ float | UniformDistribution | HyperparameterBounds[float] | None
147
+ ) = None,
148
+ muon_newton_schulz_steps: int | None = None,
149
+ muon_nesterov: bool | None = None,
150
+ muon_skip_zero_updates: bool | None = None,
151
+ weight_decay: (
152
+ float | LogUniformDistribution | HyperparameterBounds[float] | None
153
+ ) = None,
154
+ batch_size: int | None = None,
155
+ max_epochs: (
156
+ int | IntUniformDistribution | HyperparameterBounds[int] | None
157
+ ) = None,
158
+ early_stopping_patience: int | None = 16,
159
+ # Per-model hyperparameters
160
+ per_model_hyperparameters: list[PerModelHyperparameters] | None = None,
161
+ # Evaluation
162
+ eval_datasets: list[DatasetName] | None = None,
163
+ eval_metrics: list[MetricName] | None = None,
164
+ eval_criterion_dataset: DatasetName | None = None,
165
+ eval_criterion_metric: MetricName | Literal['none'] | None = None,
166
+ eval_batch_size: int | None = None,
167
+ # Ensembles
168
+ ensembles: (
169
+ Ensembles | dict[str, GreedyEnsemble | TopKEnsemble | KWArgs] | None
170
+ ) = None,
171
+ ensemble_eval_datasets: list[DatasetName] | None = None,
172
+ ensemble_eval_metrics: list[MetricName] | None = None,
173
+ ensemble_eval_criterion_dataset: DatasetName | None = None,
174
+ ensemble_eval_criterion_metric: MetricName | Literal['none'] | None = None,
175
+ stop_when_all_ensembles_are_finalized: bool = True,
176
+ # Reproducibility
177
+ random_state: int | np.random.RandomState | None = None,
178
+ # Efficiency
179
+ max_pack_size: int | None = None,
180
+ device: str | torch.device | Literal['auto'] = 'auto',
181
+ aux_device: str | torch.device | Literal['auto'] = 'auto',
182
+ move_datasets_to_device: bool | list[DatasetName] = False,
183
+ autocast_dtype: Literal['bfloat16'] | None = None,
184
+ compile_model: bool = False,
185
+ compile_model_numerical_embeddings: bool = False,
186
+ compile_model_first_block: bool = False,
187
+ compile_optimizer_muon_newton_schulz: bool = False,
188
+ backbone_forward_impl: Literal['indexed', 'dense'] = 'dense',
189
+ # Printing
190
+ print_progress: bool = True,
191
+ show_progress_bar: bool | None = None,
192
+ # Prediction
193
+ prediction_ensemble: str | None = None,
194
+ prediction_batch_size: int | None = None,
195
+ prediction_autocast_dtype: Literal['bfloat16'] | None = None,
196
+ prediction_backbone_forward_impl: Literal['indexed', 'dense'] = 'dense',
197
+ ) -> None:
198
+ """TabPack.
199
+
200
+ Note:
201
+ In the docs and code, the MLPs are usually referred to as "models".
202
+
203
+ TabPack trains many MLPs with different hyperparameters in parallel and
204
+ builds ensembles as they train. Hyperparameters can be fixed or sampled
205
+ for each MLP. Use distributions or sampling bounds to define a search
206
+ space, or `per_model_hyperparameters` to specify the models directly.
207
+
208
+ The fitted TabPack provides access to training results through `tabpack_`.
209
+ After fitting, consider using `TabPack.refit` to refit a selected ensemble on
210
+ the full training data for potentially better performance.
211
+
212
+ Args:
213
+ n_models: the total number of models to train.
214
+ n_blocks: the number of backbone blocks per model.
215
+ d_block: the width of each backbone block.
216
+ dropout: the dropout rate.
217
+ numerical_embeddings: the embedding method for numerical features.
218
+ numerical_embeddings_d: the embedding size per numerical feature.
219
+ numerical_embeddings_init_scale: the initialization scale for cosine
220
+ embeddings.
221
+ optimizer: the optimization method.
222
+ adamw_lr: the AdamW learning rate.
223
+ adamw_beta1: the decay rate for AdamW's first moment estimate.
224
+ adamw_beta2: the decay rate for AdamW's second moment estimate.
225
+ adamw_eps: the term used for numerical stability in AdamW.
226
+ muon_lr: the Muon learning rate.
227
+ muon_momentum: the Muon momentum.
228
+ muon_newton_schulz_steps: the number of Newton-Schulz iterations in Muon.
229
+ muon_nesterov: whether Muon uses Nesterov momentum.
230
+ muon_skip_zero_updates: whether Muon skips zero updates.
231
+ weight_decay: the weight decay strength.
232
+ batch_size: the number of training examples per model per step.
233
+ max_epochs: the maximum number of training epochs per model.
234
+ early_stopping_patience: the number of epochs without improvement
235
+ tolerated before stopping each model.
236
+ per_model_hyperparameters: hyperparameters for each model, overriding
237
+ the shared values.
238
+ eval_datasets: the datasets used to evaluate individual models.
239
+ eval_metrics: the metrics used to evaluate individual models.
240
+ eval_criterion_dataset: the dataset used to select each model's best
241
+ checkpoint and guide early stopping.
242
+ eval_criterion_metric: the metric used to select each model's best
243
+ checkpoint and guide early stopping.
244
+ eval_batch_size: the initial number of examples per evaluation batch.
245
+ ensembles: the named ensembles to build during training.
246
+ ensemble_eval_datasets: the datasets used to evaluate ensembles.
247
+ ensemble_eval_metrics: the metrics used to evaluate ensembles.
248
+ ensemble_eval_criterion_dataset: the dataset used to select the best
249
+ version of each ensemble and guide its early stopping.
250
+ ensemble_eval_criterion_metric: the metric used to select the best
251
+ version of each ensemble and guide its early stopping.
252
+ stop_when_all_ensembles_are_finalized: whether to end training once all
253
+ ensembles are finalized, even if some models are still training.
254
+ random_state: controls randomness in hyperparameter sampling and training.
255
+ max_pack_size: the maximum number of models trained in parallel.
256
+ device: the device used for training and prediction.
257
+ aux_device: the device used to compute evaluation metrics and store
258
+ predictions and checkpoints.
259
+ move_datasets_to_device: whether to keep all datasets on the training
260
+ device, or the names of specific datasets to keep there.
261
+ autocast_dtype: the precision used for mixed-precision training.
262
+ compile_model: whether to compile the whole model.
263
+ compile_model_numerical_embeddings: whether to compile the numerical
264
+ embeddings.
265
+ compile_model_first_block: whether to compile the first backbone block.
266
+ compile_optimizer_muon_newton_schulz: whether to compile Muon's
267
+ Newton-Schulz computation.
268
+ backbone_forward_impl: the method used to evaluate the backbone.
269
+ print_progress: whether to print a training summary after each epoch.
270
+ show_progress_bar: whether to show training progress within each epoch.
271
+ prediction_ensemble: the ensemble used for prediction. Required when
272
+ training multiple ensembles.
273
+ prediction_batch_size: the number of examples per prediction batch.
274
+ prediction_autocast_dtype: the precision used for mixed-precision
275
+ prediction.
276
+ prediction_backbone_forward_impl: the method used to evaluate the
277
+ backbone during prediction.
278
+ """
279
+ self.n_models = n_models
280
+ self.n_blocks = n_blocks
281
+ self.d_block = d_block
282
+ self.dropout = dropout
283
+ self.numerical_embeddings = numerical_embeddings
284
+ self.numerical_embeddings_d = numerical_embeddings_d
285
+ self.numerical_embeddings_init_scale = numerical_embeddings_init_scale
286
+
287
+ self.optimizer = optimizer
288
+ self.adamw_lr = adamw_lr
289
+ self.adamw_beta1 = adamw_beta1
290
+ self.adamw_beta2 = adamw_beta2
291
+ self.adamw_eps = adamw_eps
292
+ self.muon_lr = muon_lr
293
+ self.muon_momentum = muon_momentum
294
+ self.muon_newton_schulz_steps = muon_newton_schulz_steps
295
+ self.muon_nesterov = muon_nesterov
296
+ self.muon_skip_zero_updates = muon_skip_zero_updates
297
+ self.weight_decay = weight_decay
298
+ self.batch_size = batch_size
299
+ self.max_epochs = max_epochs
300
+ self.early_stopping_patience = early_stopping_patience
301
+
302
+ self.per_model_hyperparameters = per_model_hyperparameters
303
+
304
+ self.eval_datasets = eval_datasets
305
+ self.eval_metrics = eval_metrics
306
+ self.eval_criterion_dataset = eval_criterion_dataset
307
+ self.eval_criterion_metric = eval_criterion_metric
308
+ self.eval_batch_size = eval_batch_size
309
+
310
+ self.ensembles = ensembles
311
+ self.ensemble_eval_datasets = ensemble_eval_datasets
312
+ self.ensemble_eval_metrics = ensemble_eval_metrics
313
+ self.ensemble_eval_criterion_dataset = ensemble_eval_criterion_dataset
314
+ self.ensemble_eval_criterion_metric = ensemble_eval_criterion_metric
315
+ self.stop_when_all_ensembles_are_finalized = (
316
+ stop_when_all_ensembles_are_finalized
317
+ )
318
+
319
+ self.random_state = random_state
320
+
321
+ self.max_pack_size = max_pack_size
322
+ self.device = device
323
+ self.aux_device = aux_device
324
+ self.move_datasets_to_device = move_datasets_to_device
325
+ self.autocast_dtype = autocast_dtype
326
+ self.compile_model = compile_model
327
+ self.compile_model_numerical_embeddings = compile_model_numerical_embeddings
328
+ self.compile_model_first_block = compile_model_first_block
329
+ self.compile_optimizer_muon_newton_schulz = compile_optimizer_muon_newton_schulz
330
+ self.backbone_forward_impl = backbone_forward_impl
331
+
332
+ self.print_progress = print_progress
333
+ self.show_progress_bar = show_progress_bar
334
+
335
+ self.prediction_ensemble = prediction_ensemble
336
+ self.prediction_batch_size = prediction_batch_size
337
+ self.prediction_autocast_dtype = prediction_autocast_dtype
338
+ self.prediction_backbone_forward_impl = prediction_backbone_forward_impl
339
+
340
+ def __sklearn_is_fitted__(self) -> bool:
341
+ return getattr(self, '_is_fitted', False)
342
+
343
+ def _make_hyperparameter_configs(
344
+ self,
345
+ *,
346
+ n_numerical_features: int,
347
+ d_out: int,
348
+ rng: random.Random,
349
+ ) -> tuple[KWArgs, KWArgs, list[KWArgs]]:
350
+ unused_hyperparameters = set()
351
+ if self.optimizer == 'adamw':
352
+ unused_hyperparameters.update(
353
+ (
354
+ 'muon_lr',
355
+ 'muon_momentum',
356
+ 'muon_newton_schulz_steps',
357
+ 'muon_nesterov',
358
+ 'muon_skip_zero_updates',
359
+ )
360
+ )
361
+ elif (
362
+ self.muon_newton_schulz_steps is not None
363
+ and self.muon_newton_schulz_steps < 0
364
+ ):
365
+ raise ValueError('`muon_newton_schulz_steps` must be non-negative')
366
+
367
+ use_numerical_embeddings = (
368
+ n_numerical_features > 0 and self.numerical_embeddings != 'none'
369
+ )
370
+ if not use_numerical_embeddings:
371
+ unused_hyperparameters.add('numerical_embeddings_d')
372
+ if not use_numerical_embeddings or self.numerical_embeddings != 'cosine':
373
+ unused_hyperparameters.add('numerical_embeddings_init_scale')
374
+
375
+ for name in unused_hyperparameters:
376
+ if getattr(self, name) is not None:
377
+ raise ValueError(
378
+ f'`{name}` does not apply to the current model and optimizer'
379
+ ' configuration; leave it as `None`'
380
+ )
381
+
382
+ model_config: KWArgs = {
383
+ 'd_out': d_out,
384
+ 'forward_impl': self.backbone_forward_impl,
385
+ }
386
+ if use_numerical_embeddings:
387
+ model_config['numerical_embeddings'] = {
388
+ 'type': (
389
+ nn.CosineEmbeddingsPack
390
+ if self.numerical_embeddings == 'cosine'
391
+ else nn.LinearReLUEmbeddingsPack
392
+ ),
393
+ }
394
+
395
+ optimizer_config: KWArgs = {
396
+ 'type': (
397
+ optim.AdamWPack if self.optimizer == 'adamw' else optim.MuonAdamWPack
398
+ )
399
+ }
400
+ for name in (
401
+ 'muon_newton_schulz_steps',
402
+ 'muon_nesterov',
403
+ 'muon_skip_zero_updates',
404
+ ):
405
+ value = getattr(self, name)
406
+ if value is not None:
407
+ optimizer_config[name] = value
408
+
409
+ # >>> Explicit per-model configuration
410
+
411
+ if self.per_model_hyperparameters is not None:
412
+ hyperparameter_names = PerModelHyperparameters.__annotations__.keys()
413
+ shared_hyperparameters = {
414
+ name: getattr(self, name)
415
+ for name in hyperparameter_names
416
+ if getattr(self, name) is not None
417
+ }
418
+ for name, value in shared_hyperparameters.items():
419
+ if isinstance(value, (_config.Distribution, dict)):
420
+ raise TypeError(
421
+ f'`{name}` must be a scalar when `per_model_hyperparameters` is'
422
+ ' provided'
423
+ )
424
+
425
+ per_model_hyperparameter_names = {
426
+ name for entry in self.per_model_hyperparameters for name in entry
427
+ }
428
+
429
+ required_hyperparameters = {
430
+ 'n_blocks',
431
+ 'dropout',
432
+ 'adamw_lr',
433
+ 'weight_decay',
434
+ }
435
+ if self.optimizer == 'muon-adamw':
436
+ required_hyperparameters.add('muon_lr')
437
+ if use_numerical_embeddings:
438
+ required_hyperparameters.add('numerical_embeddings_d')
439
+ if self.numerical_embeddings == 'cosine':
440
+ required_hyperparameters.add('numerical_embeddings_init_scale')
441
+ missing_hyperparameters = (
442
+ required_hyperparameters
443
+ - shared_hyperparameters.keys()
444
+ - per_model_hyperparameter_names
445
+ )
446
+ if missing_hyperparameters:
447
+ raise ValueError(
448
+ 'Missing explicit hyperparameters:'
449
+ f' {sorted(missing_hyperparameters)}'
450
+ )
451
+
452
+ if self.d_block is None and 'd_block' not in per_model_hyperparameter_names:
453
+ shared_hyperparameters['d_block'] = 384
454
+
455
+ def translate(values: Mapping[str, Any], config: KWArgs) -> None:
456
+ for name, value in values.items():
457
+ if name in unused_hyperparameters:
458
+ raise ValueError(
459
+ f'`{name}` does not apply to the current model and'
460
+ ' optimizer configuration; omit it from'
461
+ ' `per_model_hyperparameters`'
462
+ )
463
+ if value is not None:
464
+ _config.validate_hyperparameter(name, value)
465
+
466
+ if name in ('n_blocks', 'd_block', 'dropout'):
467
+ config['model'][name] = value
468
+ elif name in (
469
+ 'numerical_embeddings_d',
470
+ 'numerical_embeddings_init_scale',
471
+ ):
472
+ embedding_key = (
473
+ 'd_embedding'
474
+ if name == 'numerical_embeddings_d'
475
+ else 'init_scale'
476
+ )
477
+ config['model'].setdefault('numerical_embeddings', {})[
478
+ embedding_key
479
+ ] = value
480
+ elif name == 'max_epochs':
481
+ config[name] = value
482
+ else:
483
+ optimizer_key = (
484
+ name.removeprefix('adamw_')
485
+ if self.optimizer == 'adamw'
486
+ else name
487
+ )
488
+ config['optimizer'][optimizer_key] = value
489
+
490
+ translate(
491
+ shared_hyperparameters,
492
+ {'model': model_config, 'optimizer': optimizer_config},
493
+ )
494
+
495
+ explicit_hyperparameters = []
496
+ for entry in self.per_model_hyperparameters:
497
+ config: KWArgs = {'model': {}, 'optimizer': {}}
498
+ translate(entry, config)
499
+ explicit_hyperparameters.append(config)
500
+
501
+ return model_config, optimizer_config, explicit_hyperparameters
502
+
503
+ # >>> Hyperparameter resolution
504
+
505
+ default_hyperparameters = {
506
+ 'n_blocks': IntUniformDistribution(1, 3),
507
+ 'd_block': 384,
508
+ 'dropout': OptionalDistribution(UniformDistribution(0.0, 0.5)),
509
+ 'adamw_lr': LogUniformDistribution(1e-4, 5e-3),
510
+ 'weight_decay': LogUniformDistribution(1e-3, 1.0),
511
+ 'adamw_beta1': None,
512
+ 'adamw_beta2': None,
513
+ 'adamw_eps': None,
514
+ 'max_epochs': None,
515
+ }
516
+ if use_numerical_embeddings:
517
+ default_hyperparameters['numerical_embeddings_d'] = IntUniformDistribution(
518
+ 8, 32
519
+ )
520
+ if self.numerical_embeddings == 'cosine':
521
+ default_hyperparameters['numerical_embeddings_init_scale'] = (
522
+ LogUniformDistribution(0.01, 10.0)
523
+ )
524
+ if self.optimizer == 'muon-adamw':
525
+ default_hyperparameters['muon_lr'] = LogUniformDistribution(1e-3, 1e-1)
526
+ default_hyperparameters['muon_momentum'] = None
527
+
528
+ resolved_hyperparameters = {}
529
+ for name, default in default_hyperparameters.items():
530
+ value = getattr(self, name)
531
+ value = default if value is None else value
532
+ if value is None:
533
+ continue
534
+
535
+ if isinstance(value, dict):
536
+ bounds = value.copy()
537
+ if name == 'dropout':
538
+ optional = bounds.pop('optional')
539
+ value = UniformDistribution(**bounds)
540
+ value = OptionalDistribution(value) if optional else value
541
+ elif name in (
542
+ 'n_blocks',
543
+ 'd_block',
544
+ 'numerical_embeddings_d',
545
+ 'max_epochs',
546
+ ):
547
+ value = IntUniformDistribution(**bounds)
548
+ elif name in ('adamw_beta1', 'adamw_beta2', 'muon_momentum'):
549
+ value = UniformDistribution(**bounds)
550
+ else:
551
+ value = LogUniformDistribution(**bounds)
552
+
553
+ _config.validate_hyperparameter(name, value)
554
+ resolved_hyperparameters[name] = value
555
+
556
+ def sample(name: str) -> int | float:
557
+ value = resolved_hyperparameters[name]
558
+ return (
559
+ value.sample(rng) if isinstance(value, _config.Distribution) else value
560
+ )
561
+
562
+ # >>> Optimizer
563
+
564
+ per_model_optimizer_hyperparameters = {}
565
+ for name in (
566
+ 'adamw_lr',
567
+ 'weight_decay',
568
+ 'muon_lr',
569
+ 'adamw_beta1',
570
+ 'adamw_beta2',
571
+ 'adamw_eps',
572
+ 'muon_momentum',
573
+ ):
574
+ if name not in resolved_hyperparameters:
575
+ continue
576
+
577
+ optimizer_key = (
578
+ name.removeprefix('adamw_') if self.optimizer == 'adamw' else name
579
+ )
580
+ if name in ('adamw_lr', 'weight_decay', 'muon_lr') or isinstance(
581
+ resolved_hyperparameters[name], _config.Distribution
582
+ ):
583
+ per_model_optimizer_hyperparameters[optimizer_key] = name
584
+ else:
585
+ optimizer_config[optimizer_key] = resolved_hyperparameters[name]
586
+
587
+ # >>> Per-model sampling
588
+
589
+ per_model_hyperparameters = []
590
+ for _ in range(64 if self.n_models is None else self.n_models):
591
+ model_hyperparameters = {}
592
+ if use_numerical_embeddings:
593
+ model_hyperparameters['numerical_embeddings'] = {
594
+ 'd_embedding': sample('numerical_embeddings_d'),
595
+ }
596
+ if self.numerical_embeddings == 'cosine':
597
+ model_hyperparameters['numerical_embeddings']['init_scale'] = (
598
+ sample('numerical_embeddings_init_scale')
599
+ )
600
+
601
+ model_hyperparameters['n_blocks'] = sample('n_blocks')
602
+ if isinstance(resolved_hyperparameters['d_block'], _config.Distribution):
603
+ model_hyperparameters['d_block'] = sample('d_block')
604
+ model_hyperparameters['dropout'] = sample('dropout')
605
+
606
+ hyperparameters = {
607
+ 'model': model_hyperparameters,
608
+ 'optimizer': {
609
+ key: sample(name)
610
+ for key, name in per_model_optimizer_hyperparameters.items()
611
+ },
612
+ }
613
+ if isinstance(
614
+ resolved_hyperparameters.get('max_epochs'), _config.Distribution
615
+ ):
616
+ hyperparameters['max_epochs'] = sample('max_epochs')
617
+ per_model_hyperparameters.append(hyperparameters)
618
+
619
+ if not isinstance(resolved_hyperparameters['d_block'], _config.Distribution):
620
+ model_config['d_block'] = resolved_hyperparameters['d_block']
621
+
622
+ return model_config, optimizer_config, per_model_hyperparameters
623
+
624
+ def _make_train_kwargs(
625
+ self,
626
+ *,
627
+ n_numerical_features: int,
628
+ d_out: int,
629
+ rng: random.Random,
630
+ default_dataset: DatasetName | None,
631
+ labels: dict[DatasetName, np.ndarray],
632
+ device: torch.device,
633
+ ) -> KWArgs:
634
+ batch_size = self.batch_size
635
+ if batch_size is None:
636
+ n_train = len(labels['train'])
637
+ batch_size = (
638
+ 32
639
+ if n_train < 2048
640
+ else 64
641
+ if n_train < 4096
642
+ else 128
643
+ if n_train < 8192
644
+ else 256
645
+ if n_train < 32768
646
+ else 512
647
+ if n_train < 131072
648
+ else 1024
649
+ )
650
+
651
+ eval_batch_size = self.eval_batch_size
652
+ eval_batch_size = (
653
+ (4096 if device.type == 'cuda' else 512)
654
+ if eval_batch_size is None
655
+ else eval_batch_size
656
+ )
657
+
658
+ aux_device = (
659
+ (device if device.type == 'mps' else torch.device('cpu'))
660
+ if self.aux_device == 'auto'
661
+ else _config.resolve_device(self.aux_device)
662
+ )
663
+
664
+ eval_criterion = _config.resolve_criterion(
665
+ self.eval_criterion_dataset,
666
+ self.eval_criterion_metric,
667
+ default_dataset=default_dataset,
668
+ default_metric='accuracy' if self._classification else 'mse',
669
+ d_out=d_out,
670
+ )
671
+ eval_metrics = _config.resolve_eval_metrics(
672
+ self.eval_datasets,
673
+ self.eval_metrics,
674
+ eval_criterion=eval_criterion,
675
+ d_out=d_out,
676
+ labels=labels,
677
+ )
678
+
679
+ ensembles = self.ensembles
680
+ ensembles = (
681
+ Ensembles({'default': GreedyEnsemble()})
682
+ if ensembles is None
683
+ else (Ensembles(ensembles) if isinstance(ensembles, dict) else ensembles)
684
+ )
685
+
686
+ ensemble_eval_criterion = None
687
+ ensemble_eval_metrics = {}
688
+ if ensembles.members:
689
+ ensemble_eval_criterion = _config.resolve_criterion(
690
+ self.ensemble_eval_criterion_dataset,
691
+ self.ensemble_eval_criterion_metric,
692
+ default_dataset=(
693
+ default_dataset
694
+ if eval_criterion is None
695
+ else eval_criterion.dataset
696
+ ),
697
+ default_metric=(
698
+ None if eval_criterion is None else eval_criterion.metric
699
+ ),
700
+ d_out=d_out,
701
+ )
702
+ ensemble_eval_metrics = _config.resolve_eval_metrics(
703
+ self.ensemble_eval_datasets,
704
+ self.ensemble_eval_metrics,
705
+ eval_criterion=ensemble_eval_criterion,
706
+ d_out=d_out,
707
+ labels=labels,
708
+ )
709
+
710
+ ensemble_configs = ensembles.make_config(
711
+ d_out=d_out,
712
+ eval_criterion=ensemble_eval_criterion,
713
+ default_algorithm_dataset=default_dataset,
714
+ labels=labels,
715
+ device=device,
716
+ )
717
+
718
+ model_config, optimizer_config, per_model_hyperparameters = (
719
+ self._make_hyperparameter_configs(
720
+ n_numerical_features=n_numerical_features,
721
+ d_out=d_out,
722
+ rng=rng,
723
+ )
724
+ )
725
+
726
+ max_epochs = self.max_epochs
727
+ max_epochs = (
728
+ None
729
+ if isinstance(max_epochs, (IntUniformDistribution, dict))
730
+ else max_epochs
731
+ )
732
+
733
+ train_kwargs = {
734
+ 'model': model_config,
735
+ 'prediction_transform': (
736
+ partial(torch.softmax, dim=-1)
737
+ if self._classification
738
+ else partial(torch.squeeze, dim=-1)
739
+ ),
740
+ 'loss_fn': _cross_entropy if self._classification else _mse_loss,
741
+ 'optimizer': optimizer_config,
742
+ 'batch_size': batch_size,
743
+ 'max_epochs': max_epochs,
744
+ 'per_model_hyperparameters': per_model_hyperparameters,
745
+ 'eval_metrics': eval_metrics,
746
+ 'eval_criterion': eval_criterion,
747
+ 'eval_batch_size': eval_batch_size,
748
+ 'ensembles': ensemble_configs,
749
+ 'max_pack_size': self.max_pack_size,
750
+ 'device': device,
751
+ 'aux_device': aux_device,
752
+ 'autocast_dtype': self.autocast_dtype,
753
+ 'compile_model': self.compile_model,
754
+ 'compile_model_numerical_embeddings': (
755
+ self.compile_model_numerical_embeddings
756
+ ),
757
+ 'compile_model_first_block': self.compile_model_first_block,
758
+ 'compile_optimizer_muon_newton_schulz': (
759
+ self.compile_optimizer_muon_newton_schulz
760
+ ),
761
+ 'print_progress': self.print_progress,
762
+ 'show_progress_bar': self.show_progress_bar,
763
+ }
764
+
765
+ if self.n_models is not None:
766
+ train_kwargs['n_models'] = self.n_models
767
+ if eval_criterion is not None:
768
+ train_kwargs['early_stopping_patience'] = self.early_stopping_patience
769
+ if ensemble_configs:
770
+ train_kwargs.update(
771
+ ensemble_eval_metrics={
772
+ dataset: {
773
+ name: partial(_config.compute_scalar_metric, metric_fn)
774
+ for name, metric_fn in dataset_metrics.items()
775
+ }
776
+ for dataset, dataset_metrics in ensemble_eval_metrics.items()
777
+ },
778
+ ensemble_eval_criterion=ensemble_eval_criterion,
779
+ stop_when_all_ensembles_are_finalized=self.stop_when_all_ensembles_are_finalized,
780
+ )
781
+
782
+ return train_kwargs
783
+
784
+ def _resolve_prediction_ensemble(self, ensemble_names: list[str]) -> str | None:
785
+ if self.prediction_ensemble is None:
786
+ if len(ensemble_names) > 1:
787
+ raise ValueError(
788
+ 'Set prediction_ensemble when using multiple ensembles'
789
+ )
790
+ return ensemble_names[0] if ensemble_names else None
791
+
792
+ if self.prediction_ensemble not in ensemble_names:
793
+ raise ValueError('Unknown prediction ensemble')
794
+ return self.prediction_ensemble
795
+
796
+ def _get_predictor_key(self, ensemble_name: str) -> _PredictorKey:
797
+ return _PredictorKey(
798
+ ensemble_name=ensemble_name,
799
+ autocast_dtype=_config.resolve_autocast_dtype(
800
+ self.prediction_autocast_dtype
801
+ ),
802
+ backbone_forward_impl=self.prediction_backbone_forward_impl,
803
+ )
804
+
805
+ def _set_fitted_data(
806
+ self,
807
+ feature_schema: _data.FeatureSchema,
808
+ label_schema: _data.LabelSchema,
809
+ feature_encoder: OrdinalEncoder | None,
810
+ ) -> None:
811
+ self._feature_schema = feature_schema
812
+ self._label_schema = label_schema
813
+ self._feature_encoder = feature_encoder
814
+
815
+ self.n_features_in_ = feature_schema.n_features
816
+ if feature_schema.columns is None:
817
+ if hasattr(self, 'feature_names_in_'):
818
+ del self.feature_names_in_
819
+ else:
820
+ self.feature_names_in_ = np.array(feature_schema.columns, dtype=object)
821
+
822
+ if label_schema.classes is not None:
823
+ self.classes_ = label_schema.classes
824
+
825
+ def fit(
826
+ self,
827
+ X: XLike,
828
+ y: YLike,
829
+ *,
830
+ extra_set: tuple[XLike, YLike] | dict[DatasetName, tuple[XLike, YLike]],
831
+ ) -> Self:
832
+ """Train models and build ensembles using the extra data for validation.
833
+
834
+ Args:
835
+ X: the training features.
836
+ y: the training labels or regression targets.
837
+ extra_set: extra data as an `(X, y)` pair or a collection of named
838
+ pairs. Use these datasets to evaluate models and build ensembles.
839
+ When passing named datasets, choose the validation dataset through
840
+ `eval_criterion_dataset`.
841
+
842
+ Returns:
843
+ The fitted estimator.
844
+ """
845
+ # >>> Data validation
846
+
847
+ feature_schema = _data.FeatureSchema.from_data(X)
848
+ label_schema = _data.LabelSchema.from_data(
849
+ y, classification=self._classification
850
+ )
851
+ if len(X) != len(y):
852
+ raise ValueError('Features and labels must have the same number of rows')
853
+
854
+ default_dataset = None if isinstance(extra_set, dict) else 'extra'
855
+ extra_datasets = (
856
+ extra_set if isinstance(extra_set, dict) else {'extra': extra_set}
857
+ )
858
+ if 'train' in extra_datasets:
859
+ raise ValueError('The dataset name "train" is reserved')
860
+
861
+ for X_extra, y_extra in extra_datasets.values():
862
+ if _data.FeatureSchema.from_data(X_extra) != feature_schema:
863
+ raise ValueError(
864
+ 'Extra datasets must have the same feature schema as the training'
865
+ ' data'
866
+ )
867
+ label_schema.validate(y_extra)
868
+ if len(X_extra) != len(y_extra):
869
+ raise ValueError(
870
+ 'Features and labels must have the same number of rows'
871
+ )
872
+
873
+ # >>> Encoding
874
+
875
+ feature_encoder = feature_schema.make_encoder(X)
876
+ arrays = {
877
+ name: {
878
+ **feature_schema.encode(features, feature_encoder),
879
+ 'y': label_schema.encode(target),
880
+ }
881
+ for name, (features, target) in {'train': (X, y), **extra_datasets}.items()
882
+ }
883
+
884
+ # >>> Training configuration
885
+
886
+ device = _config.resolve_device(self.device)
887
+ base_seed = int(
888
+ check_random_state(self.random_state).randint(0, 2**32 - 2, dtype=np.int64)
889
+ )
890
+ rng = random.Random(base_seed)
891
+ train_kwargs = self._make_train_kwargs(
892
+ n_numerical_features=feature_schema.n_numerical_features,
893
+ d_out=1 if label_schema.classes is None else len(label_schema.classes),
894
+ rng=random.Random(_training.get_next_seed(rng)),
895
+ default_dataset=default_dataset,
896
+ labels={name: data['y'] for name, data in arrays.items()},
897
+ device=device,
898
+ )
899
+ train_kwargs['model']['categorical_cardinalities'] = (
900
+ []
901
+ if feature_encoder is None
902
+ else [len(categories) for categories in feature_encoder.categories_]
903
+ )
904
+ prediction_ensemble = self._resolve_prediction_ensemble(
905
+ list(train_kwargs['ensembles'])
906
+ )
907
+ predictor_key = (
908
+ None
909
+ if prediction_ensemble is None
910
+ else self._get_predictor_key(prediction_ensemble)
911
+ )
912
+
913
+ # >>> Dataset placement
914
+
915
+ move_datasets_to_device = self.move_datasets_to_device
916
+ datasets_on_device = (
917
+ set(arrays)
918
+ if move_datasets_to_device is True
919
+ else (
920
+ set()
921
+ if move_datasets_to_device is False
922
+ else set(move_datasets_to_device)
923
+ )
924
+ )
925
+ if not datasets_on_device <= arrays.keys():
926
+ raise ValueError('Unknown dataset in move_datasets_to_device')
927
+
928
+ datasets = {
929
+ name: _data.make_dataset(
930
+ data,
931
+ device=(
932
+ device if name in datasets_on_device else train_kwargs['aux_device']
933
+ ),
934
+ )
935
+ for name, data in arrays.items()
936
+ }
937
+ del arrays
938
+
939
+ # >>> Training
940
+
941
+ fit_train_kwargs = copy.deepcopy(train_kwargs)
942
+ train_device = _config.resolve_device(datasets['train']['y'].device)
943
+ tabpack = tabpackage.train(
944
+ **train_kwargs,
945
+ datasets=datasets,
946
+ seed=_training.get_next_seed(rng),
947
+ )
948
+ predictor = (
949
+ None
950
+ if predictor_key is None
951
+ else _make_predictor(tabpack, predictor_key, device=device)
952
+ )
953
+
954
+ # >>> Fitted state
955
+
956
+ # Preserve prior fitted state if training or predictor construction fails.
957
+ self._set_fitted_data(feature_schema, label_schema, feature_encoder)
958
+ self.tabpack_ = tabpack
959
+ self._fitted_device = device
960
+ self._predictor = predictor
961
+ self._predictor_key = predictor_key
962
+ self._default_prediction_batch_size = train_kwargs['eval_batch_size']
963
+ self._fit_train_kwargs = fit_train_kwargs
964
+ self._refit_seed_rng = rng
965
+ self._fit_train_device = train_device
966
+ self._refit_ensemble: _RefitEnsemble | None = None
967
+ self._is_fitted = True
968
+
969
+ return self
970
+
971
+ def refit(
972
+ self,
973
+ X: XLike,
974
+ y: YLike,
975
+ *,
976
+ ensemble: str | None = None,
977
+ max_n_models: int | None = None,
978
+ n_repeats: int | None = None,
979
+ ) -> Self:
980
+ """Retrain an ensemble from scratch on full training data."""
981
+ check_is_fitted(self)
982
+ if self._refit_ensemble is not None:
983
+ raise RuntimeError('Refit requires a fresh fit')
984
+
985
+ feature_schema = _data.FeatureSchema.from_data(X)
986
+ label_schema = _data.LabelSchema.from_data(
987
+ y, classification=self._classification
988
+ )
989
+ if len(X) != len(y):
990
+ raise ValueError('Features and labels must have the same number of rows')
991
+
992
+ if self._classification:
993
+ assert label_schema.classes is not None
994
+ assert self._label_schema.classes is not None
995
+ if not np.array_equal(label_schema.classes, self._label_schema.classes):
996
+ raise ValueError('Refit requires the same classification labels')
997
+
998
+ ensemble_names = self.tabpack_.get_ensemble_names()
999
+ if ensemble is None:
1000
+ if len(ensemble_names) != 1:
1001
+ raise ValueError(
1002
+ 'Omitting ensemble requires exactly one fitted ensemble'
1003
+ )
1004
+ ensemble = ensemble_names[0]
1005
+ else:
1006
+ if ensemble not in ensemble_names:
1007
+ raise ValueError('Unknown refit ensemble')
1008
+
1009
+ if max_n_models is not None and n_repeats is not None:
1010
+ raise ValueError('Specify either max_n_models or n_repeats, not both')
1011
+
1012
+ # >>> Checkpoint recipes
1013
+
1014
+ member_ids = self.tabpack_.load_ensemble_member_ids(ensemble)
1015
+ member_weights = self.tabpack_.load_ensemble_member_weights(ensemble)
1016
+ n_repeats = (
1017
+ n_repeats if max_n_models is None else max_n_models // len(member_ids)
1018
+ )
1019
+ n_repeats = 1 if n_repeats is None else n_repeats
1020
+ if n_repeats <= 0:
1021
+ raise ValueError('Refit must include at least one complete ensemble')
1022
+
1023
+ n_steps_per_epoch = self.tabpack_.get_n_steps_per_epoch()
1024
+ member_steps = self.tabpack_.load_ensemble_member_steps(ensemble).tolist()
1025
+ assert all(step % n_steps_per_epoch == 0 for step in member_steps)
1026
+ member_epochs = [step // n_steps_per_epoch for step in member_steps]
1027
+
1028
+ per_model_hyperparameters = self.tabpack_.load_per_model_hyperparameters(
1029
+ member_ids.tolist()
1030
+ )
1031
+ assert per_model_hyperparameters is not None
1032
+ for hyperparameters, epochs in zip(
1033
+ per_model_hyperparameters, member_epochs, strict=True
1034
+ ):
1035
+ hyperparameters['max_epochs'] = epochs
1036
+ if feature_schema.n_numerical_features == 0:
1037
+ hyperparameters.get('model', {}).pop('numerical_embeddings', None)
1038
+
1039
+ refit_ensemble = _RefitEnsemble(
1040
+ name=ensemble,
1041
+ member_weights=(
1042
+ None
1043
+ if member_weights is None
1044
+ else member_weights.repeat(n_repeats) / n_repeats
1045
+ ),
1046
+ prediction_transform=self.tabpack_.load_ensemble_prediction_transform(
1047
+ ensemble
1048
+ ),
1049
+ )
1050
+
1051
+ # >>> Training configuration and datasets
1052
+
1053
+ feature_encoder = feature_schema.make_encoder(X)
1054
+
1055
+ train_kwargs = copy.deepcopy(self._fit_train_kwargs)
1056
+ train_kwargs['model']['categorical_cardinalities'] = (
1057
+ []
1058
+ if feature_encoder is None
1059
+ else [len(categories) for categories in feature_encoder.categories_]
1060
+ )
1061
+
1062
+ if feature_schema.n_numerical_features == 0:
1063
+ train_kwargs['model'].pop('numerical_embeddings', None)
1064
+
1065
+ for name in (
1066
+ 'n_models',
1067
+ 'early_stopping_patience',
1068
+ 'ensemble_eval_metrics',
1069
+ 'ensemble_eval_criterion',
1070
+ 'stop_when_all_ensembles_are_finalized',
1071
+ ):
1072
+ train_kwargs.pop(name, None)
1073
+ train_kwargs.update(
1074
+ max_epochs=None,
1075
+ per_model_hyperparameters=[
1076
+ copy.deepcopy(hyperparameters)
1077
+ for _ in range(n_repeats)
1078
+ for hyperparameters in per_model_hyperparameters
1079
+ ],
1080
+ eval_metrics={},
1081
+ eval_criterion=None,
1082
+ ensembles={},
1083
+ )
1084
+ predictor_key = self._get_predictor_key(ensemble)
1085
+ datasets = {
1086
+ 'train': _data.make_dataset(
1087
+ {
1088
+ **feature_schema.encode(X, feature_encoder),
1089
+ 'y': label_schema.encode(y),
1090
+ },
1091
+ device=self._fit_train_device,
1092
+ )
1093
+ }
1094
+
1095
+ # >>> Training
1096
+
1097
+ rng = copy.deepcopy(self._refit_seed_rng)
1098
+ with warnings.catch_warnings():
1099
+ # Refit can contain fewer models than the original training run.
1100
+ warnings.filterwarnings(
1101
+ 'ignore',
1102
+ message=(
1103
+ r'^The provided `max_pack_size=\d+` is no less than'
1104
+ r' the number of models \d+ and therefore will have'
1105
+ r' no effect\. Omit `max_pack_size` to avoid this warning$'
1106
+ ),
1107
+ category=UserWarning,
1108
+ )
1109
+ tabpack = tabpackage.train(
1110
+ **train_kwargs,
1111
+ datasets=datasets,
1112
+ seed=_training.get_next_seed(rng),
1113
+ )
1114
+ predictor = _make_predictor(
1115
+ tabpack,
1116
+ predictor_key,
1117
+ refit_ensemble=refit_ensemble,
1118
+ device=train_kwargs['device'],
1119
+ )
1120
+
1121
+ # >>> Fitted state
1122
+
1123
+ self._set_fitted_data(feature_schema, label_schema, feature_encoder)
1124
+ self.tabpack_ = tabpack
1125
+ self._fitted_device = train_kwargs['device']
1126
+ self._predictor = predictor
1127
+ self._predictor_key = predictor_key
1128
+ self._default_prediction_batch_size = train_kwargs['eval_batch_size']
1129
+ self._refit_ensemble = refit_ensemble
1130
+ self.prediction_ensemble = ensemble
1131
+
1132
+ return self
1133
+
1134
+ def _predict(self, X: XLike) -> YPredLike:
1135
+ check_is_fitted(self)
1136
+ if _data.FeatureSchema.from_data(X) != self._feature_schema:
1137
+ raise ValueError(
1138
+ 'Prediction data must have the same feature schema as the training data'
1139
+ )
1140
+
1141
+ prediction_ensemble = self._resolve_prediction_ensemble(
1142
+ self.tabpack_.get_ensemble_names()
1143
+ if self._refit_ensemble is None
1144
+ else [self._refit_ensemble.name]
1145
+ )
1146
+ if prediction_ensemble is None:
1147
+ raise RuntimeError('No ensembles were trained; prediction is unavailable')
1148
+
1149
+ batch_size = self.prediction_batch_size
1150
+ batch_size = (
1151
+ self._default_prediction_batch_size if batch_size is None else batch_size
1152
+ )
1153
+ if batch_size <= 0:
1154
+ raise ValueError('The prediction batch size must be positive')
1155
+
1156
+ predictor_key = self._get_predictor_key(prediction_ensemble)
1157
+ if predictor_key != self._predictor_key:
1158
+ predictor = _make_predictor(
1159
+ self.tabpack_,
1160
+ predictor_key,
1161
+ refit_ensemble=self._refit_ensemble,
1162
+ device=self._fitted_device,
1163
+ )
1164
+ self._predictor = predictor
1165
+ self._predictor_key = predictor_key
1166
+
1167
+ dataset = _data.make_dataset(
1168
+ self._feature_schema.encode(X, self._feature_encoder),
1169
+ device=torch.device('cpu'),
1170
+ )
1171
+
1172
+ assert self._predictor is not None
1173
+ prediction_batches = []
1174
+ with torch.inference_mode():
1175
+ for start in range(0, len(X), batch_size):
1176
+ batch = {
1177
+ name: values[start : start + batch_size].to(self._fitted_device)
1178
+ for name, values in dataset.items()
1179
+ }
1180
+ _, predictions = self._predictor(batch.get('x_num'), batch.get('x_cat'))
1181
+ prediction_batches.append(
1182
+ predictions[prediction_ensemble].cpu().numpy()
1183
+ )
1184
+
1185
+ return np.concatenate(prediction_batches)
1186
+
1187
+
1188
+ class TabPackClassifier(ClassifierMixin, _TabPackEstimator):
1189
+ """A TabPack classifier."""
1190
+
1191
+ _classification = True
1192
+
1193
+ def predict_proba(self, X: XLike) -> YPredLike:
1194
+ """Predict class probabilities, with columns in the order of `classes_`."""
1195
+ return self._predict(X)
1196
+
1197
+ def predict(self, X: XLike) -> YPredLike:
1198
+ """Predict class labels."""
1199
+ probabilities = self.predict_proba(X)
1200
+ return self.classes_[probabilities.argmax(axis=1)]
1201
+
1202
+
1203
+ class TabPackRegressor(RegressorMixin, _TabPackEstimator):
1204
+ """A TabPack regressor."""
1205
+
1206
+ _classification = False
1207
+
1208
+ def predict(self, X: XLike) -> YPredLike:
1209
+ """Predict regression targets."""
1210
+ return self._predict(X)