tabpackage 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,18 @@
1
+ """The Scikit-learn-compatible part of TabPackage."""
2
+
3
+ from tabpackage.sklearn._config import ( # noqa: F401
4
+ Ensembles,
5
+ GreedyEnsemble,
6
+ HyperparameterBounds,
7
+ IntUniformDistribution,
8
+ LogUniformDistribution,
9
+ OptionalDistribution,
10
+ OptionalHyperparameterBounds,
11
+ PerModelHyperparameters,
12
+ TopKEnsemble,
13
+ UniformDistribution,
14
+ )
15
+ from tabpackage.sklearn._estimators import ( # noqa: F401
16
+ TabPackClassifier,
17
+ TabPackRegressor,
18
+ )
@@ -0,0 +1,561 @@
1
+ import math
2
+ import random
3
+ from functools import partial
4
+ from typing import Any, Generic, Literal, Self, TypeAlias, TypedDict, TypeVar
5
+
6
+ import numpy as np
7
+ import torch
8
+ from sklearn.base import BaseEstimator
9
+ from sklearn.utils.metaestimators import _BaseComposition
10
+ from torch import Tensor
11
+
12
+ import tabpackage
13
+ from tabpackage import ensemble, metrics
14
+
15
+ KWArgs: TypeAlias = dict[str, Any]
16
+ DatasetName: TypeAlias = str
17
+ MetricName: TypeAlias = Literal['accuracy', 'log_loss', 'roc_auc', 'mse', 'mae', 'r2']
18
+
19
+
20
+ # >>> Per-model hyperparameters
21
+
22
+
23
+ class Distribution(BaseEstimator):
24
+ def _get_bounds(self) -> tuple[float, float]:
25
+ raise NotImplementedError
26
+
27
+ def sample(self, rng: random.Random) -> float:
28
+ """Draw a value using a given random generator."""
29
+ raise NotImplementedError
30
+
31
+
32
+ class IntUniformDistribution(Distribution):
33
+ """A uniform distribution over integers within inclusive bounds."""
34
+
35
+ def __init__(self, min: int, max: int) -> None:
36
+ self.min = min
37
+ self.max = max
38
+
39
+ def _get_bounds(self) -> tuple[int, int]:
40
+ if self.min > self.max:
41
+ raise ValueError('`min` must not exceed `max`')
42
+
43
+ return self.min, self.max
44
+
45
+ def sample(self, rng: random.Random) -> int:
46
+ self._get_bounds()
47
+ return rng.randint(self.min, self.max)
48
+
49
+
50
+ class UniformDistribution(Distribution):
51
+ """A uniform distribution between finite bounds."""
52
+
53
+ def __init__(self, min: float, max: float) -> None:
54
+ self.min = min
55
+ self.max = max
56
+
57
+ def _get_bounds(self) -> tuple[float, float]:
58
+ if not math.isfinite(self.min) or not math.isfinite(self.max):
59
+ raise ValueError('`min` and `max` must be finite')
60
+ if self.min > self.max:
61
+ raise ValueError('`min` must not exceed `max`')
62
+
63
+ return self.min, self.max
64
+
65
+ def sample(self, rng: random.Random) -> float:
66
+ self._get_bounds()
67
+ return rng.uniform(self.min, self.max)
68
+
69
+
70
+ class LogUniformDistribution(UniformDistribution):
71
+ """A log-uniform distribution between positive finite bounds."""
72
+
73
+ def _get_bounds(self) -> tuple[float, float]:
74
+ bounds = super()._get_bounds()
75
+ if self.min <= 0.0:
76
+ raise ValueError('`min` must be positive')
77
+
78
+ return bounds
79
+
80
+ def sample(self, rng: random.Random) -> float:
81
+ self._get_bounds()
82
+ return math.exp(rng.uniform(math.log(self.min), math.log(self.max)))
83
+
84
+
85
+ _DistributionT = TypeVar('_DistributionT', bound=Distribution)
86
+
87
+
88
+ class OptionalDistribution(Distribution, Generic[_DistributionT]):
89
+ """A mixture of a given distribution (probability p) and zero."""
90
+
91
+ def __init__(self, distribution: _DistributionT, p: float = 0.5) -> None:
92
+ self.distribution = distribution
93
+ self.p = p
94
+
95
+ def _get_bounds(self) -> tuple[float, float]:
96
+ if not math.isfinite(self.p) or self.p < 0.0 or self.p > 1.0:
97
+ raise ValueError('`p` must be finite and between 0 and 1, inclusive')
98
+
99
+ min_value, max_value = self.distribution._get_bounds()
100
+ return min(0.0, min_value), max(0.0, max_value)
101
+
102
+ def sample(self, rng: random.Random) -> float:
103
+ self._get_bounds()
104
+ return self.distribution.sample(rng) if rng.random() < self.p else 0.0
105
+
106
+
107
+ _HyperparameterT = TypeVar('_HyperparameterT', int, float)
108
+
109
+
110
+ class HyperparameterBounds(TypedDict, Generic[_HyperparameterT]):
111
+ """The lower and upper bounds for sampling a hyperparameter."""
112
+
113
+ min: _HyperparameterT
114
+ max: _HyperparameterT
115
+
116
+
117
+ class OptionalHyperparameterBounds(HyperparameterBounds[float]):
118
+ """Hyperparameter sampling bounds for an optimal hyperparameter."""
119
+
120
+ optional: bool
121
+
122
+
123
+ class PerModelHyperparameters(TypedDict, total=False):
124
+ """Hyperparameters for one model, overriding the shared hyperparameters."""
125
+
126
+ n_blocks: int
127
+ d_block: int
128
+ dropout: float
129
+ numerical_embeddings_d: int
130
+ numerical_embeddings_init_scale: float
131
+ adamw_lr: float
132
+ adamw_beta1: float
133
+ adamw_beta2: float
134
+ adamw_eps: float
135
+ muon_lr: float
136
+ muon_momentum: float
137
+ weight_decay: float
138
+ max_epochs: int | None
139
+
140
+
141
+ def validate_hyperparameter(name: str, value: float | Distribution) -> None:
142
+ if not isinstance(value, Distribution):
143
+ if not math.isfinite(value):
144
+ raise ValueError(f'`{name}` must be finite')
145
+
146
+ if name in ('numerical_embeddings_init_scale', 'adamw_lr', 'muon_lr'):
147
+ if value <= 0.0:
148
+ raise ValueError(f'`{name}` must be positive')
149
+ elif name == 'dropout' and value >= 1.0:
150
+ raise ValueError('`dropout` must be less than 1')
151
+
152
+ return
153
+
154
+ min_value, max_value = value._get_bounds()
155
+ if not math.isfinite(min_value) or not math.isfinite(max_value):
156
+ raise ValueError(f'The distribution bounds for `{name}` must be finite')
157
+
158
+ if name in ('dropout', 'adamw_beta1', 'adamw_beta2', 'muon_momentum'):
159
+ if min_value < 0.0 or min_value > max_value or max_value >= 1.0:
160
+ raise ValueError(
161
+ f'The distribution bounds for `{name}` must be ordered and lie in [0,'
162
+ ' 1)'
163
+ )
164
+ elif name == 'weight_decay':
165
+ if min_value < 0.0:
166
+ raise ValueError(
167
+ 'The lower distribution bound for `weight_decay` must be non-negative'
168
+ )
169
+ else:
170
+ if min_value <= 0:
171
+ raise ValueError(
172
+ f'The lower distribution bound for `{name}` must be positive'
173
+ )
174
+
175
+
176
+ # >>> Evaluation
177
+
178
+
179
+ def _accuracy_pack(*, y_true: Tensor, y_pred: Tensor) -> Tensor:
180
+ return metrics.accuracy_pack(y_true=y_true, y_pred=y_pred.argmax(dim=-1))
181
+
182
+
183
+ def _log_loss_pack(*, y_true: Tensor, y_pred: Tensor) -> Tensor:
184
+ return metrics.multiclass_cross_entropy_pack(y_true=y_true, y_proba=y_pred)
185
+
186
+
187
+ def _roc_auc_pack(*, y_true: Tensor, y_pred: Tensor) -> Tensor:
188
+ return metrics.binary_roc_auc_pack(y_true=y_true, y_score=y_pred[..., 1])
189
+
190
+
191
+ def _resolve_metric(
192
+ metric: str, d_out: int
193
+ ) -> tuple[tabpackage.MetricFn, tabpackage.MetricDirection]:
194
+ allowed_metrics = (
195
+ ('accuracy', 'log_loss', 'roc_auc') if d_out > 1 else ('mse', 'mae', 'r2')
196
+ )
197
+ if metric not in allowed_metrics:
198
+ raise ValueError(f'Unsupported metric for this task: {metric!r}')
199
+ if metric == 'roc_auc' and d_out != 2:
200
+ raise ValueError('ROC AUC requires binary classification')
201
+
202
+ metric_fn = {
203
+ 'accuracy': _accuracy_pack,
204
+ 'log_loss': _log_loss_pack,
205
+ 'roc_auc': _roc_auc_pack,
206
+ 'mse': metrics.mse_pack,
207
+ 'mae': metrics.mae_pack,
208
+ 'r2': metrics.r2_pack,
209
+ }[metric]
210
+ metric_direction = (
211
+ tabpackage.MetricDirection.MAXIMIZE
212
+ if metric in ('accuracy', 'roc_auc', 'r2')
213
+ else tabpackage.MetricDirection.MINIMIZE
214
+ )
215
+ return metric_fn, metric_direction
216
+
217
+
218
+ def compute_scalar_metric(
219
+ metric_fn: tabpackage.MetricFn, *, y_true: Tensor, y_pred: Tensor
220
+ ) -> Tensor:
221
+ return metric_fn(y_true=y_true, y_pred=y_pred.unsqueeze(0))[0]
222
+
223
+
224
+ def _validate_metric_data(
225
+ metric: str, dataset: DatasetName, labels: dict[DatasetName, np.ndarray]
226
+ ) -> None:
227
+ if dataset not in labels:
228
+ raise ValueError(f'Unknown dataset: {dataset!r}')
229
+
230
+ if metric == 'roc_auc' and len(np.unique(labels[dataset])) != 2:
231
+ raise ValueError(f'ROC AUC requires both classes in dataset {dataset!r}')
232
+ elif metric == 'r2' and len(labels[dataset]) < 2:
233
+ raise ValueError(f'R2 requires at least two rows in dataset {dataset!r}')
234
+
235
+
236
+ def resolve_criterion(
237
+ dataset: DatasetName | None,
238
+ metric: str | None,
239
+ *,
240
+ default_dataset: DatasetName | None,
241
+ default_metric: str | None,
242
+ d_out: int,
243
+ ) -> tabpackage.EvalCriterionConfig | None:
244
+ metric = default_metric if metric is None else metric
245
+ if metric is None or metric == 'none':
246
+ return None
247
+
248
+ dataset = default_dataset if dataset is None else dataset
249
+ if dataset is None:
250
+ raise ValueError('An enabled criterion requires an explicit dataset')
251
+
252
+ _, metric_direction = _resolve_metric(metric, d_out)
253
+ return tabpackage.EvalCriterionConfig(
254
+ dataset=dataset, metric=metric, metric_direction=metric_direction
255
+ )
256
+
257
+
258
+ def resolve_eval_metrics(
259
+ dataset_names: list[DatasetName] | None,
260
+ metric_names: list[MetricName] | None,
261
+ *,
262
+ eval_criterion: tabpackage.EvalCriterionConfig | None,
263
+ d_out: int,
264
+ labels: dict[DatasetName, np.ndarray],
265
+ ) -> dict[DatasetName, dict[str, tabpackage.MetricFn]]:
266
+ if eval_criterion is None:
267
+ if (dataset_names is None) != (metric_names is None):
268
+ raise ValueError(
269
+ 'Reporting without a criterion requires both datasets and metrics'
270
+ )
271
+
272
+ dataset_names = [] if dataset_names is None else dataset_names
273
+ resolved_metric_names = [] if metric_names is None else metric_names
274
+ else:
275
+ dataset_names = (
276
+ [eval_criterion.dataset] if dataset_names is None else dataset_names
277
+ )
278
+ resolved_metric_names = (
279
+ [eval_criterion.metric] if metric_names is None else metric_names
280
+ )
281
+
282
+ for dataset in dataset_names:
283
+ if dataset not in labels:
284
+ raise ValueError(f'Unknown dataset: {dataset!r}')
285
+
286
+ metric_fns = {
287
+ name: _resolve_metric(name, d_out)[0] for name in resolved_metric_names
288
+ }
289
+ eval_metrics = {
290
+ dataset: metric_fns.copy() for dataset in dataset_names if metric_fns
291
+ }
292
+ if eval_criterion is not None:
293
+ eval_metrics.setdefault(eval_criterion.dataset, {})[eval_criterion.metric] = (
294
+ _resolve_metric(eval_criterion.metric, d_out)[0]
295
+ )
296
+
297
+ for dataset, dataset_metrics in eval_metrics.items():
298
+ for metric in dataset_metrics:
299
+ _validate_metric_data(metric, dataset, labels)
300
+
301
+ return eval_metrics
302
+
303
+
304
+ # >>> Devices and precision
305
+
306
+
307
+ def resolve_device(device: str | torch.device | Literal['auto']) -> torch.device:
308
+ if device == 'auto':
309
+ device = (
310
+ 'cuda'
311
+ if torch.cuda.is_available()
312
+ else 'mps'
313
+ if torch.backends.mps.is_available()
314
+ else 'cpu'
315
+ )
316
+
317
+ device = torch.device(device)
318
+ if device.type == 'cuda':
319
+ if not torch.cuda.is_available():
320
+ raise RuntimeError('CUDA is not available')
321
+
322
+ index = torch.cuda.current_device() if device.index is None else device.index
323
+ if index >= torch.cuda.device_count():
324
+ raise ValueError('The CUDA device index is out of range')
325
+
326
+ return torch.device('cuda', index)
327
+ if device.type == 'mps':
328
+ if not torch.backends.mps.is_available():
329
+ raise RuntimeError('MPS is not available')
330
+ if device.index not in (None, 0):
331
+ raise ValueError('The MPS device index must be unspecified or zero')
332
+
333
+ return torch.device('mps')
334
+ if device.type != 'cpu':
335
+ raise ValueError('The device type is not supported')
336
+
337
+ return torch.device('cpu')
338
+
339
+
340
+ def resolve_autocast_dtype(dtype: Literal['bfloat16'] | None) -> torch.dtype | None:
341
+ return {None: None, 'bfloat16': torch.bfloat16}[dtype]
342
+
343
+
344
+ # >>> Ensembles
345
+
346
+
347
+ class _Ensemble(BaseEstimator):
348
+ online: bool
349
+ early_stopping_patience: int | None
350
+ algorithm_metric: MetricName | None
351
+ algorithm_dataset: DatasetName | None
352
+ algorithm_device: str | torch.device | Literal['auto']
353
+
354
+ def _make_algorithm_config(self) -> KWArgs:
355
+ raise NotImplementedError
356
+
357
+ def _make_config(
358
+ self,
359
+ *,
360
+ d_out: int,
361
+ eval_criterion: tabpackage.EvalCriterionConfig | None,
362
+ default_algorithm_dataset: DatasetName | None,
363
+ labels: dict[DatasetName, np.ndarray],
364
+ device: torch.device,
365
+ ) -> tabpackage.EnsembleConfig:
366
+ if self.algorithm_metric is None and eval_criterion is None:
367
+ raise ValueError(
368
+ 'Without the evaluation criterion, `algorithm_metric` must be specified'
369
+ )
370
+ if (
371
+ self.algorithm_dataset is None
372
+ and eval_criterion is None
373
+ and default_algorithm_dataset is None
374
+ ):
375
+ raise ValueError(
376
+ 'Without the evaluation criterion and with extra datasets passed as'
377
+ ' dictionaries, `algorithm_dataset` must be specified'
378
+ )
379
+
380
+ if self.algorithm_metric is None:
381
+ assert eval_criterion is not None
382
+ metric = eval_criterion.metric
383
+ else:
384
+ metric = self.algorithm_metric
385
+
386
+ dataset = (
387
+ self.algorithm_dataset
388
+ if self.algorithm_dataset is not None
389
+ else (
390
+ eval_criterion.dataset
391
+ if eval_criterion is not None
392
+ else default_algorithm_dataset
393
+ )
394
+ )
395
+ assert dataset is not None
396
+
397
+ _validate_metric_data(metric, dataset, labels)
398
+ metric_fn, metric_direction = _resolve_metric(metric, d_out)
399
+
400
+ return tabpackage.EnsembleConfig(
401
+ algorithm=self._make_algorithm_config(),
402
+ online=self.online,
403
+ prediction_transform=(
404
+ partial(torch.clamp_max, max=1.0) if d_out > 1 else None
405
+ ),
406
+ early_stopping_patience=(
407
+ None if eval_criterion is None else self.early_stopping_patience
408
+ ),
409
+ algorithm_metric=metric_fn,
410
+ algorithm_metric_direction=metric_direction,
411
+ algorithm_dataset=dataset,
412
+ algorithm_device=(
413
+ device
414
+ if self.algorithm_device == 'auto'
415
+ else resolve_device(self.algorithm_device)
416
+ ),
417
+ )
418
+
419
+
420
+ class GreedyEnsemble(_Ensemble):
421
+ """Configuration of a greedy ensemble."""
422
+
423
+ def __init__(
424
+ self,
425
+ *,
426
+ init_top_k: int = 1,
427
+ max_ensemble_size: int | None = 32,
428
+ with_replacement: bool = False,
429
+ online: bool = True,
430
+ early_stopping_patience: int | None = 32,
431
+ algorithm_metric: MetricName | None = None,
432
+ algorithm_dataset: DatasetName | None = None,
433
+ algorithm_device: str | torch.device | Literal['auto'] = 'auto',
434
+ ) -> None:
435
+ """Greedy ensemble configuration.
436
+
437
+ Args:
438
+ init_top_k: the number of top-performing models to start with.
439
+ max_ensemble_size: the maximum number of ensemble members.
440
+ with_replacement: whether a model can be selected more than once.
441
+ online: whether to consider models that are still training.
442
+ early_stopping_patience: the number of updates without improvement
443
+ tolerated before finalizing the ensemble.
444
+ algorithm_metric: the metric used to select ensemble members.
445
+ algorithm_dataset: the dataset used to select ensemble members.
446
+ algorithm_device: the device used to build the ensemble.
447
+ """
448
+ self.init_top_k = init_top_k
449
+ self.max_ensemble_size = max_ensemble_size
450
+ self.with_replacement = with_replacement
451
+ self.online = online
452
+ self.early_stopping_patience = early_stopping_patience
453
+ self.algorithm_metric = algorithm_metric
454
+ self.algorithm_dataset = algorithm_dataset
455
+ self.algorithm_device = algorithm_device
456
+
457
+ def _make_algorithm_config(self) -> KWArgs:
458
+ return {
459
+ 'type': ensemble.greedy_ensemble,
460
+ 'init_top_k': self.init_top_k,
461
+ 'max_ensemble_size': self.max_ensemble_size,
462
+ 'with_replacement': self.with_replacement,
463
+ }
464
+
465
+
466
+ class TopKEnsemble(_Ensemble):
467
+ """Configuration of an equally weighted ensemble of the top-k performers."""
468
+
469
+ def __init__(
470
+ self,
471
+ *,
472
+ k: int,
473
+ online: bool = True,
474
+ early_stopping_patience: int | None = 32,
475
+ algorithm_metric: MetricName | None = None,
476
+ algorithm_dataset: DatasetName | None = None,
477
+ algorithm_device: str | torch.device | Literal['auto'] = 'auto',
478
+ ) -> None:
479
+ """Top-K ensemble configuration.
480
+
481
+ Args:
482
+ k: the number of models to select.
483
+ online: whether to consider models that are still training.
484
+ early_stopping_patience: the number of updates without improvement
485
+ tolerated before finalizing the ensemble.
486
+ algorithm_metric: the metric used to rank models.
487
+ algorithm_dataset: the dataset used to rank models.
488
+ algorithm_device: the device used to build the ensemble.
489
+ """
490
+ self.k = k
491
+ self.online = online
492
+ self.early_stopping_patience = early_stopping_patience
493
+ self.algorithm_metric = algorithm_metric
494
+ self.algorithm_dataset = algorithm_dataset
495
+ self.algorithm_device = algorithm_device
496
+
497
+ def _make_algorithm_config(self) -> KWArgs:
498
+ return {'type': ensemble.topk_ensemble, 'k': self.k}
499
+
500
+
501
+ class Ensembles(_BaseComposition):
502
+ """A collection of named ensemble configurations."""
503
+
504
+ def __init__(
505
+ self, members: dict[str, GreedyEnsemble | TopKEnsemble | KWArgs]
506
+ ) -> None:
507
+ """Named ensemble configurations.
508
+
509
+ Each ensemble configuration is accessible through `get_params` and `set_params`
510
+ through names such as `my_ensemble__early_stopping_patience`.
511
+ """
512
+ self.members = members
513
+
514
+ @property
515
+ def _ensemble__items(
516
+ self,
517
+ ) -> list[tuple[str, GreedyEnsemble | TopKEnsemble | KWArgs]]:
518
+ return list(self.members.items())
519
+
520
+ @_ensemble__items.setter
521
+ def _ensemble__items(
522
+ self, items: list[tuple[str, GreedyEnsemble | TopKEnsemble | KWArgs]]
523
+ ) -> None:
524
+ self.members = dict(items)
525
+
526
+ def get_params(self, deep: bool = True) -> KWArgs:
527
+ return self._get_params('_ensemble__items', deep=deep)
528
+
529
+ def set_params(self, **params) -> Self:
530
+ if 'members' in params:
531
+ self.members = params.pop('members')
532
+ return self._set_params('_ensemble__items', **params)
533
+
534
+ def make_config(
535
+ self,
536
+ *,
537
+ d_out: int,
538
+ eval_criterion: tabpackage.EvalCriterionConfig | None,
539
+ default_algorithm_dataset: DatasetName | None,
540
+ labels: dict[DatasetName, np.ndarray],
541
+ device: torch.device,
542
+ ) -> dict[str, tabpackage.EnsembleConfig]:
543
+ self._validate_names(list(self.members))
544
+
545
+ configs = {}
546
+ for name, member in self.members.items():
547
+ if isinstance(member, dict):
548
+ kwargs = member.copy()
549
+ ensemble_type = {'greedy': GreedyEnsemble, 'topk': TopKEnsemble}[
550
+ kwargs.pop('type', 'greedy')
551
+ ]
552
+ member = ensemble_type(**kwargs)
553
+
554
+ configs[name] = member._make_config(
555
+ d_out=d_out,
556
+ eval_criterion=eval_criterion,
557
+ default_algorithm_dataset=default_algorithm_dataset,
558
+ labels=labels,
559
+ device=device,
560
+ )
561
+ return configs