structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/_warnings.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import warnings
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
|
|
8
|
+
_warned_once: set[str] = set()
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def warn_once(
|
|
12
|
+
key: str,
|
|
13
|
+
message: str,
|
|
14
|
+
*,
|
|
15
|
+
category: type[Warning] = RuntimeWarning,
|
|
16
|
+
stacklevel: int = 2,
|
|
17
|
+
) -> None:
|
|
18
|
+
r"""Issue a warning once per process."""
|
|
19
|
+
if not torch.is_warn_always_enabled() and key in _warned_once:
|
|
20
|
+
return
|
|
21
|
+
|
|
22
|
+
if not torch.is_warn_always_enabled():
|
|
23
|
+
message += (
|
|
24
|
+
" This warning will be suppressed for the remainder of this "
|
|
25
|
+
"process."
|
|
26
|
+
)
|
|
27
|
+
_warned_once.add(key)
|
|
28
|
+
|
|
29
|
+
warnings.warn(message, category=category, stacklevel=stacklevel)
|
sdm/cache.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from collections.abc import (
|
|
5
|
+
Callable,
|
|
6
|
+
Hashable,
|
|
7
|
+
Iterable,
|
|
8
|
+
Iterator,
|
|
9
|
+
Mapping,
|
|
10
|
+
MutableMapping,
|
|
11
|
+
)
|
|
12
|
+
from enum import StrEnum
|
|
13
|
+
from typing import NamedTuple, Self
|
|
14
|
+
|
|
15
|
+
from torch import Tensor
|
|
16
|
+
|
|
17
|
+
from sdm.tensor.mixin import DeviceMixin
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class _KVCacheEntry(NamedTuple):
|
|
21
|
+
key: Tensor
|
|
22
|
+
value: Tensor
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class KVCacheEntry(_KVCacheEntry, DeviceMixin):
|
|
26
|
+
r"""Cached key/value projections for a single transformer block.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
key: Cached key projection tensor.
|
|
30
|
+
value: Cached value projection tensor.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def _tensors(self) -> Iterator[Tensor]:
|
|
34
|
+
yield self.key
|
|
35
|
+
yield self.value
|
|
36
|
+
|
|
37
|
+
def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self:
|
|
38
|
+
return self.__class__(key=fn(self.key), value=fn(self.value))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class Cache(MutableMapping[Hashable, object], DeviceMixin):
|
|
42
|
+
r"""A mutable mapping of model cache values."""
|
|
43
|
+
|
|
44
|
+
class Mode(StrEnum):
|
|
45
|
+
r"""The operating mode of a :class:`Cache`.
|
|
46
|
+
|
|
47
|
+
A cache alternates between two phases: (1) recording key/value
|
|
48
|
+
projections from a fit pass, and (2) replaying them across subsequent
|
|
49
|
+
predict passes.
|
|
50
|
+
Possible values are:
|
|
51
|
+
|
|
52
|
+
Attributes:
|
|
53
|
+
record: Allows recording of data.
|
|
54
|
+
replay: Allows replaying of data.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
record = "record"
|
|
58
|
+
replay = "replay"
|
|
59
|
+
|
|
60
|
+
def __init__(
|
|
61
|
+
self,
|
|
62
|
+
*args: Mapping[Hashable, object] | Iterable[tuple[Hashable, object]],
|
|
63
|
+
**kwargs: object,
|
|
64
|
+
) -> None:
|
|
65
|
+
self._mode = Cache.Mode.record
|
|
66
|
+
self._items: dict[Hashable, object] = dict(*args, **kwargs)
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def is_recording(self) -> bool:
|
|
70
|
+
r"""Whether the cache is in recording mode."""
|
|
71
|
+
return self._mode == Cache.Mode.record
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def is_replaying(self) -> bool:
|
|
75
|
+
r"""Whether the cache is in replaying mode."""
|
|
76
|
+
return self._mode == Cache.Mode.replay
|
|
77
|
+
|
|
78
|
+
def size(self) -> int:
|
|
79
|
+
r"""The size in bytes of tensor data stored in this cache."""
|
|
80
|
+
return sum(
|
|
81
|
+
tensor.numel() * tensor.element_size()
|
|
82
|
+
for tensor in self._tensors()
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
def freeze(self) -> Self:
|
|
86
|
+
r"""Freeze the cache to replay mode."""
|
|
87
|
+
|
|
88
|
+
def _freeze(value: object) -> None:
|
|
89
|
+
if isinstance(value, Cache):
|
|
90
|
+
value._mode = Cache.Mode.replay
|
|
91
|
+
for item in value.values():
|
|
92
|
+
_freeze(item)
|
|
93
|
+
elif isinstance(value, (list, tuple)):
|
|
94
|
+
for item in value:
|
|
95
|
+
_freeze(item)
|
|
96
|
+
elif isinstance(value, Mapping):
|
|
97
|
+
for item in value.values():
|
|
98
|
+
_freeze(item)
|
|
99
|
+
|
|
100
|
+
_freeze(self)
|
|
101
|
+
return self
|
|
102
|
+
|
|
103
|
+
def __setitem__(self, key: Hashable, value: object) -> None:
|
|
104
|
+
if not self.is_recording:
|
|
105
|
+
raise RuntimeError(
|
|
106
|
+
"'__setitem__' requires the cache to be in 'record' mode"
|
|
107
|
+
)
|
|
108
|
+
self._items[key] = value
|
|
109
|
+
|
|
110
|
+
def __delitem__(self, key: Hashable) -> None:
|
|
111
|
+
if not self.is_recording:
|
|
112
|
+
raise RuntimeError(
|
|
113
|
+
"'__delitem__' requires the cache to be in 'record' mode"
|
|
114
|
+
)
|
|
115
|
+
del self._items[key]
|
|
116
|
+
|
|
117
|
+
def __getitem__(self, key: Hashable) -> object:
|
|
118
|
+
return self._items[key]
|
|
119
|
+
|
|
120
|
+
def __iter__(self) -> Iterator[Hashable]:
|
|
121
|
+
return iter(self._items)
|
|
122
|
+
|
|
123
|
+
def __len__(self) -> int:
|
|
124
|
+
return len(self._items)
|
|
125
|
+
|
|
126
|
+
def __repr__(self) -> str:
|
|
127
|
+
return repr(self._items)
|
|
128
|
+
|
|
129
|
+
def _tensors(self) -> Iterator[Tensor]:
|
|
130
|
+
def _iter_tensors(value: object) -> Iterator[Tensor]:
|
|
131
|
+
if isinstance(value, Tensor):
|
|
132
|
+
yield value
|
|
133
|
+
elif isinstance(value, DeviceMixin):
|
|
134
|
+
yield from value._tensors()
|
|
135
|
+
elif isinstance(value, list | tuple):
|
|
136
|
+
for item in value:
|
|
137
|
+
yield from _iter_tensors(item)
|
|
138
|
+
elif isinstance(value, Mapping):
|
|
139
|
+
for item in value.values():
|
|
140
|
+
yield from _iter_tensors(item)
|
|
141
|
+
|
|
142
|
+
for value in self.values():
|
|
143
|
+
yield from _iter_tensors(value)
|
|
144
|
+
|
|
145
|
+
def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self:
|
|
146
|
+
def _apply(value: object) -> object:
|
|
147
|
+
if isinstance(value, Tensor):
|
|
148
|
+
return fn(value)
|
|
149
|
+
if isinstance(value, DeviceMixin):
|
|
150
|
+
return value._apply_tensor(fn)
|
|
151
|
+
if isinstance(value, list):
|
|
152
|
+
return [_apply(item) for item in value]
|
|
153
|
+
if isinstance(value, tuple):
|
|
154
|
+
return tuple(_apply(item) for item in value)
|
|
155
|
+
if isinstance(value, Mapping):
|
|
156
|
+
return {key: _apply(item) for key, item in value.items()}
|
|
157
|
+
return value
|
|
158
|
+
|
|
159
|
+
out = self.__class__(
|
|
160
|
+
{key: _apply(value) for key, value in self.items()}
|
|
161
|
+
)
|
|
162
|
+
out._mode = self._mode
|
|
163
|
+
return out
|
sdm/ensemble.py
ADDED
|
@@ -0,0 +1,501 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import abc
|
|
7
|
+
import copy
|
|
8
|
+
from collections.abc import Callable, Iterable, Iterator, Sequence
|
|
9
|
+
from typing import Generic, Self, TypeVar, cast, overload
|
|
10
|
+
|
|
11
|
+
import torch
|
|
12
|
+
from torch import Tensor
|
|
13
|
+
|
|
14
|
+
from sdm import Stype, StypeLike
|
|
15
|
+
from sdm.tensor import TableTensor
|
|
16
|
+
from sdm.tensor.mixin import DeviceMixin
|
|
17
|
+
|
|
18
|
+
T = TypeVar("T")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class EnsembleData(abc.ABC, Generic[T]):
|
|
22
|
+
"""Represent an ordered collection of ensemble member values.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
groups: Values containing the ensemble members.
|
|
26
|
+
locations: Group index and position for each ensemble member.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
_groups: tuple[T, ...]
|
|
30
|
+
_locations: tuple[tuple[int, int], ...]
|
|
31
|
+
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
groups: Sequence[T],
|
|
35
|
+
locations: Sequence[tuple[int, int]],
|
|
36
|
+
) -> None:
|
|
37
|
+
self._groups = tuple(groups)
|
|
38
|
+
self._locations = tuple(locations)
|
|
39
|
+
|
|
40
|
+
def __len__(self) -> int:
|
|
41
|
+
return len(self._locations)
|
|
42
|
+
|
|
43
|
+
@overload
|
|
44
|
+
def __getitem__(self, index: int) -> T: ...
|
|
45
|
+
|
|
46
|
+
@overload
|
|
47
|
+
def __getitem__(self, index: slice | Sequence[int]) -> Self: ...
|
|
48
|
+
|
|
49
|
+
def __getitem__(self, index: int | slice | Sequence[int]) -> T | Self:
|
|
50
|
+
if isinstance(index, int):
|
|
51
|
+
group_id, position = self._locations[index]
|
|
52
|
+
return self._select_member(self._groups[group_id], position)
|
|
53
|
+
if isinstance(index, slice):
|
|
54
|
+
index = range(*index.indices(len(self)))
|
|
55
|
+
return self._select_members(index)
|
|
56
|
+
|
|
57
|
+
def _iter_groups(self) -> Iterator[T]:
|
|
58
|
+
return iter(self._groups)
|
|
59
|
+
|
|
60
|
+
@staticmethod
|
|
61
|
+
@abc.abstractmethod
|
|
62
|
+
def _select_member(group: T, position: int) -> T:
|
|
63
|
+
pass
|
|
64
|
+
|
|
65
|
+
@abc.abstractmethod
|
|
66
|
+
def _select_members(self, member_ids: Sequence[int]) -> Self:
|
|
67
|
+
pass
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class EnsembleTable(DeviceMixin, EnsembleData[TableTensor]):
|
|
71
|
+
"""Store and group input :class:`~sdm.tensor.TableTensor` as an ensemble.
|
|
72
|
+
|
|
73
|
+
Each ensemble member is associated with one
|
|
74
|
+
:class:`~sdm.tensor.TableTensor`.
|
|
75
|
+
Shared tables are stored only once. Compatible tables are stacked along a
|
|
76
|
+
leading dimension and form a group.
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
groups: Sequence of :class:`~sdm.tensor.TableTensor`, optionally
|
|
80
|
+
stacked along their leading dimension.
|
|
81
|
+
locations: ``(group, batch)`` location of each ensemble member.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
def from_table(
|
|
86
|
+
cls,
|
|
87
|
+
table: TableTensor,
|
|
88
|
+
*,
|
|
89
|
+
num_members: int,
|
|
90
|
+
) -> Self:
|
|
91
|
+
"""Create an ensemble, sharing one table across all members.
|
|
92
|
+
|
|
93
|
+
Args:
|
|
94
|
+
table: Table used across members.
|
|
95
|
+
num_members: Number of ensemble members.
|
|
96
|
+
|
|
97
|
+
Returns:
|
|
98
|
+
An :class:`~sdm.EnsembleTable` with one group.
|
|
99
|
+
"""
|
|
100
|
+
return cls(
|
|
101
|
+
groups=(cast(TableTensor, table.unsqueeze(0)),),
|
|
102
|
+
locations=((0, 0),) * num_members,
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
@classmethod
|
|
106
|
+
def from_tables(
|
|
107
|
+
cls,
|
|
108
|
+
tables: Sequence[TableTensor],
|
|
109
|
+
member_table_ids: Sequence[int],
|
|
110
|
+
) -> Self:
|
|
111
|
+
"""Create an ensemble table from tables and their member assignments.
|
|
112
|
+
|
|
113
|
+
Each referenced input table forms a separate group.
|
|
114
|
+
``member_table_ids`` contains one table index per member. For
|
|
115
|
+
example, ``(0, 1, 0)`` assigns the first table to members 0 and 2 and
|
|
116
|
+
the second table to member 1.
|
|
117
|
+
|
|
118
|
+
Args:
|
|
119
|
+
tables: Tables available to the ensemble members.
|
|
120
|
+
member_table_ids: Index into ``tables`` for each ensemble member.
|
|
121
|
+
|
|
122
|
+
Returns:
|
|
123
|
+
An ensemble table preserving member order.
|
|
124
|
+
"""
|
|
125
|
+
referenced_table_ids = set(member_table_ids)
|
|
126
|
+
table_ids = tuple(
|
|
127
|
+
table_id
|
|
128
|
+
for table_id in range(len(tables))
|
|
129
|
+
if table_id in referenced_table_ids
|
|
130
|
+
)
|
|
131
|
+
group_ids = {
|
|
132
|
+
table_id: group_id for group_id, table_id in enumerate(table_ids)
|
|
133
|
+
}
|
|
134
|
+
return cls(
|
|
135
|
+
groups=tuple(
|
|
136
|
+
cast(TableTensor, tables[table_id].unsqueeze(0))
|
|
137
|
+
for table_id in table_ids
|
|
138
|
+
),
|
|
139
|
+
locations=tuple(
|
|
140
|
+
(group_ids[table_id], 0) for table_id in member_table_ids
|
|
141
|
+
),
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
@staticmethod
|
|
145
|
+
def _pack_tables(
|
|
146
|
+
tables: Sequence[TableTensor],
|
|
147
|
+
) -> tuple[tuple[TableTensor, ...], tuple[tuple[int, int], ...]]:
|
|
148
|
+
compatible_groups: dict[tuple[object, ...], list[int]] = {}
|
|
149
|
+
for index, table in enumerate(tables):
|
|
150
|
+
# Shape, schema, block layout, device, and categorical vocabularies
|
|
151
|
+
# must match for torch.stack to preserve member semantics.
|
|
152
|
+
compatibility_key = (
|
|
153
|
+
tuple(
|
|
154
|
+
(stype, columns)
|
|
155
|
+
for stype, columns in table.columns.items()
|
|
156
|
+
),
|
|
157
|
+
tuple(
|
|
158
|
+
(
|
|
159
|
+
stype,
|
|
160
|
+
type(block),
|
|
161
|
+
block.size(),
|
|
162
|
+
block.layout,
|
|
163
|
+
block.dtype,
|
|
164
|
+
)
|
|
165
|
+
for stype, block in table.items()
|
|
166
|
+
),
|
|
167
|
+
table.device,
|
|
168
|
+
tuple(
|
|
169
|
+
id(category) for category in table.categorical.categories
|
|
170
|
+
),
|
|
171
|
+
)
|
|
172
|
+
compatible_groups.setdefault(compatibility_key, []).append(index)
|
|
173
|
+
|
|
174
|
+
groups: list[TableTensor] = []
|
|
175
|
+
locations = [(-1, -1)] * len(tables)
|
|
176
|
+
for indices in compatible_groups.values():
|
|
177
|
+
group_index = len(groups)
|
|
178
|
+
groups.append(
|
|
179
|
+
cast(TableTensor, tables[indices[0]].unsqueeze(0))
|
|
180
|
+
if len(indices) == 1
|
|
181
|
+
else cast(
|
|
182
|
+
TableTensor,
|
|
183
|
+
torch.stack(
|
|
184
|
+
tensors=[tables[index] for index in indices],
|
|
185
|
+
dim=0,
|
|
186
|
+
),
|
|
187
|
+
)
|
|
188
|
+
)
|
|
189
|
+
for position, index in enumerate(indices):
|
|
190
|
+
locations[index] = (group_index, position)
|
|
191
|
+
|
|
192
|
+
return tuple(groups), tuple(locations)
|
|
193
|
+
|
|
194
|
+
def _select_members(self, member_ids: Sequence[int]) -> Self:
|
|
195
|
+
member_ids = tuple(member_ids)
|
|
196
|
+
if member_ids == tuple(range(len(self))):
|
|
197
|
+
return self
|
|
198
|
+
|
|
199
|
+
locations = tuple(
|
|
200
|
+
self._locations[member_id] for member_id in member_ids
|
|
201
|
+
)
|
|
202
|
+
positions_by_group: dict[int, dict[int, int]] = {}
|
|
203
|
+
for group_id, position in locations:
|
|
204
|
+
positions = positions_by_group.setdefault(group_id, {})
|
|
205
|
+
positions.setdefault(position, len(positions))
|
|
206
|
+
|
|
207
|
+
groups: list[TableTensor] = []
|
|
208
|
+
group_ids: dict[int, int] = {}
|
|
209
|
+
for new_group_id, (group_id, positions) in enumerate(
|
|
210
|
+
positions_by_group.items()
|
|
211
|
+
):
|
|
212
|
+
group = self._groups[group_id]
|
|
213
|
+
selected_positions = tuple(positions)
|
|
214
|
+
group_ids[group_id] = new_group_id
|
|
215
|
+
start = selected_positions[0]
|
|
216
|
+
step = (
|
|
217
|
+
selected_positions[1] - start
|
|
218
|
+
if len(selected_positions) > 1
|
|
219
|
+
else 1
|
|
220
|
+
)
|
|
221
|
+
stop = start + step * len(selected_positions)
|
|
222
|
+
if selected_positions == tuple(range(group.size(0))):
|
|
223
|
+
groups.append(group)
|
|
224
|
+
elif step > 0 and selected_positions == tuple(
|
|
225
|
+
range(start, stop, step)
|
|
226
|
+
):
|
|
227
|
+
# Evenly spaced members are selected as a view, not a copy.
|
|
228
|
+
groups.append(group[start:stop:step])
|
|
229
|
+
else:
|
|
230
|
+
groups.append(
|
|
231
|
+
cast(
|
|
232
|
+
TableTensor,
|
|
233
|
+
torch.stack(
|
|
234
|
+
[
|
|
235
|
+
group[position]
|
|
236
|
+
for position in selected_positions
|
|
237
|
+
],
|
|
238
|
+
dim=0,
|
|
239
|
+
),
|
|
240
|
+
)
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
ensemble = copy.copy(self)
|
|
244
|
+
ensemble._groups = tuple(groups)
|
|
245
|
+
ensemble._locations = tuple(
|
|
246
|
+
(group_ids[group_id], positions_by_group[group_id][position])
|
|
247
|
+
for group_id, position in locations
|
|
248
|
+
)
|
|
249
|
+
return ensemble
|
|
250
|
+
|
|
251
|
+
def gather_members(
|
|
252
|
+
self,
|
|
253
|
+
tables: Sequence[Self],
|
|
254
|
+
member_ids: Sequence[int],
|
|
255
|
+
) -> Self:
|
|
256
|
+
"""Gather members from multiple ensemble tables.
|
|
257
|
+
|
|
258
|
+
``tables[i][member_ids[i]]`` supplies output member ``i``.
|
|
259
|
+
|
|
260
|
+
Args:
|
|
261
|
+
tables: Source ensemble table for each output member.
|
|
262
|
+
member_ids: Logical source member position for each output member.
|
|
263
|
+
|
|
264
|
+
Returns:
|
|
265
|
+
An ensemble table preserving current group boundaries.
|
|
266
|
+
"""
|
|
267
|
+
if len(tables) != len(member_ids):
|
|
268
|
+
raise ValueError("Expected one source member per ensemble table")
|
|
269
|
+
|
|
270
|
+
outputs: list[TableTensor] = []
|
|
271
|
+
output_id_by_source: dict[tuple[int, tuple[int, int]], int] = {}
|
|
272
|
+
member_table_ids = []
|
|
273
|
+
for table, member_id in zip(tables, member_ids, strict=True):
|
|
274
|
+
location = table._locations[member_id]
|
|
275
|
+
key = (id(table), location)
|
|
276
|
+
output_id = output_id_by_source.get(key)
|
|
277
|
+
if output_id is None:
|
|
278
|
+
output_id = len(outputs)
|
|
279
|
+
output_id_by_source[key] = output_id
|
|
280
|
+
outputs.append(table[member_id])
|
|
281
|
+
member_table_ids.append(output_id)
|
|
282
|
+
|
|
283
|
+
return self.replace_tables(
|
|
284
|
+
tables=outputs,
|
|
285
|
+
member_table_ids=member_table_ids,
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
@property
|
|
289
|
+
def num_groups(self) -> int:
|
|
290
|
+
"""Return the number of table groups."""
|
|
291
|
+
return len(self._groups)
|
|
292
|
+
|
|
293
|
+
def num_members_in_group(self, group_id: int) -> int:
|
|
294
|
+
"""Return the number of members of a group.
|
|
295
|
+
|
|
296
|
+
Args:
|
|
297
|
+
group_id: Zero-based group index.
|
|
298
|
+
"""
|
|
299
|
+
return sum(i == group_id for i, _ in self._locations)
|
|
300
|
+
|
|
301
|
+
@staticmethod
|
|
302
|
+
def _select_member(
|
|
303
|
+
group: TableTensor,
|
|
304
|
+
position: int,
|
|
305
|
+
) -> TableTensor:
|
|
306
|
+
return group[position]
|
|
307
|
+
|
|
308
|
+
def expanded_group(self, group_id: int) -> TableTensor:
|
|
309
|
+
"""Return the logical members assigned to one group.
|
|
310
|
+
|
|
311
|
+
Args:
|
|
312
|
+
group_id: Zero-based group index.
|
|
313
|
+
"""
|
|
314
|
+
group = self._groups[group_id]
|
|
315
|
+
positions = tuple(
|
|
316
|
+
position for i, position in self._locations if i == group_id
|
|
317
|
+
)
|
|
318
|
+
if positions == tuple(range(group.size(0))):
|
|
319
|
+
return group
|
|
320
|
+
if group.size(0) == 1:
|
|
321
|
+
return cast(
|
|
322
|
+
TableTensor,
|
|
323
|
+
group.expand(len(positions), *group.size()[1:]),
|
|
324
|
+
)
|
|
325
|
+
index = torch.tensor(positions, device=group.device)
|
|
326
|
+
return cast(TableTensor, group.index_select(0, index))
|
|
327
|
+
|
|
328
|
+
def _tensors(self) -> Iterator[Tensor]:
|
|
329
|
+
yield from self._groups
|
|
330
|
+
|
|
331
|
+
def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self:
|
|
332
|
+
return self.replace_groups(
|
|
333
|
+
[cast(TableTensor, fn(group)) for group in self._groups]
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
def select_stypes(
|
|
337
|
+
self,
|
|
338
|
+
stypes: StypeLike | Iterable[StypeLike],
|
|
339
|
+
) -> Self:
|
|
340
|
+
r"""Return an ensemble table containing only ``stypes`` columns.
|
|
341
|
+
|
|
342
|
+
Args:
|
|
343
|
+
stypes: The semantic type or semantic types to select.
|
|
344
|
+
|
|
345
|
+
Returns:
|
|
346
|
+
An ensemble table preserving its logical member assignment.
|
|
347
|
+
"""
|
|
348
|
+
if isinstance(stypes, (str, Stype)):
|
|
349
|
+
stypes = (stypes,)
|
|
350
|
+
stypes = tuple(Stype(stype) for stype in stypes)
|
|
351
|
+
|
|
352
|
+
if len(stypes) == 0:
|
|
353
|
+
return self.replace_groups(
|
|
354
|
+
[group.select_columns(()) for group in self._groups]
|
|
355
|
+
)
|
|
356
|
+
return self.replace_groups(
|
|
357
|
+
[group.select_stypes(stypes) for group in self._groups]
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
@classmethod
|
|
361
|
+
def concatenate_columns(cls, tables: Sequence[Self]) -> Self:
|
|
362
|
+
r"""Concatenate ensemble tables column-wise by logical member.
|
|
363
|
+
|
|
364
|
+
Args:
|
|
365
|
+
tables: Ensemble tables with the same number of logical members.
|
|
366
|
+
|
|
367
|
+
Returns:
|
|
368
|
+
An ensemble table preserving logical member order.
|
|
369
|
+
"""
|
|
370
|
+
if len(tables) == 0:
|
|
371
|
+
raise ValueError("Expected at least one ensemble table")
|
|
372
|
+
|
|
373
|
+
first = tables[0]
|
|
374
|
+
if any(len(table) != len(first) for table in tables[1:]):
|
|
375
|
+
raise ValueError(
|
|
376
|
+
"Cannot concatenate ensemble tables with different member "
|
|
377
|
+
"counts"
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
nonempty_tables = tuple(
|
|
381
|
+
table
|
|
382
|
+
for table in tables
|
|
383
|
+
if any(group.size(-1) > 0 for group in table._groups)
|
|
384
|
+
)
|
|
385
|
+
if len(nonempty_tables) > 0:
|
|
386
|
+
tables = nonempty_tables
|
|
387
|
+
first = tables[0]
|
|
388
|
+
if len(tables) == 1:
|
|
389
|
+
return first
|
|
390
|
+
|
|
391
|
+
if all(table._locations == first._locations for table in tables[1:]):
|
|
392
|
+
return first.replace_groups(
|
|
393
|
+
[
|
|
394
|
+
cast(TableTensor, torch.cat(groups, dim=-1))
|
|
395
|
+
for groups in zip(
|
|
396
|
+
*(table._groups for table in tables),
|
|
397
|
+
strict=True,
|
|
398
|
+
)
|
|
399
|
+
]
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
# TODO: Concatenate compatible groups directly and unpack logical
|
|
403
|
+
# members only when their layouts differ.
|
|
404
|
+
outputs: list[TableTensor] = []
|
|
405
|
+
output_id_by_locations: dict[tuple[tuple[int, int], ...], int] = {}
|
|
406
|
+
member_table_ids = []
|
|
407
|
+
for member_id in range(len(first)):
|
|
408
|
+
locations = tuple(table._locations[member_id] for table in tables)
|
|
409
|
+
output_id = output_id_by_locations.get(locations)
|
|
410
|
+
if output_id is None:
|
|
411
|
+
output_id = len(outputs)
|
|
412
|
+
output_id_by_locations[locations] = output_id
|
|
413
|
+
outputs.append(
|
|
414
|
+
cast(
|
|
415
|
+
TableTensor,
|
|
416
|
+
torch.cat(
|
|
417
|
+
tuple(table[member_id] for table in tables),
|
|
418
|
+
dim=-1,
|
|
419
|
+
),
|
|
420
|
+
)
|
|
421
|
+
)
|
|
422
|
+
member_table_ids.append(output_id)
|
|
423
|
+
|
|
424
|
+
return cls.from_tables(
|
|
425
|
+
tables=outputs,
|
|
426
|
+
member_table_ids=member_table_ids,
|
|
427
|
+
)
|
|
428
|
+
|
|
429
|
+
def replace_tables(
|
|
430
|
+
self,
|
|
431
|
+
tables: Sequence[TableTensor],
|
|
432
|
+
member_table_ids: Sequence[int],
|
|
433
|
+
) -> Self:
|
|
434
|
+
"""Replace member tables.
|
|
435
|
+
|
|
436
|
+
Args:
|
|
437
|
+
tables: Replacement tables available to the ensemble members.
|
|
438
|
+
member_table_ids: Index into ``tables`` for each ensemble member.
|
|
439
|
+
|
|
440
|
+
Returns:
|
|
441
|
+
An ensemble table preserving existing group boundaries.
|
|
442
|
+
"""
|
|
443
|
+
if len(member_table_ids) != len(self):
|
|
444
|
+
raise ValueError("Expected one replacement table per member")
|
|
445
|
+
|
|
446
|
+
member_ids_by_group: list[list[int]] = [
|
|
447
|
+
[] for _ in range(self.num_groups)
|
|
448
|
+
]
|
|
449
|
+
for member_id, (group_id, _) in enumerate(self._locations):
|
|
450
|
+
member_ids_by_group[group_id].append(member_id)
|
|
451
|
+
|
|
452
|
+
groups: list[TableTensor] = []
|
|
453
|
+
locations = [(-1, -1)] * len(self)
|
|
454
|
+
for member_ids in member_ids_by_group:
|
|
455
|
+
table_ids = tuple(
|
|
456
|
+
dict.fromkeys(
|
|
457
|
+
member_table_ids[member_id] for member_id in member_ids
|
|
458
|
+
)
|
|
459
|
+
)
|
|
460
|
+
packed_groups, packed_locations = self._pack_tables(
|
|
461
|
+
tuple(tables[table_id] for table_id in table_ids)
|
|
462
|
+
)
|
|
463
|
+
group_offset = len(groups)
|
|
464
|
+
groups.extend(packed_groups)
|
|
465
|
+
locations_by_table_id = {
|
|
466
|
+
table_id: (group_id + group_offset, position)
|
|
467
|
+
for table_id, (group_id, position) in zip(
|
|
468
|
+
table_ids, packed_locations, strict=True
|
|
469
|
+
)
|
|
470
|
+
}
|
|
471
|
+
for member_id in member_ids:
|
|
472
|
+
locations[member_id] = locations_by_table_id[
|
|
473
|
+
member_table_ids[member_id]
|
|
474
|
+
]
|
|
475
|
+
|
|
476
|
+
return self.__class__(groups=groups, locations=locations)
|
|
477
|
+
|
|
478
|
+
def replace_groups(self, groups: Sequence[TableTensor]) -> Self:
|
|
479
|
+
"""Return an ensemble table with its groups replaced.
|
|
480
|
+
|
|
481
|
+
Args:
|
|
482
|
+
groups: One replacement group per current group.
|
|
483
|
+
|
|
484
|
+
Returns:
|
|
485
|
+
An ensemble table over ``groups`` with the current member
|
|
486
|
+
assignment.
|
|
487
|
+
"""
|
|
488
|
+
if len(groups) != self.num_groups:
|
|
489
|
+
raise ValueError(
|
|
490
|
+
f"Expected one replacement per group "
|
|
491
|
+
f"({self.num_groups}), got {len(groups)}."
|
|
492
|
+
)
|
|
493
|
+
ensemble = copy.copy(self)
|
|
494
|
+
ensemble._groups = tuple(groups)
|
|
495
|
+
return ensemble
|
|
496
|
+
|
|
497
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
498
|
+
return (
|
|
499
|
+
f"{' ' * indent}{self.__class__.__name__}("
|
|
500
|
+
f"num_members={len(self)}, num_groups={self.num_groups})"
|
|
501
|
+
)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Evaluation utilities."""
|
|
5
|
+
|
|
6
|
+
from sdm.evaluation.classification import to_class_indices, to_binary_class
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"to_class_indices",
|
|
10
|
+
"to_binary_class",
|
|
11
|
+
]
|