structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from typing import cast
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
|
|
8
|
+
from sdm import EnsembleTable, Stype, TableTensor
|
|
9
|
+
from sdm.processing import EnsembleProcessor
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class DropConstantColumns(EnsembleProcessor):
|
|
13
|
+
"""Remove non-informative numerical columns learned during fit.
|
|
14
|
+
|
|
15
|
+
Columns are retained when they have more than ``threshold`` distinct
|
|
16
|
+
values. When the number of samples is less than or equal to ``threshold``,
|
|
17
|
+
all columns are preserved. NaN is counted once as a distinct value.
|
|
18
|
+
|
|
19
|
+
Only numerical columns are supported. Convert other feature stypes before
|
|
20
|
+
this step, for example with :class:`~sdm.processing.ToNumerical`.
|
|
21
|
+
Fitting expects data with shape ``[N, C]``, where ``N`` is the number of
|
|
22
|
+
rows and ``C`` is the number of numerical columns. The learned selection
|
|
23
|
+
can transform later tables with shape ``[..., C]``.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
threshold: Columns with at most this many unique values are removed.
|
|
27
|
+
Must be positive.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
31
|
+
requires_fit = True
|
|
32
|
+
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
*,
|
|
36
|
+
threshold: int = 1,
|
|
37
|
+
) -> None:
|
|
38
|
+
super().__init__()
|
|
39
|
+
if threshold <= 0:
|
|
40
|
+
raise ValueError("threshold must be positive")
|
|
41
|
+
|
|
42
|
+
self.threshold = threshold
|
|
43
|
+
# TODO: Consider recording the fitted column names if transforms should
|
|
44
|
+
# verify that the numerical schema and order match fit.
|
|
45
|
+
self._kept_indices: tuple[tuple[int, ...], ...] = ()
|
|
46
|
+
|
|
47
|
+
def get_extra_state(self) -> tuple[tuple[int, ...], ...]:
|
|
48
|
+
r""":meta private:""" # noqa: D415
|
|
49
|
+
return self._kept_indices
|
|
50
|
+
|
|
51
|
+
def set_extra_state(self, state: object) -> None:
|
|
52
|
+
r""":meta private:""" # noqa: D415
|
|
53
|
+
self._kept_indices = cast(tuple[tuple[int, ...], ...], state)
|
|
54
|
+
|
|
55
|
+
def _keep_mask(self, data: torch.Tensor) -> torch.Tensor:
|
|
56
|
+
# [N, C] or [..., N, C] -> [C] or [..., C].
|
|
57
|
+
# Preserve the schema when too few rows can exceed the threshold.
|
|
58
|
+
if data.size(-2) <= self.threshold:
|
|
59
|
+
return data.new_ones(
|
|
60
|
+
(*data.shape[:-2], data.size(-1)),
|
|
61
|
+
dtype=torch.bool,
|
|
62
|
+
)
|
|
63
|
+
if self.threshold == 1:
|
|
64
|
+
# Any mismatch with the first row proves a second unique value.
|
|
65
|
+
different = (data != data[..., :1, :]).any(dim=-2)
|
|
66
|
+
return different & ~data.isnan().all(dim=-2)
|
|
67
|
+
|
|
68
|
+
# A sorted column with k unique values has k - 1 transitions.
|
|
69
|
+
values = data.sort(dim=-2).values
|
|
70
|
+
left, right = values[..., :-1, :], values[..., 1:, :]
|
|
71
|
+
changed = (right != left) & ~(right.isnan() & left.isnan())
|
|
72
|
+
return changed.sum(dim=-2) >= self.threshold
|
|
73
|
+
|
|
74
|
+
@staticmethod
|
|
75
|
+
def _select_columns(
|
|
76
|
+
table: TableTensor,
|
|
77
|
+
kept_indices: tuple[int, ...],
|
|
78
|
+
) -> TableTensor:
|
|
79
|
+
numerical_columns = table.columns[Stype.numerical]
|
|
80
|
+
if len(kept_indices) == len(numerical_columns):
|
|
81
|
+
return table
|
|
82
|
+
kept_numerical_columns = tuple(
|
|
83
|
+
numerical_columns[index] for index in kept_indices
|
|
84
|
+
)
|
|
85
|
+
columns = [
|
|
86
|
+
column
|
|
87
|
+
for stype, stype_columns in table.columns.items()
|
|
88
|
+
for column in (
|
|
89
|
+
kept_numerical_columns
|
|
90
|
+
if stype == Stype.numerical
|
|
91
|
+
else stype_columns
|
|
92
|
+
)
|
|
93
|
+
]
|
|
94
|
+
return table.select_columns(columns)
|
|
95
|
+
|
|
96
|
+
def _fit(
|
|
97
|
+
self,
|
|
98
|
+
table: TableTensor,
|
|
99
|
+
*,
|
|
100
|
+
generator: torch.Generator | None = None,
|
|
101
|
+
) -> None:
|
|
102
|
+
keep = self._keep_mask(table.numerical).tolist()
|
|
103
|
+
self._kept_indices = (
|
|
104
|
+
tuple(index for index, kept in enumerate(keep) if kept),
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
108
|
+
return self._select_columns(table, self._kept_indices[0])
|
|
109
|
+
|
|
110
|
+
def _fit_transform(
|
|
111
|
+
self,
|
|
112
|
+
table: TableTensor,
|
|
113
|
+
*,
|
|
114
|
+
generator: torch.Generator | None = None,
|
|
115
|
+
) -> TableTensor:
|
|
116
|
+
self._fit(table, generator=generator)
|
|
117
|
+
return self._transform(table)
|
|
118
|
+
|
|
119
|
+
def _fit_ensemble(
|
|
120
|
+
self,
|
|
121
|
+
ensemble_table: EnsembleTable,
|
|
122
|
+
*,
|
|
123
|
+
generator: torch.Generator | None = None,
|
|
124
|
+
) -> None:
|
|
125
|
+
groups = tuple(ensemble_table._iter_groups())
|
|
126
|
+
if sum(group.size(0) for group in groups) == 1:
|
|
127
|
+
group = groups[0]
|
|
128
|
+
keep = self._keep_mask(group.numerical)[0].tolist()
|
|
129
|
+
kept_indices = tuple(
|
|
130
|
+
index for index, kept in enumerate(keep) if kept
|
|
131
|
+
)
|
|
132
|
+
self._kept_indices = (kept_indices,) * len(ensemble_table)
|
|
133
|
+
return
|
|
134
|
+
|
|
135
|
+
masks = ensemble_table.replace_groups(
|
|
136
|
+
[
|
|
137
|
+
TableTensor.from_tensor(
|
|
138
|
+
tensor=self._keep_mask(group.numerical)
|
|
139
|
+
.to(dtype=group.numerical.dtype)
|
|
140
|
+
.unsqueeze(-2),
|
|
141
|
+
columns=group.columns[Stype.numerical],
|
|
142
|
+
)
|
|
143
|
+
for group in groups
|
|
144
|
+
]
|
|
145
|
+
)
|
|
146
|
+
masks_by_size_and_device: dict[
|
|
147
|
+
tuple[int, torch.device],
|
|
148
|
+
list[tuple[int, torch.Tensor]],
|
|
149
|
+
] = {}
|
|
150
|
+
for member_id in range(len(masks)):
|
|
151
|
+
table = masks[member_id]
|
|
152
|
+
key = (table.numerical.size(-1), table.device)
|
|
153
|
+
masks_by_size_and_device.setdefault(key, []).append(
|
|
154
|
+
(member_id, table.numerical[0].bool())
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
kept_indices_by_member: list[tuple[int, ...]]
|
|
158
|
+
kept_indices_by_member = [()] * len(masks)
|
|
159
|
+
for member_masks in masks_by_size_and_device.values():
|
|
160
|
+
keep_by_member = torch.stack(
|
|
161
|
+
[keep for _, keep in member_masks]
|
|
162
|
+
).tolist()
|
|
163
|
+
for (member_id, _), keep in zip(
|
|
164
|
+
member_masks,
|
|
165
|
+
keep_by_member,
|
|
166
|
+
strict=True,
|
|
167
|
+
):
|
|
168
|
+
kept_indices_by_member[member_id] = tuple(
|
|
169
|
+
index for index, kept in enumerate(keep) if kept
|
|
170
|
+
)
|
|
171
|
+
self._kept_indices = tuple(kept_indices_by_member)
|
|
172
|
+
|
|
173
|
+
def _transform_ensemble(
|
|
174
|
+
self,
|
|
175
|
+
ensemble_table: EnsembleTable,
|
|
176
|
+
) -> EnsembleTable:
|
|
177
|
+
if len(self._kept_indices) != len(ensemble_table):
|
|
178
|
+
raise RuntimeError(
|
|
179
|
+
"DropConstantColumns must be fitted with the same number of "
|
|
180
|
+
"ensemble members before transform."
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
member_ids_by_kept_indices: dict[tuple[int, ...], list[int]] = {}
|
|
184
|
+
for member_id, kept_indices in enumerate(self._kept_indices):
|
|
185
|
+
member_ids_by_kept_indices.setdefault(kept_indices, []).append(
|
|
186
|
+
member_id
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
if len(member_ids_by_kept_indices) == 1:
|
|
190
|
+
kept_indices = next(iter(member_ids_by_kept_indices))
|
|
191
|
+
return ensemble_table.replace_groups(
|
|
192
|
+
[
|
|
193
|
+
self._select_columns(group, kept_indices)
|
|
194
|
+
for group in ensemble_table._iter_groups()
|
|
195
|
+
]
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
if sum(
|
|
199
|
+
group.size(0) for group in ensemble_table._iter_groups()
|
|
200
|
+
) == len(ensemble_table):
|
|
201
|
+
tables = [
|
|
202
|
+
self._select_columns(
|
|
203
|
+
ensemble_table[member_id],
|
|
204
|
+
kept_indices,
|
|
205
|
+
)
|
|
206
|
+
for member_id, kept_indices in enumerate(self._kept_indices)
|
|
207
|
+
]
|
|
208
|
+
return ensemble_table.replace_tables(
|
|
209
|
+
tables=tables,
|
|
210
|
+
member_table_ids=range(len(ensemble_table)),
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
outputs: dict[tuple[int, ...], EnsembleTable] = {}
|
|
214
|
+
for kept_indices, member_ids in member_ids_by_kept_indices.items():
|
|
215
|
+
selected = ensemble_table[member_ids]
|
|
216
|
+
outputs[kept_indices] = selected.replace_groups(
|
|
217
|
+
[
|
|
218
|
+
self._select_columns(group, kept_indices)
|
|
219
|
+
for group in selected._iter_groups()
|
|
220
|
+
]
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
tables = []
|
|
224
|
+
member_ids = []
|
|
225
|
+
next_member_id_by_kept_indices: dict[tuple[int, ...], int] = {}
|
|
226
|
+
for kept_indices in self._kept_indices:
|
|
227
|
+
tables.append(outputs[kept_indices])
|
|
228
|
+
member_id = next_member_id_by_kept_indices.get(kept_indices, 0)
|
|
229
|
+
member_ids.append(member_id)
|
|
230
|
+
next_member_id_by_kept_indices[kept_indices] = member_id + 1
|
|
231
|
+
|
|
232
|
+
return ensemble_table.gather_members(
|
|
233
|
+
tables=tables,
|
|
234
|
+
member_ids=member_ids,
|
|
235
|
+
)
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
from torch import Tensor
|
|
6
|
+
|
|
7
|
+
from sdm import EnsembleTable, Stype
|
|
8
|
+
from sdm.nn._buffer import BufferList
|
|
9
|
+
from sdm.processing import EnsembleInvertibleMixin, EnsembleProcessor
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class FlipSign(EnsembleProcessor, EnsembleInvertibleMixin):
|
|
13
|
+
"""Randomly negate numerical columns.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
probability: Probability of negating each numerical column.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
20
|
+
requires_fit = True
|
|
21
|
+
|
|
22
|
+
def __init__(
|
|
23
|
+
self,
|
|
24
|
+
probability: float = 0.5,
|
|
25
|
+
) -> None:
|
|
26
|
+
super().__init__()
|
|
27
|
+
self.probability = probability
|
|
28
|
+
self._signs: BufferList[Tensor] = BufferList()
|
|
29
|
+
|
|
30
|
+
def _fit_ensemble(
|
|
31
|
+
self,
|
|
32
|
+
ensemble_table: EnsembleTable,
|
|
33
|
+
*,
|
|
34
|
+
generator: torch.Generator | None = None,
|
|
35
|
+
) -> None:
|
|
36
|
+
signs = []
|
|
37
|
+
for member_id in range(len(ensemble_table)):
|
|
38
|
+
numerical = ensemble_table[member_id].numerical
|
|
39
|
+
sign = numerical.new_empty(
|
|
40
|
+
(*numerical.size()[:-2], 1, numerical.size(-1))
|
|
41
|
+
)
|
|
42
|
+
sign.bernoulli_(self.probability, generator=generator)
|
|
43
|
+
sign.mul_(-2).add_(1)
|
|
44
|
+
signs.append(sign)
|
|
45
|
+
self._signs = BufferList(signs)
|
|
46
|
+
|
|
47
|
+
def _transform_ensemble(
|
|
48
|
+
self,
|
|
49
|
+
ensemble_table: EnsembleTable,
|
|
50
|
+
) -> EnsembleTable:
|
|
51
|
+
|
|
52
|
+
member_ids_by_group = [[] for _ in range(ensemble_table.num_groups)]
|
|
53
|
+
for member_id, (group_id, _) in enumerate(ensemble_table._locations):
|
|
54
|
+
member_ids_by_group[group_id].append(member_id)
|
|
55
|
+
|
|
56
|
+
groups = []
|
|
57
|
+
locations = [(-1, -1)] * len(ensemble_table)
|
|
58
|
+
for group_id, member_ids in enumerate(member_ids_by_group):
|
|
59
|
+
group = ensemble_table.expanded_group(group_id)
|
|
60
|
+
sign = torch.stack([self._signs[i] for i in member_ids], dim=0)
|
|
61
|
+
group = group.replace_blocks(numerical=group.numerical * sign)
|
|
62
|
+
groups.append(group)
|
|
63
|
+
for position, member_id in enumerate(member_ids):
|
|
64
|
+
locations[member_id] = (group_id, position)
|
|
65
|
+
|
|
66
|
+
return EnsembleTable(groups, locations)
|
|
67
|
+
|
|
68
|
+
def _inverse_transform_ensemble(
|
|
69
|
+
self,
|
|
70
|
+
ensemble_table: EnsembleTable,
|
|
71
|
+
) -> EnsembleTable:
|
|
72
|
+
return self._transform_ensemble(ensemble_table)
|
|
73
|
+
|
|
74
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
75
|
+
return (
|
|
76
|
+
f"{' ' * indent}{self.__class__.__name__}"
|
|
77
|
+
f"(probability={self.probability})"
|
|
78
|
+
)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
|
|
6
|
+
from sdm import Stype, TableTensor
|
|
7
|
+
from sdm.processing import Processor
|
|
8
|
+
from sdm.processing.numerical._stats import _isfinite
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ImputeMean(Processor):
|
|
12
|
+
"""Replace NaN feature values with fitted per-column means.
|
|
13
|
+
|
|
14
|
+
Infinite values are ignored when fitting the mean but preserved during
|
|
15
|
+
transformation.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
fill_value: Finite value used for columns whose fitted mean is
|
|
19
|
+
undefined (columns without finite values).
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
23
|
+
requires_fit = True
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
*,
|
|
28
|
+
fill_value: float = 0.0,
|
|
29
|
+
) -> None:
|
|
30
|
+
super().__init__()
|
|
31
|
+
self.fill_value = fill_value
|
|
32
|
+
self.register_buffer("_mean", torch.empty(0))
|
|
33
|
+
|
|
34
|
+
def _fit(
|
|
35
|
+
self,
|
|
36
|
+
table: TableTensor,
|
|
37
|
+
*,
|
|
38
|
+
generator: torch.Generator | None = None,
|
|
39
|
+
) -> None:
|
|
40
|
+
numerical = table.numerical
|
|
41
|
+
mean = torch.nanmean(
|
|
42
|
+
numerical.masked_fill(~_isfinite(numerical), torch.nan),
|
|
43
|
+
dim=-2,
|
|
44
|
+
keepdim=True,
|
|
45
|
+
)
|
|
46
|
+
self._mean = torch.where(mean.isnan(), self.fill_value, mean)
|
|
47
|
+
|
|
48
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
49
|
+
"""Replace NaNs with the fitted per-column means."""
|
|
50
|
+
numerical = table.numerical
|
|
51
|
+
numerical = torch.where(numerical.isnan(), self._mean, numerical)
|
|
52
|
+
return table.replace_blocks(numerical=numerical)
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from typing import cast
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
|
|
8
|
+
from sdm import Stype, TableTensor
|
|
9
|
+
from sdm.processing import Processor
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class PCA(Processor):
|
|
13
|
+
r"""Project numerical columns onto their principal components.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
num_components: Number of principal components to keep.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
20
|
+
requires_fit = True
|
|
21
|
+
|
|
22
|
+
def __init__(self, num_components: int) -> None:
|
|
23
|
+
super().__init__()
|
|
24
|
+
self.num_components = num_components
|
|
25
|
+
self.register_buffer("mean", torch.empty(0))
|
|
26
|
+
self.register_buffer("components", torch.empty(0))
|
|
27
|
+
|
|
28
|
+
def _fit(
|
|
29
|
+
self,
|
|
30
|
+
table: TableTensor,
|
|
31
|
+
*,
|
|
32
|
+
generator: torch.Generator | None = None,
|
|
33
|
+
) -> None:
|
|
34
|
+
|
|
35
|
+
if table.numerical.size(-1) == 0:
|
|
36
|
+
raise ValueError(
|
|
37
|
+
f"{self.__class__.__name__!r} requires 'table' to have "
|
|
38
|
+
f"numerical features"
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
self.mean = table.numerical.mean(dim=-2, keepdim=True)
|
|
42
|
+
_, _, vh = torch.linalg.svd(
|
|
43
|
+
table.numerical - self.mean,
|
|
44
|
+
full_matrices=False,
|
|
45
|
+
)
|
|
46
|
+
num_components = min(
|
|
47
|
+
self.num_components,
|
|
48
|
+
table.numerical.size(-2),
|
|
49
|
+
table.numerical.size(-1),
|
|
50
|
+
)
|
|
51
|
+
self.components = vh[..., :num_components, :].transpose(-2, -1)
|
|
52
|
+
|
|
53
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
54
|
+
x = (table.numerical - self.mean) @ self.components
|
|
55
|
+
out = TableTensor(
|
|
56
|
+
columns={Stype.numerical: [f"pca_{i}" for i in range(x.size(-1))]},
|
|
57
|
+
numerical=x,
|
|
58
|
+
)
|
|
59
|
+
return cast(
|
|
60
|
+
TableTensor,
|
|
61
|
+
torch.cat([table.drop_stypes(Stype.numerical), out], dim=-1),
|
|
62
|
+
)
|