structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import copy
|
|
7
|
+
from collections.abc import Mapping, Sequence
|
|
8
|
+
from typing import NamedTuple, cast
|
|
9
|
+
|
|
10
|
+
import torch
|
|
11
|
+
from torch import Tensor
|
|
12
|
+
|
|
13
|
+
import sdm.processing as sp
|
|
14
|
+
from sdm import EnsembleTable, Recipe, RelatedTables, Stype, TableTensor
|
|
15
|
+
from sdm.processing import EnsembleInvertibleMixin, EnsembleProcessor
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class MemberContext(NamedTuple):
|
|
19
|
+
"""Transformed context tables for one ensemble member."""
|
|
20
|
+
|
|
21
|
+
x: TableTensor
|
|
22
|
+
y: TableTensor
|
|
23
|
+
related_tables: RelatedTables[TableTensor] | None
|
|
24
|
+
#: Semantic types of the raw context columns before ``recipe.features``.
|
|
25
|
+
input_stypes: Mapping[str, Stype]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class MemberQuery(NamedTuple):
|
|
29
|
+
"""Transformed query tables for one ensemble member."""
|
|
30
|
+
|
|
31
|
+
x: TableTensor
|
|
32
|
+
related_tables: RelatedTables[TableTensor] | None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class RecipeExecution:
|
|
36
|
+
"""Recipe execution manager during model processing."""
|
|
37
|
+
|
|
38
|
+
def __init__(self, recipe: Recipe) -> None:
|
|
39
|
+
self.recipe = recipe
|
|
40
|
+
|
|
41
|
+
self._related_processors: Mapping[str, EnsembleProcessor] | None = None
|
|
42
|
+
self._related_locations: (
|
|
43
|
+
Mapping[str, tuple[tuple[int, int], ...]] | None
|
|
44
|
+
) = None
|
|
45
|
+
self._num_estimators: int | None = None
|
|
46
|
+
self._y_locations: tuple[tuple[int, int], ...] | None = None
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def num_members(self) -> int:
|
|
50
|
+
r"""The number of fitted members."""
|
|
51
|
+
assert self._y_locations is not None
|
|
52
|
+
return len(self._y_locations)
|
|
53
|
+
|
|
54
|
+
def fit_transform(
|
|
55
|
+
self,
|
|
56
|
+
x: Tensor | TableTensor | EnsembleTable,
|
|
57
|
+
y: Tensor | TableTensor | EnsembleTable,
|
|
58
|
+
related_tables: RelatedTables | None,
|
|
59
|
+
*,
|
|
60
|
+
num_members: int | None = None,
|
|
61
|
+
generator: torch.Generator | None = None,
|
|
62
|
+
) -> tuple[MemberContext, ...]:
|
|
63
|
+
"""Fit and transform context data."""
|
|
64
|
+
# Transform target first to be able to resolve task type. Inverse
|
|
65
|
+
# target transforms require distinct member assignment:
|
|
66
|
+
y = _to_ensemble_table(y, num_members, expand=True)
|
|
67
|
+
y = self.recipe.target.fit_transform_ensemble(y, generator=generator)
|
|
68
|
+
|
|
69
|
+
self._num_estimators = num_members
|
|
70
|
+
self._y_locations = y._locations
|
|
71
|
+
|
|
72
|
+
task_dispatchers = tuple(
|
|
73
|
+
module
|
|
74
|
+
for processor in (self.recipe.features, self.recipe.output)
|
|
75
|
+
for module in processor.modules()
|
|
76
|
+
if isinstance(module, sp.TaskDispatch)
|
|
77
|
+
)
|
|
78
|
+
if len(task_dispatchers) > 0:
|
|
79
|
+
if any(group.size(-1) != 1 for group in y._iter_groups()):
|
|
80
|
+
raise ValueError(
|
|
81
|
+
"Expected the transformed target to contain exactly one "
|
|
82
|
+
"column"
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
if all(
|
|
86
|
+
group.numerical.size(-1) == 1 for group in y._iter_groups()
|
|
87
|
+
):
|
|
88
|
+
task = "regression"
|
|
89
|
+
elif all(
|
|
90
|
+
group.categorical.size(-1) == 1 for group in y._iter_groups()
|
|
91
|
+
):
|
|
92
|
+
task = "classification"
|
|
93
|
+
else:
|
|
94
|
+
raise ValueError(
|
|
95
|
+
"'Recipe.target' must resolve to a single task type"
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
for task_dispatcher in task_dispatchers:
|
|
99
|
+
task_dispatcher._task = task
|
|
100
|
+
|
|
101
|
+
self._related_processors = None
|
|
102
|
+
self._related_locations = None
|
|
103
|
+
related_ensembles: Mapping[str, EnsembleTable] = {}
|
|
104
|
+
if related_tables is not None:
|
|
105
|
+
self._related_processors = {}
|
|
106
|
+
self._related_locations = {}
|
|
107
|
+
for name, table in related_tables.tables.items():
|
|
108
|
+
processor = copy.deepcopy(self.recipe.features)
|
|
109
|
+
for module in processor.modules():
|
|
110
|
+
if isinstance(module, sp.TableDispatch):
|
|
111
|
+
module._route = "related"
|
|
112
|
+
self._related_processors[name] = processor
|
|
113
|
+
ensemble_table = _to_ensemble_table(table, num_members)
|
|
114
|
+
self._related_locations[name] = ensemble_table._locations
|
|
115
|
+
related_ensembles[name] = processor.fit_transform_ensemble(
|
|
116
|
+
ensemble_table,
|
|
117
|
+
generator=generator,
|
|
118
|
+
)
|
|
119
|
+
if len(related_ensembles[name]) != self.num_members:
|
|
120
|
+
raise ValueError(
|
|
121
|
+
"Expected inputs to map to the same number of "
|
|
122
|
+
"ensemble members"
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
for module in self.recipe.features.modules():
|
|
126
|
+
if isinstance(module, sp.TableDispatch):
|
|
127
|
+
module._route = "task"
|
|
128
|
+
|
|
129
|
+
inputs = _to_ensemble_table(x, num_members)
|
|
130
|
+
x = self.recipe.features.fit_transform_ensemble(
|
|
131
|
+
inputs,
|
|
132
|
+
generator=generator,
|
|
133
|
+
)
|
|
134
|
+
if len(x) != self.num_members:
|
|
135
|
+
raise ValueError(
|
|
136
|
+
"Expected inputs to map to the same number of ensemble members"
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
members: list[MemberContext] = []
|
|
140
|
+
for member_id in range(self.num_members):
|
|
141
|
+
related_tables_i: RelatedTables[TableTensor] | None = None
|
|
142
|
+
if related_tables is not None:
|
|
143
|
+
related_tables_i = RelatedTables(
|
|
144
|
+
tables={
|
|
145
|
+
name: table[member_id]
|
|
146
|
+
for name, table in related_ensembles.items()
|
|
147
|
+
},
|
|
148
|
+
relationships=related_tables.relationships,
|
|
149
|
+
task_links=related_tables.task_links,
|
|
150
|
+
)
|
|
151
|
+
members.append(
|
|
152
|
+
MemberContext(
|
|
153
|
+
x=x[member_id],
|
|
154
|
+
y=y[member_id],
|
|
155
|
+
related_tables=related_tables_i,
|
|
156
|
+
input_stypes=inputs[member_id].stypes,
|
|
157
|
+
)
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
return tuple(members)
|
|
161
|
+
|
|
162
|
+
def transform(
|
|
163
|
+
self,
|
|
164
|
+
x: Tensor | TableTensor | EnsembleTable,
|
|
165
|
+
related_tables: RelatedTables | None,
|
|
166
|
+
) -> tuple[MemberQuery, ...]:
|
|
167
|
+
"""Transform query data."""
|
|
168
|
+
x = _to_ensemble_table(x, self._num_estimators)
|
|
169
|
+
x = self.recipe.features.transform_ensemble(x)
|
|
170
|
+
if len(x) != self.num_members:
|
|
171
|
+
raise ValueError(
|
|
172
|
+
"Expected inputs to map to the same number of ensemble members"
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
related_ensembles: Mapping[str, EnsembleTable] = {}
|
|
176
|
+
if related_tables is not None:
|
|
177
|
+
assert self._related_processors is not None
|
|
178
|
+
assert self._related_locations is not None
|
|
179
|
+
for name, table in related_tables.tables.items():
|
|
180
|
+
processor = self._related_processors[name]
|
|
181
|
+
ensemble_table = _to_ensemble_table(
|
|
182
|
+
table, self._num_estimators
|
|
183
|
+
)
|
|
184
|
+
ensemble_table = _align_to_fitted_groups(
|
|
185
|
+
ensemble_table,
|
|
186
|
+
self._related_locations[name],
|
|
187
|
+
)
|
|
188
|
+
related_ensembles[name] = processor.transform_ensemble(
|
|
189
|
+
ensemble_table
|
|
190
|
+
)
|
|
191
|
+
if len(related_ensembles[name]) != self.num_members:
|
|
192
|
+
raise ValueError(
|
|
193
|
+
"Expected inputs to map to the same number of "
|
|
194
|
+
"ensemble members"
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
members: list[MemberQuery] = []
|
|
198
|
+
for member_id in range(self.num_members):
|
|
199
|
+
related_tables_i: RelatedTables[TableTensor] | None = None
|
|
200
|
+
if related_tables is not None:
|
|
201
|
+
related_tables_i = RelatedTables(
|
|
202
|
+
tables={
|
|
203
|
+
name: table[member_id]
|
|
204
|
+
for name, table in related_ensembles.items()
|
|
205
|
+
},
|
|
206
|
+
relationships=related_tables.relationships,
|
|
207
|
+
task_links=related_tables.task_links,
|
|
208
|
+
)
|
|
209
|
+
members.append(
|
|
210
|
+
MemberQuery(
|
|
211
|
+
x=x[member_id],
|
|
212
|
+
related_tables=related_tables_i,
|
|
213
|
+
)
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
return tuple(members)
|
|
217
|
+
|
|
218
|
+
def inverse_transform_target(
|
|
219
|
+
self,
|
|
220
|
+
outputs: Sequence[TableTensor],
|
|
221
|
+
) -> tuple[TableTensor, ...]:
|
|
222
|
+
"""Invert fitted target transforms on member outputs."""
|
|
223
|
+
assert len(outputs) == self.num_members
|
|
224
|
+
|
|
225
|
+
# Reconstruct the group layout of the transformed target:
|
|
226
|
+
assert self._y_locations is not None
|
|
227
|
+
num_groups = max(group for group, _ in self._y_locations) + 1
|
|
228
|
+
groups: list[list[TableTensor | None]] = [
|
|
229
|
+
[] for _ in range(num_groups)
|
|
230
|
+
]
|
|
231
|
+
for group_id, _ in self._y_locations:
|
|
232
|
+
groups[group_id].append(None)
|
|
233
|
+
for i, (group_id, position) in enumerate(self._y_locations):
|
|
234
|
+
groups[group_id][position] = outputs[i]
|
|
235
|
+
|
|
236
|
+
table = EnsembleTable(
|
|
237
|
+
groups=[
|
|
238
|
+
cast(
|
|
239
|
+
TableTensor,
|
|
240
|
+
group[0].unsqueeze(0) # type: ignore
|
|
241
|
+
if len(group) == 1
|
|
242
|
+
else torch.stack(group, dim=0), # type: ignore
|
|
243
|
+
)
|
|
244
|
+
for group in groups
|
|
245
|
+
],
|
|
246
|
+
locations=self._y_locations,
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
if not isinstance(self.recipe.target, EnsembleInvertibleMixin):
|
|
250
|
+
raise RuntimeError("Target recipe is not invertible")
|
|
251
|
+
table = self.recipe.target.inverse_transform_ensemble(table)
|
|
252
|
+
return tuple(table[i] for i in range(len(table)))
|
|
253
|
+
|
|
254
|
+
def transform_output(
|
|
255
|
+
self,
|
|
256
|
+
outputs: Sequence[TableTensor],
|
|
257
|
+
) -> TableTensor:
|
|
258
|
+
"""Apply ``recipe.output`` to member outputs."""
|
|
259
|
+
if len(outputs) == 1:
|
|
260
|
+
out = outputs[0].unsqueeze(0)
|
|
261
|
+
else:
|
|
262
|
+
expected = set(outputs[0].columns[Stype.numerical])
|
|
263
|
+
for output in outputs[1:]:
|
|
264
|
+
if set(output.columns[Stype.numerical]) != expected:
|
|
265
|
+
raise ValueError(
|
|
266
|
+
"Expected all model outputs to have the same columns "
|
|
267
|
+
"before applying 'Recipe.output'. Ensure every target "
|
|
268
|
+
"contains the same set of classes."
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
out = torch.stack(list(outputs), dim=0)
|
|
272
|
+
|
|
273
|
+
return self.recipe.output.transform(cast(TableTensor, out))
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _align_to_fitted_groups(
|
|
277
|
+
ensemble_table: EnsembleTable,
|
|
278
|
+
fitted_locations: Sequence[tuple[int, int]],
|
|
279
|
+
) -> EnsembleTable:
|
|
280
|
+
"""Align query groups with the fitted related-table groups."""
|
|
281
|
+
fitted_locations = tuple(fitted_locations)
|
|
282
|
+
if ensemble_table._locations == fitted_locations:
|
|
283
|
+
return ensemble_table
|
|
284
|
+
|
|
285
|
+
members_by_fitted_position: list[dict[int, list[int]]] = [
|
|
286
|
+
{} for _ in range(max(group for group, _ in fitted_locations) + 1)
|
|
287
|
+
]
|
|
288
|
+
for member_id, (group, position) in enumerate(fitted_locations):
|
|
289
|
+
members_by_fitted_position[group].setdefault(position, []).append(
|
|
290
|
+
member_id
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
groups = []
|
|
294
|
+
locations = list(fitted_locations)
|
|
295
|
+
for group_id, fitted_positions in enumerate(members_by_fitted_position):
|
|
296
|
+
positions_and_members = tuple(sorted(fitted_positions.items()))
|
|
297
|
+
sources_by_position = tuple(
|
|
298
|
+
tuple(
|
|
299
|
+
dict.fromkeys(
|
|
300
|
+
ensemble_table._locations[member_id]
|
|
301
|
+
for member_id in member_ids
|
|
302
|
+
)
|
|
303
|
+
)
|
|
304
|
+
for _, member_ids in positions_and_members
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
# A fitted position may fan out only when its state can broadcast over
|
|
308
|
+
# the entire group; otherwise states could map to the wrong queries.
|
|
309
|
+
if len(sources_by_position) > 1 and any(
|
|
310
|
+
len(sources) > 1 for sources in sources_by_position
|
|
311
|
+
):
|
|
312
|
+
raise ValueError(
|
|
313
|
+
"Cannot align query tables with fitted related-table groups"
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
if len(sources_by_position) == 1:
|
|
317
|
+
source_locations = sources_by_position[0]
|
|
318
|
+
position_by_source = {
|
|
319
|
+
source: position
|
|
320
|
+
for position, source in enumerate(source_locations)
|
|
321
|
+
}
|
|
322
|
+
for member_id in positions_and_members[0][1]:
|
|
323
|
+
locations[member_id] = (
|
|
324
|
+
group_id,
|
|
325
|
+
position_by_source[ensemble_table._locations[member_id]],
|
|
326
|
+
)
|
|
327
|
+
else:
|
|
328
|
+
source_locations = tuple(
|
|
329
|
+
sources[0] for sources in sources_by_position
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
tables = tuple(
|
|
333
|
+
ensemble_table._groups[group][position]
|
|
334
|
+
for group, position in source_locations
|
|
335
|
+
)
|
|
336
|
+
# Reuse one shared query without materializing a copy per fitted state.
|
|
337
|
+
if all(source == source_locations[0] for source in source_locations):
|
|
338
|
+
table = tables[0]
|
|
339
|
+
output = cast(
|
|
340
|
+
TableTensor,
|
|
341
|
+
table.unsqueeze(0).expand(len(tables), *table.size()),
|
|
342
|
+
)
|
|
343
|
+
else:
|
|
344
|
+
packed_groups, _ = EnsembleTable._pack_tables(tables)
|
|
345
|
+
if len(packed_groups) != 1:
|
|
346
|
+
raise ValueError(
|
|
347
|
+
"Cannot align query tables with fitted related-table "
|
|
348
|
+
"groups"
|
|
349
|
+
)
|
|
350
|
+
output = packed_groups[0]
|
|
351
|
+
groups.append(output)
|
|
352
|
+
|
|
353
|
+
return EnsembleTable(groups=groups, locations=locations)
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _to_ensemble_table(
|
|
357
|
+
x: Tensor | TableTensor | EnsembleTable,
|
|
358
|
+
num_estimators: int | None,
|
|
359
|
+
*,
|
|
360
|
+
expand: bool = False,
|
|
361
|
+
) -> EnsembleTable:
|
|
362
|
+
r"""Convert model input data into an ensemble-aware representation.
|
|
363
|
+
|
|
364
|
+
Args:
|
|
365
|
+
x: The input table.
|
|
366
|
+
num_estimators: Number of estimators to represent. If ``None``, a 2D
|
|
367
|
+
input creates one estimator and higher-rank inputs infer the
|
|
368
|
+
estimator count from their leading dimension.
|
|
369
|
+
expand: Whether to expand shared inputs into distinct logical member
|
|
370
|
+
positions.
|
|
371
|
+
"""
|
|
372
|
+
if isinstance(x, EnsembleTable):
|
|
373
|
+
if num_estimators is not None and num_estimators != len(x):
|
|
374
|
+
raise ValueError(
|
|
375
|
+
f"Expected {num_estimators} members in 'EnsembleTable' "
|
|
376
|
+
f"(got {len(x)})"
|
|
377
|
+
)
|
|
378
|
+
if expand:
|
|
379
|
+
groups = [x.expanded_group(i) for i in range(x.num_groups)]
|
|
380
|
+
locations = []
|
|
381
|
+
next_pos = [0] * x.num_groups
|
|
382
|
+
for i, _ in x._locations:
|
|
383
|
+
locations.append((i, next_pos[i]))
|
|
384
|
+
next_pos[i] += 1
|
|
385
|
+
x = EnsembleTable(
|
|
386
|
+
groups=tuple(groups),
|
|
387
|
+
locations=tuple(locations),
|
|
388
|
+
)
|
|
389
|
+
if len(x) < 1:
|
|
390
|
+
raise ValueError("'num_estimators' needs to be positive")
|
|
391
|
+
return x
|
|
392
|
+
|
|
393
|
+
if not isinstance(x, TableTensor):
|
|
394
|
+
x = TableTensor.from_tensor(x)
|
|
395
|
+
|
|
396
|
+
# Treat leading dimension as ensemble dimension:
|
|
397
|
+
if x.dim() > 2 and num_estimators is None:
|
|
398
|
+
locations = tuple((0, i) for i in range(x.size(0)))
|
|
399
|
+
x = EnsembleTable(groups=(x,), locations=locations)
|
|
400
|
+
if len(x) < 1:
|
|
401
|
+
raise ValueError("'num_estimators' needs to be positive")
|
|
402
|
+
return x
|
|
403
|
+
|
|
404
|
+
num_estimators = 1 if num_estimators is None else num_estimators
|
|
405
|
+
if num_estimators < 1:
|
|
406
|
+
raise ValueError("'num_estimators' needs to be positive")
|
|
407
|
+
|
|
408
|
+
# Add a leading ensemble dimension:
|
|
409
|
+
x = x.unsqueeze(0)
|
|
410
|
+
if expand:
|
|
411
|
+
x = x.expand(num_estimators, *x.size()[1:])
|
|
412
|
+
|
|
413
|
+
if x.size(0) == 1:
|
|
414
|
+
locations = ((0, 0),) * num_estimators
|
|
415
|
+
else:
|
|
416
|
+
locations = tuple((0, i) for i in range(num_estimators))
|
|
417
|
+
|
|
418
|
+
return EnsembleTable(groups=(cast(TableTensor, x),), locations=locations)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Numerical preprocessing transforms."""
|
|
5
|
+
|
|
6
|
+
from sdm.processing.numerical.cast import Cast
|
|
7
|
+
from sdm.processing.numerical.clip import Clip
|
|
8
|
+
from sdm.processing.numerical.quantile_clip import ClipQuantiles
|
|
9
|
+
from sdm.processing.numerical.sigma_clip import ClipSigma
|
|
10
|
+
from sdm.processing.numerical.clip_soft import ClipSoft
|
|
11
|
+
from sdm.processing.numerical.impute import ImputeMean
|
|
12
|
+
from sdm.processing.numerical.power import PowerTransform
|
|
13
|
+
from sdm.processing.numerical.quantile import QuantileTransform
|
|
14
|
+
from sdm.processing.numerical.rank_gaussian import RankGaussian
|
|
15
|
+
from sdm.processing.numerical.standardize import Standardize
|
|
16
|
+
from sdm.processing.numerical.robust_scale import RobustScale
|
|
17
|
+
from sdm.processing.numerical.flip_sign import FlipSign
|
|
18
|
+
from sdm.processing.numerical.constant import DropConstantColumns
|
|
19
|
+
from sdm.processing.numerical.pca import PCA
|
|
20
|
+
from sdm.processing.numerical.random_projection import RandomProjection
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"Cast",
|
|
24
|
+
"Clip",
|
|
25
|
+
"ClipQuantiles",
|
|
26
|
+
"ClipSigma",
|
|
27
|
+
"ClipSoft",
|
|
28
|
+
"ImputeMean",
|
|
29
|
+
"PowerTransform",
|
|
30
|
+
"QuantileTransform",
|
|
31
|
+
"RankGaussian",
|
|
32
|
+
"Standardize",
|
|
33
|
+
"RobustScale",
|
|
34
|
+
"FlipSign",
|
|
35
|
+
"DropConstantColumns",
|
|
36
|
+
"PCA",
|
|
37
|
+
"RandomProjection",
|
|
38
|
+
]
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import math
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
from torch import Tensor
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _ndtri(x: Tensor) -> Tensor:
|
|
11
|
+
# MPS does not implement 'torch.special.ndtri', so use 'erfinv' instead.
|
|
12
|
+
if x.device.type == "mps":
|
|
13
|
+
return x.mul(2.0).sub_(1.0).erfinv_().mul_(math.sqrt(2.0))
|
|
14
|
+
return torch.special.ndtri(x)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _isfinite(x: Tensor) -> Tensor:
|
|
18
|
+
# Equal to 'x.isfinite()', which allocates 'x.abs()' on the way.
|
|
19
|
+
return x.gt(-math.inf).logical_and_(x.lt(math.inf))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _constant_feature_mask(
|
|
23
|
+
var: Tensor,
|
|
24
|
+
mean: Tensor,
|
|
25
|
+
num_samples: int | Tensor,
|
|
26
|
+
) -> Tensor:
|
|
27
|
+
eps = torch.finfo(var.dtype).eps
|
|
28
|
+
upper_bound = num_samples * eps * var + (num_samples * mean * eps) ** 2
|
|
29
|
+
return var <= upper_bound
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
|
|
6
|
+
from sdm import Stype, TableTensor
|
|
7
|
+
from sdm.processing import Processor
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Cast(Processor):
|
|
11
|
+
"""Cast numerical columns to a floating-point dtype.
|
|
12
|
+
|
|
13
|
+
Args:
|
|
14
|
+
dtype: The floating-point dtype of the numerical columns.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
18
|
+
requires_fit = False
|
|
19
|
+
|
|
20
|
+
def __init__(self, dtype: torch.dtype) -> None:
|
|
21
|
+
super().__init__()
|
|
22
|
+
if not dtype.is_floating_point:
|
|
23
|
+
raise ValueError(f"Expected a floating-point dtype (got {dtype})")
|
|
24
|
+
self.dtype = dtype
|
|
25
|
+
|
|
26
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
27
|
+
dtype = self.dtype
|
|
28
|
+
if table.device.type == "mps" and dtype == torch.float64:
|
|
29
|
+
dtype = torch.float32 # MPS does not support float64.
|
|
30
|
+
return table.replace_blocks(numerical=table.numerical.to(dtype))
|
|
31
|
+
|
|
32
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
33
|
+
return f"{' ' * indent}{self.__class__.__name__}({self.dtype})"
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from sdm import Stype, TableTensor
|
|
5
|
+
from sdm.processing import Processor
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Clip(Processor):
|
|
9
|
+
"""Clamp numerical values to a fixed interval.
|
|
10
|
+
|
|
11
|
+
Values below ``min_value`` are set to ``min_value``, and values above
|
|
12
|
+
``max_value`` are set to ``max_value``. Values within the interval are
|
|
13
|
+
unchanged.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
min_value: Inclusive lower bound for every numerical value.
|
|
17
|
+
max_value: Inclusive upper bound for every numerical value.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
21
|
+
requires_fit = False
|
|
22
|
+
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
min_value: float,
|
|
26
|
+
max_value: float,
|
|
27
|
+
) -> None:
|
|
28
|
+
super().__init__()
|
|
29
|
+
if min_value > max_value:
|
|
30
|
+
raise ValueError(
|
|
31
|
+
"min_value must be less than or equal to max_value"
|
|
32
|
+
)
|
|
33
|
+
self.min_value = min_value
|
|
34
|
+
self.max_value = max_value
|
|
35
|
+
|
|
36
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
37
|
+
"""Clamp ``table`` to the configured interval."""
|
|
38
|
+
numerical = table.numerical.clamp(
|
|
39
|
+
min=self.min_value,
|
|
40
|
+
max=self.max_value,
|
|
41
|
+
)
|
|
42
|
+
return table.replace_blocks(numerical=numerical)
|
|
43
|
+
|
|
44
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
45
|
+
return (
|
|
46
|
+
f"{' ' * indent}{self.__class__.__name__}("
|
|
47
|
+
f"{self.min_value}, {self.max_value})"
|
|
48
|
+
)
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
|
|
6
|
+
from sdm import Stype, TableTensor
|
|
7
|
+
from sdm.processing import Processor
|
|
8
|
+
from sdm.processing.numerical._stats import _isfinite
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ClipSoft(Processor):
|
|
12
|
+
r"""Soft-clip numerical values toward ``±max_absolute_value``.
|
|
13
|
+
|
|
14
|
+
Values pass through
|
|
15
|
+
|
|
16
|
+
.. math::
|
|
17
|
+
|
|
18
|
+
\frac{x}{\sqrt{1 + (x / b)^{2}}}
|
|
19
|
+
|
|
20
|
+
which maps monotonically onto ``±max_absolute_value`` (``b``) without a
|
|
21
|
+
hard edge. NaN is preserved. ``±Inf`` maps to ``±max_absolute_value``.
|
|
22
|
+
Unlike :class:`~sdm.processing.ClipSigma`, the bound is fixed rather
|
|
23
|
+
than fitted from per-column statistics.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
max_absolute_value: Bound of the soft clip.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
30
|
+
requires_fit = False
|
|
31
|
+
|
|
32
|
+
def __init__(self, max_absolute_value: float) -> None:
|
|
33
|
+
super().__init__()
|
|
34
|
+
if max_absolute_value <= 0:
|
|
35
|
+
raise ValueError("max_absolute_value must be positive.")
|
|
36
|
+
self.max_absolute_value = max_absolute_value
|
|
37
|
+
|
|
38
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
39
|
+
numerical = table.numerical
|
|
40
|
+
assert not numerical.requires_grad
|
|
41
|
+
bound = self.max_absolute_value
|
|
42
|
+
unit = numerical.div(bound).abs_()
|
|
43
|
+
root = unit.square().add_(1).sqrt_()
|
|
44
|
+
unit.div_(root).masked_fill_(~_isfinite(root), 1.0)
|
|
45
|
+
del root
|
|
46
|
+
clipped = unit.mul_(bound).mul_(numerical.sign())
|
|
47
|
+
torch.where(numerical.isnan(), numerical, clipped, out=clipped)
|
|
48
|
+
return table.replace_blocks(numerical=clipped)
|
|
49
|
+
|
|
50
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
51
|
+
return (
|
|
52
|
+
f"{' ' * indent}{self.__class__.__name__}"
|
|
53
|
+
f"({self.max_absolute_value})"
|
|
54
|
+
)
|