structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import copy
|
|
5
|
+
from itertools import repeat
|
|
6
|
+
from typing import cast
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
from torch.nn import ModuleList
|
|
10
|
+
|
|
11
|
+
from sdm import EnsembleTable, Stype
|
|
12
|
+
from sdm.processing import (
|
|
13
|
+
EnsembleInvertibleMixin,
|
|
14
|
+
EnsembleProcessor,
|
|
15
|
+
InvertibleMixin,
|
|
16
|
+
Processor,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class EnsembleProcessorAdapter(EnsembleProcessor, EnsembleInvertibleMixin):
|
|
21
|
+
"""Adapt an ordinary processor to ensemble-aware processing.
|
|
22
|
+
|
|
23
|
+
The adapter turns a :class:`~sdm.processing.base.Processor` into an
|
|
24
|
+
:class:`~sdm.processing.ensemble.EnsembleProcessor`. It copies and fits the
|
|
25
|
+
processor separately for each group of compatible tables in an
|
|
26
|
+
:class:`~sdm.EnsembleTable`.
|
|
27
|
+
|
|
28
|
+
The wrapped processor must preserve row and leading dimensions as required
|
|
29
|
+
by the :class:`~sdm.processing.base.Processor` contract. A processor that
|
|
30
|
+
changes the ensemble structure must implement
|
|
31
|
+
:class:`~sdm.processing.ensemble.EnsembleProcessor` directly. Inverse
|
|
32
|
+
transformation requires the wrapped processor to implement
|
|
33
|
+
:class:`~sdm.processing.base.InvertibleMixin`.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
processor: Processor to fit separately for each ensemble table group.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __init__(self, processor: Processor) -> None:
|
|
40
|
+
super().__init__()
|
|
41
|
+
self.processor = processor
|
|
42
|
+
self.requires_fit = processor.requires_fit
|
|
43
|
+
|
|
44
|
+
self._processors: ModuleList[Processor] = ModuleList()
|
|
45
|
+
self._fitted_locations: tuple[tuple[int, int], ...] = ()
|
|
46
|
+
|
|
47
|
+
def get_extra_state(
|
|
48
|
+
self,
|
|
49
|
+
) -> tuple[int, tuple[tuple[int, int], ...]]:
|
|
50
|
+
r""":meta private:""" # noqa: D415
|
|
51
|
+
return len(self._processors), self._fitted_locations
|
|
52
|
+
|
|
53
|
+
def set_extra_state(self, state: object) -> None:
|
|
54
|
+
r""":meta private:""" # noqa: D415
|
|
55
|
+
num_processors, self._fitted_locations = cast(
|
|
56
|
+
tuple[int, tuple[tuple[int, int], ...]], state
|
|
57
|
+
)
|
|
58
|
+
self._processors = ModuleList(
|
|
59
|
+
[copy.deepcopy(self.processor) for _ in range(num_processors)]
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def handles_stypes(self) -> frozenset[Stype]:
|
|
64
|
+
r""":meta private:""" # noqa: D415
|
|
65
|
+
return self.processor.handles_stypes
|
|
66
|
+
|
|
67
|
+
def _fit_ensemble(
|
|
68
|
+
self,
|
|
69
|
+
ensemble_table: EnsembleTable,
|
|
70
|
+
*,
|
|
71
|
+
generator: torch.Generator | None = None,
|
|
72
|
+
) -> None:
|
|
73
|
+
self._fitted_locations = ensemble_table._locations
|
|
74
|
+
self._processors = ModuleList()
|
|
75
|
+
for group_id, group in enumerate(ensemble_table._iter_groups()):
|
|
76
|
+
if group_id == 0:
|
|
77
|
+
processor = self.processor
|
|
78
|
+
else:
|
|
79
|
+
processor = copy.deepcopy(self.processor)
|
|
80
|
+
processor.fit(group, generator=generator)
|
|
81
|
+
self._processors.append(processor)
|
|
82
|
+
|
|
83
|
+
def _fit_transform_ensemble(
|
|
84
|
+
self,
|
|
85
|
+
ensemble_table: EnsembleTable,
|
|
86
|
+
*,
|
|
87
|
+
generator: torch.Generator | None = None,
|
|
88
|
+
) -> EnsembleTable:
|
|
89
|
+
if not self.requires_fit:
|
|
90
|
+
return self._transform_ensemble(ensemble_table)
|
|
91
|
+
|
|
92
|
+
self._fitted_locations = ensemble_table._locations
|
|
93
|
+
outputs = []
|
|
94
|
+
self._processors = ModuleList()
|
|
95
|
+
for group_id, group in enumerate(ensemble_table._iter_groups()):
|
|
96
|
+
if group_id == 0:
|
|
97
|
+
processor = self.processor
|
|
98
|
+
else:
|
|
99
|
+
processor = copy.deepcopy(self.processor)
|
|
100
|
+
outputs.append(processor.fit_transform(group, generator=generator))
|
|
101
|
+
self._processors.append(processor)
|
|
102
|
+
return ensemble_table.replace_groups(outputs)
|
|
103
|
+
|
|
104
|
+
def _transform_ensemble(
|
|
105
|
+
self,
|
|
106
|
+
ensemble_table: EnsembleTable,
|
|
107
|
+
) -> EnsembleTable:
|
|
108
|
+
if self.requires_fit:
|
|
109
|
+
processors = self._aligned_processors(ensemble_table)
|
|
110
|
+
else:
|
|
111
|
+
processors = repeat(self.processor, ensemble_table.num_groups)
|
|
112
|
+
outputs = [
|
|
113
|
+
processor.transform(group)
|
|
114
|
+
for group, processor in zip(
|
|
115
|
+
ensemble_table._iter_groups(), processors, strict=True
|
|
116
|
+
)
|
|
117
|
+
]
|
|
118
|
+
return ensemble_table.replace_groups(outputs)
|
|
119
|
+
|
|
120
|
+
def _inverse_transform_ensemble(
|
|
121
|
+
self,
|
|
122
|
+
ensemble_table: EnsembleTable,
|
|
123
|
+
) -> EnsembleTable:
|
|
124
|
+
if self.requires_fit:
|
|
125
|
+
processors = self._aligned_processors(ensemble_table)
|
|
126
|
+
else:
|
|
127
|
+
processors = repeat(self.processor, ensemble_table.num_groups)
|
|
128
|
+
|
|
129
|
+
outputs = []
|
|
130
|
+
for group, processor in zip(
|
|
131
|
+
ensemble_table._iter_groups(), processors, strict=True
|
|
132
|
+
):
|
|
133
|
+
if not isinstance(processor, InvertibleMixin):
|
|
134
|
+
raise AttributeError(
|
|
135
|
+
f"{self.processor.__class__.__name__!r} object has no "
|
|
136
|
+
"attribute 'inverse_transform'"
|
|
137
|
+
)
|
|
138
|
+
outputs.append(processor.inverse_transform(group))
|
|
139
|
+
return ensemble_table.replace_groups(outputs)
|
|
140
|
+
|
|
141
|
+
def _aligned_processors(
|
|
142
|
+
self,
|
|
143
|
+
ensemble_table: EnsembleTable,
|
|
144
|
+
) -> tuple[Processor, ...]:
|
|
145
|
+
if ensemble_table._locations == self._fitted_locations:
|
|
146
|
+
return tuple(self._processors)
|
|
147
|
+
|
|
148
|
+
# One fitted processor per transform ensemble group. Multiple
|
|
149
|
+
# transform groups can share the same fitted processor.
|
|
150
|
+
processors: list[Processor | None] = [None] * ensemble_table.num_groups
|
|
151
|
+
# Current and fitted batch of every member sharing a fitted processor.
|
|
152
|
+
batches_by_processor: dict[int, list[tuple[int, int, int]]] = {}
|
|
153
|
+
for member_id, (group_id, batch_id) in enumerate(
|
|
154
|
+
ensemble_table._locations
|
|
155
|
+
):
|
|
156
|
+
fitted_group_id, fitted_batch_id = self._fitted_locations[
|
|
157
|
+
member_id
|
|
158
|
+
]
|
|
159
|
+
processor = self._processors[fitted_group_id]
|
|
160
|
+
if processors[group_id] is None:
|
|
161
|
+
processors[group_id] = processor
|
|
162
|
+
elif processors[group_id] is not processor:
|
|
163
|
+
raise RuntimeError(
|
|
164
|
+
"Cannot apply fitted processor state to an ensemble "
|
|
165
|
+
"group containing members from different fitted groups"
|
|
166
|
+
)
|
|
167
|
+
batches_by_processor.setdefault(fitted_group_id, []).append(
|
|
168
|
+
(group_id, batch_id, fitted_batch_id)
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
# A processor fitted on a stacked tensor table stores per-batch state.
|
|
172
|
+
# It can only operate on stacked tensor tables with the same layout,
|
|
173
|
+
# but not on splits or reordered batches.
|
|
174
|
+
for batches in batches_by_processor.values():
|
|
175
|
+
if len({fitted for _, _, fitted in batches}) > 1 and (
|
|
176
|
+
len({group for group, _, _ in batches}) > 1
|
|
177
|
+
or any(current != fitted for _, current, fitted in batches)
|
|
178
|
+
):
|
|
179
|
+
raise RuntimeError(
|
|
180
|
+
"Cannot apply position-dependent fitted processor state "
|
|
181
|
+
"after its ensemble group was split or reordered"
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
return tuple(cast(Processor, processor) for processor in processors)
|
|
185
|
+
|
|
186
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
187
|
+
return self.processor.__repr__(indent=indent)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from sdm import Stype, TableTensor
|
|
5
|
+
from sdm.processing import InvertibleMixin, Processor
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Identity(Processor, InvertibleMixin):
|
|
9
|
+
r"""Return inputs unchanged."""
|
|
10
|
+
|
|
11
|
+
handles_stypes = frozenset(Stype)
|
|
12
|
+
requires_fit = False
|
|
13
|
+
|
|
14
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
15
|
+
return table
|
|
16
|
+
|
|
17
|
+
def _inverse_transform(self, table: TableTensor) -> TableTensor:
|
|
18
|
+
return table
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from collections.abc import Mapping, Sequence
|
|
5
|
+
from typing import Literal
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
|
|
9
|
+
from sdm import EnsembleTable, Stype, StypeLike, TableTensor
|
|
10
|
+
from sdm.processing import EnsembleProcessor
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class SelectColumns(EnsembleProcessor):
|
|
14
|
+
r"""Select a subset of columns for each semantic type.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
max_columns: The maximum number of columns to keep per semantic type.
|
|
18
|
+
method: The column selection method.
|
|
19
|
+
``"first"`` keeps the first columns according to their order within
|
|
20
|
+
each semantic block. ``"round_robin"`` assigns each ensemble
|
|
21
|
+
member the next consecutive chunk, wrapping to the first column
|
|
22
|
+
after the last. For five columns and ``max_columns=2``, members
|
|
23
|
+
0, 1, and 2 receive columns (0, 1), (2, 3), and (4, 0).
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
handles_stypes = frozenset(Stype)
|
|
27
|
+
requires_fit = False
|
|
28
|
+
|
|
29
|
+
def __init__(
|
|
30
|
+
self,
|
|
31
|
+
max_columns: int,
|
|
32
|
+
method: Literal["first", "round_robin"] = "first",
|
|
33
|
+
) -> None:
|
|
34
|
+
super().__init__()
|
|
35
|
+
if max_columns <= 0:
|
|
36
|
+
raise ValueError("max_columns must be positive")
|
|
37
|
+
self.max_columns = max_columns
|
|
38
|
+
self.method = method
|
|
39
|
+
|
|
40
|
+
def _transform_ensemble(
|
|
41
|
+
self,
|
|
42
|
+
ensemble_table: EnsembleTable,
|
|
43
|
+
) -> EnsembleTable:
|
|
44
|
+
if all(
|
|
45
|
+
group.size(-1) <= self.max_columns
|
|
46
|
+
for group in ensemble_table._iter_groups()
|
|
47
|
+
):
|
|
48
|
+
return ensemble_table
|
|
49
|
+
|
|
50
|
+
if self.method == "first":
|
|
51
|
+
groups = []
|
|
52
|
+
for group in ensemble_table._iter_groups():
|
|
53
|
+
columns: Mapping[StypeLike, Sequence[str]] = {
|
|
54
|
+
stype: column_names[: self.max_columns]
|
|
55
|
+
for stype, column_names in group.columns.items()
|
|
56
|
+
}
|
|
57
|
+
blocks = {
|
|
58
|
+
stype: block[..., : self.max_columns]
|
|
59
|
+
for stype, block in group.items()
|
|
60
|
+
}
|
|
61
|
+
groups.append(group.__class__(columns=columns, **blocks))
|
|
62
|
+
return ensemble_table.replace_groups(groups)
|
|
63
|
+
|
|
64
|
+
assert self.method == "round_robin"
|
|
65
|
+
tables: list[TableTensor] = []
|
|
66
|
+
for member_id in range(len(ensemble_table)):
|
|
67
|
+
table = ensemble_table[member_id]
|
|
68
|
+
columns: dict[StypeLike, tuple[str, ...]] = {}
|
|
69
|
+
blocks = {}
|
|
70
|
+
for stype, block in table.items():
|
|
71
|
+
column_names = table.columns[stype]
|
|
72
|
+
num_columns = len(column_names)
|
|
73
|
+
# Empty blocks are passed through unchanged.
|
|
74
|
+
if num_columns == 0:
|
|
75
|
+
columns[stype] = column_names
|
|
76
|
+
blocks[stype] = block
|
|
77
|
+
continue
|
|
78
|
+
chunk_size = min(self.max_columns, num_columns)
|
|
79
|
+
start = member_id * chunk_size % num_columns
|
|
80
|
+
first_count = min(chunk_size, num_columns - start)
|
|
81
|
+
wrap_count = chunk_size - first_count
|
|
82
|
+
# Consecutive chunk from ``start``, wrapping past the last
|
|
83
|
+
# column back to the first.
|
|
84
|
+
columns[stype] = (
|
|
85
|
+
column_names[start : start + first_count]
|
|
86
|
+
+ column_names[:wrap_count]
|
|
87
|
+
)
|
|
88
|
+
if wrap_count == 0:
|
|
89
|
+
blocks[stype] = block[..., start : start + first_count]
|
|
90
|
+
continue
|
|
91
|
+
blocks[stype] = torch.cat(
|
|
92
|
+
[
|
|
93
|
+
block[..., start : start + first_count],
|
|
94
|
+
block[..., :wrap_count],
|
|
95
|
+
],
|
|
96
|
+
dim=-1,
|
|
97
|
+
)
|
|
98
|
+
tables.append(
|
|
99
|
+
table.__class__(
|
|
100
|
+
columns=columns,
|
|
101
|
+
**blocks,
|
|
102
|
+
)
|
|
103
|
+
)
|
|
104
|
+
return ensemble_table.replace_tables(
|
|
105
|
+
tables=tables,
|
|
106
|
+
member_table_ids=range(len(tables)),
|
|
107
|
+
)
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from collections.abc import Iterable, Iterator
|
|
5
|
+
from typing import Self, cast
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
|
|
9
|
+
from sdm import EnsembleTable, Stype, TableTensor
|
|
10
|
+
from sdm.processing import (
|
|
11
|
+
EnsembleInvertibleMixin,
|
|
12
|
+
EnsembleProcessor,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Sequential(EnsembleProcessor, EnsembleInvertibleMixin):
|
|
17
|
+
r"""Apply processors and callables to a table in sequence.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
args: Sequence of :class:`Processor` instances or callables.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(self, *args: object) -> None:
|
|
24
|
+
super().__init__()
|
|
25
|
+
self.requires_fit = False
|
|
26
|
+
self.extend(args)
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def handles_stypes(self) -> frozenset[Stype]:
|
|
30
|
+
r""":meta private:""" # noqa: D415
|
|
31
|
+
if len(self) == 0:
|
|
32
|
+
return frozenset(Stype)
|
|
33
|
+
return frozenset(
|
|
34
|
+
stype for child in self for stype in child.handles_stypes
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
def append(self, processor: object) -> Self:
|
|
38
|
+
r"""Append a processor or callable to this sequence.
|
|
39
|
+
|
|
40
|
+
Args:
|
|
41
|
+
processor: The processor to append.
|
|
42
|
+
"""
|
|
43
|
+
processor = EnsembleProcessor.as_processor(processor)
|
|
44
|
+
if isinstance(processor, Sequential):
|
|
45
|
+
for child in processor.children():
|
|
46
|
+
self.add_module(str(len(self)), child)
|
|
47
|
+
else:
|
|
48
|
+
self.add_module(str(len(self)), processor)
|
|
49
|
+
|
|
50
|
+
self.requires_fit = any(child.requires_fit for child in self)
|
|
51
|
+
self._fitted = False
|
|
52
|
+
return self
|
|
53
|
+
|
|
54
|
+
def prepend(self, processor: object) -> Self:
|
|
55
|
+
r"""Prepend a processor or callable to this sequence.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
processor: The processor to prepend.
|
|
59
|
+
"""
|
|
60
|
+
processor = EnsembleProcessor.as_processor(processor)
|
|
61
|
+
|
|
62
|
+
if isinstance(processor, Sequential):
|
|
63
|
+
processors = tuple(processor.children())
|
|
64
|
+
else:
|
|
65
|
+
processors = (processor,)
|
|
66
|
+
|
|
67
|
+
processors = (*processors, *self.children())
|
|
68
|
+
|
|
69
|
+
self._modules.clear()
|
|
70
|
+
for child in processors:
|
|
71
|
+
self.add_module(str(len(self)), child)
|
|
72
|
+
|
|
73
|
+
self.requires_fit = any(child.requires_fit for child in self)
|
|
74
|
+
self._fitted = False
|
|
75
|
+
return self
|
|
76
|
+
|
|
77
|
+
def extend(self, processors: Iterable[object]) -> Self:
|
|
78
|
+
r"""Append multiple processors or callables to this sequence.
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
processors: The processors to append.
|
|
82
|
+
"""
|
|
83
|
+
for processor in processors:
|
|
84
|
+
self.append(processor)
|
|
85
|
+
return self
|
|
86
|
+
|
|
87
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
88
|
+
out = table
|
|
89
|
+
for child in self:
|
|
90
|
+
out = child.transform(out)
|
|
91
|
+
return out
|
|
92
|
+
|
|
93
|
+
def _fit_ensemble(
|
|
94
|
+
self,
|
|
95
|
+
ensemble_table: EnsembleTable,
|
|
96
|
+
*,
|
|
97
|
+
generator: torch.Generator | None = None,
|
|
98
|
+
) -> None:
|
|
99
|
+
out = ensemble_table
|
|
100
|
+
for i, child in enumerate(self):
|
|
101
|
+
if i < len(self) - 1:
|
|
102
|
+
out = child.fit_transform_ensemble(out, generator=generator)
|
|
103
|
+
else:
|
|
104
|
+
child.fit_ensemble(out, generator=generator)
|
|
105
|
+
|
|
106
|
+
def _fit_transform_ensemble(
|
|
107
|
+
self,
|
|
108
|
+
ensemble_table: EnsembleTable,
|
|
109
|
+
*,
|
|
110
|
+
generator: torch.Generator | None = None,
|
|
111
|
+
) -> EnsembleTable:
|
|
112
|
+
out = ensemble_table
|
|
113
|
+
for child in self:
|
|
114
|
+
out = child.fit_transform_ensemble(out, generator=generator)
|
|
115
|
+
return out
|
|
116
|
+
|
|
117
|
+
def _transform_ensemble(
|
|
118
|
+
self,
|
|
119
|
+
ensemble_table: EnsembleTable,
|
|
120
|
+
) -> EnsembleTable:
|
|
121
|
+
out = ensemble_table
|
|
122
|
+
for child in self:
|
|
123
|
+
out = child.transform_ensemble(out)
|
|
124
|
+
return out
|
|
125
|
+
|
|
126
|
+
def _inverse_transform_ensemble(
|
|
127
|
+
self,
|
|
128
|
+
ensemble_table: EnsembleTable,
|
|
129
|
+
) -> EnsembleTable:
|
|
130
|
+
out = ensemble_table
|
|
131
|
+
for child in reversed(list(self)):
|
|
132
|
+
if not isinstance(child, EnsembleInvertibleMixin):
|
|
133
|
+
raise AttributeError(
|
|
134
|
+
f"{child.__class__.__name__!r} object has no "
|
|
135
|
+
"attribute 'inverse_transform_ensemble'"
|
|
136
|
+
)
|
|
137
|
+
out = child.inverse_transform_ensemble(out)
|
|
138
|
+
return out
|
|
139
|
+
|
|
140
|
+
def __iter__(self) -> Iterator[EnsembleProcessor]:
|
|
141
|
+
return cast(Iterator[EnsembleProcessor], self.children())
|
|
142
|
+
|
|
143
|
+
def __len__(self) -> int:
|
|
144
|
+
return len(self._modules)
|
|
145
|
+
|
|
146
|
+
def __iadd__(self, other: object) -> Self:
|
|
147
|
+
self.append(EnsembleProcessor.as_processor(other))
|
|
148
|
+
return self
|
|
149
|
+
|
|
150
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
151
|
+
if len(self) == 0:
|
|
152
|
+
return super().__repr__(indent=indent)
|
|
153
|
+
reprs = "".join(
|
|
154
|
+
[f"{child.__repr__(indent=indent + 2)},\n" for child in self]
|
|
155
|
+
)
|
|
156
|
+
return (
|
|
157
|
+
f"{' ' * indent}{self.__class__.__name__}(\n{reprs}{' ' * indent})"
|
|
158
|
+
)
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from typing import Literal, cast
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
from torch import Tensor
|
|
8
|
+
|
|
9
|
+
from sdm import EnsembleTable, Stype, TableTensor
|
|
10
|
+
from sdm.nn._buffer import BufferList
|
|
11
|
+
from sdm.processing import EnsembleInvertibleMixin, EnsembleProcessor
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ShuffleColumns(EnsembleProcessor, EnsembleInvertibleMixin):
|
|
15
|
+
"""Permute numerical feature columns and their names.
|
|
16
|
+
|
|
17
|
+
Convert non-numerical feature stypes before this step, for example with
|
|
18
|
+
:class:`~sdm.processing.ToNumerical`.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
method: Permutation strategy. ``"random"`` draws independent
|
|
22
|
+
permutations, and ``"latin"`` draws coupled Latin
|
|
23
|
+
permutations.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
27
|
+
requires_fit = True
|
|
28
|
+
|
|
29
|
+
def __init__(
|
|
30
|
+
self,
|
|
31
|
+
method: Literal["random", "latin"] = "random",
|
|
32
|
+
) -> None:
|
|
33
|
+
super().__init__()
|
|
34
|
+
self.method = method
|
|
35
|
+
self._permutations: BufferList[Tensor] = BufferList()
|
|
36
|
+
self._host_permutations: tuple[tuple[int, ...], ...] = ()
|
|
37
|
+
|
|
38
|
+
def get_extra_state(self) -> tuple[tuple[int, ...], ...]:
|
|
39
|
+
r""":meta private:""" # noqa: D415
|
|
40
|
+
return self._host_permutations
|
|
41
|
+
|
|
42
|
+
def set_extra_state(self, state: object) -> None:
|
|
43
|
+
r""":meta private:""" # noqa: D415
|
|
44
|
+
self._host_permutations = cast(tuple[tuple[int, ...], ...], state)
|
|
45
|
+
|
|
46
|
+
def _fit_ensemble(
|
|
47
|
+
self,
|
|
48
|
+
ensemble_table: EnsembleTable,
|
|
49
|
+
*,
|
|
50
|
+
generator: torch.Generator | None = None,
|
|
51
|
+
) -> None:
|
|
52
|
+
device = next(ensemble_table._iter_groups()).device
|
|
53
|
+
widths = [
|
|
54
|
+
ensemble_table[member_id].numerical.size(-1)
|
|
55
|
+
for member_id in range(len(ensemble_table))
|
|
56
|
+
]
|
|
57
|
+
permutations = []
|
|
58
|
+
if self.method == "latin":
|
|
59
|
+
max_features = max(widths)
|
|
60
|
+
base_rank = torch.randperm(
|
|
61
|
+
max_features,
|
|
62
|
+
generator=generator,
|
|
63
|
+
device=device,
|
|
64
|
+
)
|
|
65
|
+
row_rank = torch.randperm(
|
|
66
|
+
max_features,
|
|
67
|
+
generator=generator,
|
|
68
|
+
device=device,
|
|
69
|
+
)
|
|
70
|
+
for member_id, n_features in enumerate(widths):
|
|
71
|
+
if n_features <= 1:
|
|
72
|
+
permutation = torch.arange(n_features, device=device)
|
|
73
|
+
else:
|
|
74
|
+
base = base_rank
|
|
75
|
+
rows = row_rank
|
|
76
|
+
if n_features < max_features:
|
|
77
|
+
base = base[base < n_features]
|
|
78
|
+
rows = rows[rows < n_features]
|
|
79
|
+
pattern = member_id % n_features
|
|
80
|
+
permutation = base[(pattern - rows) % n_features]
|
|
81
|
+
permutations.append(permutation)
|
|
82
|
+
else:
|
|
83
|
+
assert self.method == "random"
|
|
84
|
+
for n_features in widths:
|
|
85
|
+
if n_features <= 1:
|
|
86
|
+
permutation = torch.arange(n_features, device=device)
|
|
87
|
+
else:
|
|
88
|
+
permutation = torch.randperm(
|
|
89
|
+
n_features,
|
|
90
|
+
generator=generator,
|
|
91
|
+
device=device,
|
|
92
|
+
)
|
|
93
|
+
permutations.append(permutation)
|
|
94
|
+
|
|
95
|
+
self._permutations = BufferList(permutations)
|
|
96
|
+
# TODO: Add to EnsembleTable directly.
|
|
97
|
+
member_ids_by_group: list[list[int]] = [
|
|
98
|
+
[] for _ in range(ensemble_table.num_groups)
|
|
99
|
+
]
|
|
100
|
+
for member_id, (group_id, _) in enumerate(ensemble_table._locations):
|
|
101
|
+
member_ids_by_group[group_id].append(member_id)
|
|
102
|
+
host_permutations = [()] * len(permutations)
|
|
103
|
+
for member_ids in member_ids_by_group:
|
|
104
|
+
host_rows = torch.stack(
|
|
105
|
+
[permutations[member_id] for member_id in member_ids]
|
|
106
|
+
).tolist()
|
|
107
|
+
for member_id, host_row in zip(member_ids, host_rows, strict=True):
|
|
108
|
+
host_permutations[member_id] = tuple(host_row)
|
|
109
|
+
self._host_permutations = tuple(host_permutations)
|
|
110
|
+
|
|
111
|
+
def _transform_ensemble(
|
|
112
|
+
self,
|
|
113
|
+
ensemble_table: EnsembleTable,
|
|
114
|
+
) -> EnsembleTable:
|
|
115
|
+
if len(self._permutations) != len(ensemble_table):
|
|
116
|
+
raise RuntimeError(
|
|
117
|
+
f"{self.__class__.__name__!r} was fitted with "
|
|
118
|
+
f"{len(self._permutations)} ensemble members, but got "
|
|
119
|
+
f"{len(ensemble_table)}."
|
|
120
|
+
)
|
|
121
|
+
tables: list[TableTensor] = []
|
|
122
|
+
for member_id in range(len(ensemble_table)):
|
|
123
|
+
table = ensemble_table[member_id]
|
|
124
|
+
host_permutation = self._host_permutations[member_id]
|
|
125
|
+
shuffled = table.__class__(
|
|
126
|
+
columns={
|
|
127
|
+
Stype.numerical: tuple(
|
|
128
|
+
table.columns[Stype.numerical][i]
|
|
129
|
+
for i in host_permutation
|
|
130
|
+
)
|
|
131
|
+
},
|
|
132
|
+
numerical=table.numerical.index_select(
|
|
133
|
+
-1, self._permutations[member_id]
|
|
134
|
+
),
|
|
135
|
+
)
|
|
136
|
+
tables.append(
|
|
137
|
+
cast(
|
|
138
|
+
TableTensor,
|
|
139
|
+
torch.cat(
|
|
140
|
+
(table.drop_stypes(Stype.numerical), shuffled),
|
|
141
|
+
dim=-1,
|
|
142
|
+
),
|
|
143
|
+
)
|
|
144
|
+
)
|
|
145
|
+
return EnsembleTable.from_tables(
|
|
146
|
+
tables=tables,
|
|
147
|
+
member_table_ids=range(len(tables)),
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
def _inverse_transform_ensemble(
|
|
151
|
+
self,
|
|
152
|
+
ensemble_table: EnsembleTable,
|
|
153
|
+
) -> EnsembleTable:
|
|
154
|
+
if len(self._permutations) != len(ensemble_table):
|
|
155
|
+
raise RuntimeError(
|
|
156
|
+
f"{self.__class__.__name__!r} was fitted with "
|
|
157
|
+
f"{len(self._permutations)} ensemble members, but got "
|
|
158
|
+
f"{len(ensemble_table)}."
|
|
159
|
+
)
|
|
160
|
+
tables: list[TableTensor] = []
|
|
161
|
+
for member_id in range(len(ensemble_table)):
|
|
162
|
+
table = ensemble_table[member_id]
|
|
163
|
+
host_permutation = self._host_permutations[member_id]
|
|
164
|
+
inverse_host_permutation = [0] * len(host_permutation)
|
|
165
|
+
for destination, source in enumerate(host_permutation):
|
|
166
|
+
inverse_host_permutation[source] = destination
|
|
167
|
+
shuffled = table.__class__(
|
|
168
|
+
columns={
|
|
169
|
+
Stype.numerical: tuple(
|
|
170
|
+
table.columns[Stype.numerical][i]
|
|
171
|
+
for i in inverse_host_permutation
|
|
172
|
+
)
|
|
173
|
+
},
|
|
174
|
+
numerical=table.numerical.index_select(
|
|
175
|
+
-1, self._permutations[member_id].argsort()
|
|
176
|
+
),
|
|
177
|
+
)
|
|
178
|
+
tables.append(
|
|
179
|
+
cast(
|
|
180
|
+
TableTensor,
|
|
181
|
+
torch.cat(
|
|
182
|
+
(table.drop_stypes(Stype.numerical), shuffled),
|
|
183
|
+
dim=-1,
|
|
184
|
+
),
|
|
185
|
+
)
|
|
186
|
+
)
|
|
187
|
+
return EnsembleTable.from_tables(
|
|
188
|
+
tables=tables,
|
|
189
|
+
member_table_ids=range(len(tables)),
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
193
|
+
return (
|
|
194
|
+
f"{' ' * indent}{self.__class__.__name__}(method={self.method!r})"
|
|
195
|
+
)
|