structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
|
|
6
|
+
from sdm import CategoricalTensor, Stype, TableTensor
|
|
7
|
+
from sdm.nn._buffer import BufferList
|
|
8
|
+
from sdm.processing import Processor
|
|
9
|
+
from sdm.processing.categorical._categorical import (
|
|
10
|
+
_check_categorical_codes,
|
|
11
|
+
_check_categories,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ImputeMode(Processor):
|
|
16
|
+
"""Replace missing categorical values with fitted per-column modes.
|
|
17
|
+
|
|
18
|
+
Negative category codes are missing values. The fitted fill value is
|
|
19
|
+
the most frequent observed category code learned independently for every
|
|
20
|
+
categorical column and applied without changing its category vocabulary.
|
|
21
|
+
Ties select the lowest category code.
|
|
22
|
+
|
|
23
|
+
Transform inputs must use the fitted per-column category vocabularies.
|
|
24
|
+
Use :class:`~sdm.processing.AlignCategories` before this processor when
|
|
25
|
+
training and transform inputs were tensorized independently.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
handles_stypes = frozenset({Stype.categorical})
|
|
29
|
+
requires_fit = True
|
|
30
|
+
|
|
31
|
+
def __init__(self) -> None:
|
|
32
|
+
super().__init__()
|
|
33
|
+
# TODO: Add a separate processor that encodes missing values as their
|
|
34
|
+
# own category instead of imputing an observed one.
|
|
35
|
+
self._categories: BufferList[torch.Tensor] = BufferList()
|
|
36
|
+
self.register_buffer(
|
|
37
|
+
"_fill_values",
|
|
38
|
+
torch.empty(0, dtype=torch.long),
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
def _fit(
|
|
42
|
+
self,
|
|
43
|
+
table: TableTensor,
|
|
44
|
+
*,
|
|
45
|
+
generator: torch.Generator | None = None,
|
|
46
|
+
) -> None:
|
|
47
|
+
data = table.categorical
|
|
48
|
+
_check_categorical_codes(table)
|
|
49
|
+
fill_values: list[torch.Tensor] = []
|
|
50
|
+
columns = table.columns[Stype.categorical]
|
|
51
|
+
observed_mask = data.isfinite()
|
|
52
|
+
for index, category in enumerate(table.categorical.categories):
|
|
53
|
+
codes = data[..., index] # [*batch, n_samples]
|
|
54
|
+
observed = observed_mask[..., index]
|
|
55
|
+
if not bool(observed.any(dim=-1).all()):
|
|
56
|
+
raise ValueError(
|
|
57
|
+
"Cannot fit 'ImputeMode' because categorical "
|
|
58
|
+
f"column {columns[index]!r} has no observed values."
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# Accumulate category counts per batch.
|
|
62
|
+
counts = torch.zeros(
|
|
63
|
+
(*codes.shape[:-1], category.numel()),
|
|
64
|
+
dtype=torch.long,
|
|
65
|
+
device=codes.device,
|
|
66
|
+
)
|
|
67
|
+
counts.scatter_add_(
|
|
68
|
+
-1,
|
|
69
|
+
codes.clamp_min(0).to(torch.long),
|
|
70
|
+
observed.to(torch.long),
|
|
71
|
+
)
|
|
72
|
+
fill_values.append(counts.argmax(dim=-1, keepdim=True))
|
|
73
|
+
|
|
74
|
+
self._fill_values = (
|
|
75
|
+
torch.stack(fill_values, dim=-1)
|
|
76
|
+
if len(fill_values) > 0
|
|
77
|
+
else torch.empty(0, dtype=torch.long, device=data.device)
|
|
78
|
+
)
|
|
79
|
+
self._categories = BufferList(table.categorical.categories)
|
|
80
|
+
|
|
81
|
+
def _fit_transform(
|
|
82
|
+
self,
|
|
83
|
+
table: TableTensor,
|
|
84
|
+
*,
|
|
85
|
+
generator: torch.Generator | None = None,
|
|
86
|
+
) -> TableTensor:
|
|
87
|
+
self._fit(table, generator=generator)
|
|
88
|
+
return self._replace_missing(table)
|
|
89
|
+
|
|
90
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
91
|
+
_check_categories(table, self._categories)
|
|
92
|
+
_check_categorical_codes(table)
|
|
93
|
+
return self._replace_missing(table)
|
|
94
|
+
|
|
95
|
+
def _replace_missing(self, table: TableTensor) -> TableTensor:
|
|
96
|
+
code = table.categorical.where(
|
|
97
|
+
table.categorical.isfinite(),
|
|
98
|
+
self._fill_values.to(
|
|
99
|
+
dtype=table.categorical.dtype,
|
|
100
|
+
device=table.categorical.device,
|
|
101
|
+
),
|
|
102
|
+
)
|
|
103
|
+
categorical = CategoricalTensor(
|
|
104
|
+
code=code,
|
|
105
|
+
categories=table.categorical.categories,
|
|
106
|
+
)
|
|
107
|
+
return table.replace_blocks(categorical=categorical)
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from typing import Literal, cast
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
from torch import Tensor
|
|
8
|
+
|
|
9
|
+
from sdm import CategoricalTensor, EnsembleTable, Stype, TableTensor
|
|
10
|
+
from sdm.nn._buffer import BufferList
|
|
11
|
+
from sdm.processing import EnsembleProcessor
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ShuffleCategories(EnsembleProcessor):
|
|
15
|
+
"""Permute the integer codes of categorical columns.
|
|
16
|
+
|
|
17
|
+
One permutation per categorical column is drawn when the processor is
|
|
18
|
+
fitted. Codes and their corresponding category vectors are permuted
|
|
19
|
+
together so decoded values remain unchanged. Negative codes represent
|
|
20
|
+
missing values and are preserved unchanged. Only categorical columns
|
|
21
|
+
are supported; use :class:`~sdm.processing.StypeDispatch` to apply this
|
|
22
|
+
processor to the categorical block of a mixed feature table.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
method: Permutation strategy. ``"random"`` remaps codes
|
|
26
|
+
with an independent permutation. ``"shift"`` uses a cyclic
|
|
27
|
+
offset, cycling through every offset in random order.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
handles_stypes = frozenset({Stype.categorical})
|
|
31
|
+
requires_fit = True
|
|
32
|
+
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
method: Literal["shift", "random"] = "random",
|
|
36
|
+
) -> None:
|
|
37
|
+
super().__init__()
|
|
38
|
+
self.method = method
|
|
39
|
+
self._permutations: BufferList[BufferList[Tensor]] = BufferList()
|
|
40
|
+
self._permutation_ids: tuple[int, ...] = ()
|
|
41
|
+
|
|
42
|
+
def get_extra_state(self) -> tuple[int, ...]:
|
|
43
|
+
r""":meta private:""" # noqa: D415
|
|
44
|
+
return self._permutation_ids
|
|
45
|
+
|
|
46
|
+
def set_extra_state(self, state: object) -> None:
|
|
47
|
+
r""":meta private:""" # noqa: D415
|
|
48
|
+
self._permutation_ids = cast(tuple[int, ...], state)
|
|
49
|
+
|
|
50
|
+
def _draw_permutations(
|
|
51
|
+
self,
|
|
52
|
+
table: TableTensor,
|
|
53
|
+
*,
|
|
54
|
+
unused_offsets: dict[tuple[str, int], Tensor],
|
|
55
|
+
generator: torch.Generator | None = None,
|
|
56
|
+
) -> list[Tensor]:
|
|
57
|
+
device = table.categorical.device
|
|
58
|
+
permutations: list[Tensor] = []
|
|
59
|
+
for column_index, category in enumerate(table.categorical.categories):
|
|
60
|
+
n_classes = category.numel()
|
|
61
|
+
if n_classes <= 1:
|
|
62
|
+
permutation = torch.arange(n_classes, device=device)
|
|
63
|
+
elif self.method == "shift":
|
|
64
|
+
column_name = table.columns[Stype.categorical][column_index]
|
|
65
|
+
key = (column_name, n_classes)
|
|
66
|
+
offsets = unused_offsets.get(key)
|
|
67
|
+
if offsets is None or offsets.numel() == 0:
|
|
68
|
+
offsets = torch.randperm(
|
|
69
|
+
n_classes,
|
|
70
|
+
generator=generator,
|
|
71
|
+
device=device,
|
|
72
|
+
)
|
|
73
|
+
offset = offsets[:1]
|
|
74
|
+
unused_offsets[key] = offsets[1:]
|
|
75
|
+
permutation = (
|
|
76
|
+
torch.arange(n_classes, device=device) - offset
|
|
77
|
+
) % n_classes
|
|
78
|
+
else:
|
|
79
|
+
assert self.method == "random"
|
|
80
|
+
permutation = torch.randperm(
|
|
81
|
+
n_classes,
|
|
82
|
+
generator=generator,
|
|
83
|
+
device=device,
|
|
84
|
+
)
|
|
85
|
+
permutations.append(permutation)
|
|
86
|
+
return permutations
|
|
87
|
+
|
|
88
|
+
def _fit_ensemble(
|
|
89
|
+
self,
|
|
90
|
+
ensemble_table: EnsembleTable,
|
|
91
|
+
*,
|
|
92
|
+
generator: torch.Generator | None = None,
|
|
93
|
+
) -> None:
|
|
94
|
+
|
|
95
|
+
permutations_by_id = []
|
|
96
|
+
permutation_ids = []
|
|
97
|
+
permutation_id_by_key: dict[tuple[tuple[int, ...], ...], int] = {}
|
|
98
|
+
|
|
99
|
+
# (column name, n_classes) -> unused cyclic offsets for that column.
|
|
100
|
+
unused_offsets: dict[tuple[str, int], Tensor] = {}
|
|
101
|
+
for member_id in range(len(ensemble_table)):
|
|
102
|
+
permutations = self._draw_permutations(
|
|
103
|
+
ensemble_table[member_id],
|
|
104
|
+
unused_offsets=unused_offsets,
|
|
105
|
+
generator=generator,
|
|
106
|
+
)
|
|
107
|
+
key = tuple(
|
|
108
|
+
tuple(permutation.tolist()) for permutation in permutations
|
|
109
|
+
)
|
|
110
|
+
permutation_id = permutation_id_by_key.get(key)
|
|
111
|
+
if permutation_id is None:
|
|
112
|
+
permutation_id = len(permutations_by_id)
|
|
113
|
+
permutation_id_by_key[key] = permutation_id
|
|
114
|
+
permutations_by_id.append(permutations)
|
|
115
|
+
permutation_ids.append(permutation_id)
|
|
116
|
+
|
|
117
|
+
self._permutations = BufferList(
|
|
118
|
+
BufferList(permutations) for permutations in permutations_by_id
|
|
119
|
+
)
|
|
120
|
+
self._permutation_ids = tuple(permutation_ids)
|
|
121
|
+
|
|
122
|
+
def _transform_ensemble(
|
|
123
|
+
self,
|
|
124
|
+
ensemble_table: EnsembleTable,
|
|
125
|
+
) -> EnsembleTable:
|
|
126
|
+
if len(self._permutation_ids) != len(ensemble_table):
|
|
127
|
+
raise RuntimeError(
|
|
128
|
+
"ShuffleCategories must be fitted with the same number of "
|
|
129
|
+
"ensemble members before transform."
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
member_ids_by_permutation: dict[int, list[int]] = {}
|
|
133
|
+
for member_id, permutation_id in enumerate(self._permutation_ids):
|
|
134
|
+
member_ids_by_permutation.setdefault(permutation_id, []).append(
|
|
135
|
+
member_id
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
if len(member_ids_by_permutation) == len(ensemble_table):
|
|
139
|
+
member_tables: list[TableTensor] = []
|
|
140
|
+
for member_id, permutation_id in enumerate(self._permutation_ids):
|
|
141
|
+
permutations = self._permutations[permutation_id]
|
|
142
|
+
member_tables.append(
|
|
143
|
+
self._permute(
|
|
144
|
+
ensemble_table[member_id],
|
|
145
|
+
permutations,
|
|
146
|
+
)
|
|
147
|
+
)
|
|
148
|
+
return EnsembleTable.from_tables(
|
|
149
|
+
tables=member_tables,
|
|
150
|
+
member_table_ids=range(len(ensemble_table)),
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
outputs: dict[int, EnsembleTable] = {}
|
|
154
|
+
for permutation_id, member_ids in member_ids_by_permutation.items():
|
|
155
|
+
selected = ensemble_table[member_ids]
|
|
156
|
+
permutations = self._permutations[permutation_id]
|
|
157
|
+
outputs[permutation_id] = selected.replace_groups(
|
|
158
|
+
[
|
|
159
|
+
self._permute(group, permutations)
|
|
160
|
+
for group in selected._iter_groups()
|
|
161
|
+
]
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
output_tables = []
|
|
165
|
+
member_ids = []
|
|
166
|
+
next_member_id_by_permutation: dict[int, int] = {}
|
|
167
|
+
for permutation_id in self._permutation_ids:
|
|
168
|
+
output_tables.append(outputs[permutation_id])
|
|
169
|
+
member_id = next_member_id_by_permutation.get(permutation_id, 0)
|
|
170
|
+
member_ids.append(member_id)
|
|
171
|
+
next_member_id_by_permutation[permutation_id] = member_id + 1
|
|
172
|
+
|
|
173
|
+
return ensemble_table.gather_members(
|
|
174
|
+
tables=output_tables,
|
|
175
|
+
member_ids=member_ids,
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
@staticmethod
|
|
179
|
+
def _permute(
|
|
180
|
+
table: TableTensor,
|
|
181
|
+
permutations: BufferList[Tensor],
|
|
182
|
+
) -> TableTensor:
|
|
183
|
+
code = table.categorical.code.clone()
|
|
184
|
+
valid_mask = table.categorical.isfinite()
|
|
185
|
+
categories: list[Tensor] = []
|
|
186
|
+
for index, (category, permutation) in enumerate(
|
|
187
|
+
zip(table.categorical.categories, permutations, strict=True)
|
|
188
|
+
):
|
|
189
|
+
codes = code[..., index]
|
|
190
|
+
valid = valid_mask[..., index]
|
|
191
|
+
valid_codes = codes[valid].to(torch.long)
|
|
192
|
+
codes[valid] = permutation[valid_codes].to(codes.dtype)
|
|
193
|
+
categories.append(category[permutation.argsort()])
|
|
194
|
+
|
|
195
|
+
categorical = CategoricalTensor(
|
|
196
|
+
code=code,
|
|
197
|
+
categories=categories,
|
|
198
|
+
)
|
|
199
|
+
return table.replace_blocks(categorical=categorical)
|
|
200
|
+
|
|
201
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
202
|
+
return (
|
|
203
|
+
f"{' ' * indent}{self.__class__.__name__}(method={self.method!r})"
|
|
204
|
+
)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Cross-stype processors."""
|
|
5
|
+
|
|
6
|
+
from sdm.processing.common.identity import Identity
|
|
7
|
+
from sdm.processing.common.callable import Callable
|
|
8
|
+
from sdm.processing.common.sequential import Sequential
|
|
9
|
+
from sdm.processing.common.drop import DropStypes
|
|
10
|
+
from sdm.processing.common.stype import StypeDispatch
|
|
11
|
+
from sdm.processing.common.task import TaskDispatch
|
|
12
|
+
from sdm.processing.common.table import TableDispatch
|
|
13
|
+
from sdm.processing.common.ensemble import EnsembleProcessorAdapter
|
|
14
|
+
from sdm.processing.common.choice import Choice
|
|
15
|
+
from sdm.processing.common.to_numerical import ToNumerical
|
|
16
|
+
from sdm.processing.common.shuffle import ShuffleColumns
|
|
17
|
+
from sdm.processing.common.select import SelectColumns
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"Identity",
|
|
21
|
+
"Callable",
|
|
22
|
+
"Sequential",
|
|
23
|
+
"DropStypes",
|
|
24
|
+
"StypeDispatch",
|
|
25
|
+
"TaskDispatch",
|
|
26
|
+
"TableDispatch",
|
|
27
|
+
"EnsembleProcessorAdapter",
|
|
28
|
+
"Choice",
|
|
29
|
+
"ToNumerical",
|
|
30
|
+
"ShuffleColumns",
|
|
31
|
+
"SelectColumns",
|
|
32
|
+
]
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
|
|
6
|
+
from sdm import Stype, TableTensor
|
|
7
|
+
from sdm.processing import Processor
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Callable(Processor):
|
|
11
|
+
"""Adapt a stateless callable to the :class:`Processor` interface."""
|
|
12
|
+
|
|
13
|
+
handles_stypes = frozenset(Stype)
|
|
14
|
+
requires_fit = False
|
|
15
|
+
|
|
16
|
+
def __init__(self, function: Callable[[TableTensor], TableTensor]) -> None:
|
|
17
|
+
super().__init__()
|
|
18
|
+
self.function = function
|
|
19
|
+
|
|
20
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
21
|
+
return self.function(table)
|
|
22
|
+
|
|
23
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
24
|
+
name = getattr(
|
|
25
|
+
self.function,
|
|
26
|
+
"__name__",
|
|
27
|
+
self.function.__class__.__name__,
|
|
28
|
+
)
|
|
29
|
+
return f"{' ' * indent}{self.__class__.__name__}({name})"
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from collections.abc import Iterator
|
|
5
|
+
from typing import Literal, cast
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
from torch.nn import ModuleList
|
|
9
|
+
|
|
10
|
+
from sdm import EnsembleTable
|
|
11
|
+
from sdm.processing import EnsembleInvertibleMixin, EnsembleProcessor
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Choice(EnsembleProcessor, EnsembleInvertibleMixin):
|
|
15
|
+
"""Route each table or ensemble member through one selected option.
|
|
16
|
+
|
|
17
|
+
Options are selected when the processor is fitted. Only selected options
|
|
18
|
+
are fitted, and refitting selects again.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
args: Candidate processors or stateless callables.
|
|
22
|
+
method: How to select options. ``"random"`` samples uniformly;
|
|
23
|
+
``"round_robin"`` assigns options by ensemble member position and
|
|
24
|
+
selects the first option for a single table.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
requires_fit = True
|
|
28
|
+
|
|
29
|
+
def __init__(
|
|
30
|
+
self,
|
|
31
|
+
*args: object,
|
|
32
|
+
method: Literal["random", "round_robin"] = "random",
|
|
33
|
+
) -> None:
|
|
34
|
+
super().__init__()
|
|
35
|
+
options = []
|
|
36
|
+
for arg in args:
|
|
37
|
+
options.append(EnsembleProcessor.as_processor(arg))
|
|
38
|
+
self.options: ModuleList[EnsembleProcessor] = ModuleList(options)
|
|
39
|
+
self.handles_stypes = frozenset(
|
|
40
|
+
stype for option in self.options for stype in option.handles_stypes
|
|
41
|
+
)
|
|
42
|
+
self.method = method
|
|
43
|
+
self._option_ids: tuple[int, ...] = ()
|
|
44
|
+
|
|
45
|
+
def get_extra_state(self) -> tuple[int, ...]:
|
|
46
|
+
r""":meta private:""" # noqa: D415
|
|
47
|
+
return self._option_ids
|
|
48
|
+
|
|
49
|
+
def set_extra_state(self, state: object) -> None:
|
|
50
|
+
r""":meta private:""" # noqa: D415
|
|
51
|
+
self._option_ids = cast(tuple[int, ...], state)
|
|
52
|
+
|
|
53
|
+
def _draw_option_ids(
|
|
54
|
+
self,
|
|
55
|
+
ensemble_table: EnsembleTable,
|
|
56
|
+
*,
|
|
57
|
+
generator: torch.Generator | None = None,
|
|
58
|
+
) -> tuple[int, ...]:
|
|
59
|
+
if self.method == "round_robin":
|
|
60
|
+
return tuple(
|
|
61
|
+
member_id % len(self.options)
|
|
62
|
+
for member_id in range(len(ensemble_table))
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
assert self.method == "random"
|
|
66
|
+
device = (
|
|
67
|
+
next(ensemble_table._iter_groups()).device
|
|
68
|
+
if generator is None
|
|
69
|
+
else generator.device
|
|
70
|
+
)
|
|
71
|
+
return tuple(
|
|
72
|
+
torch.randint(
|
|
73
|
+
len(self.options),
|
|
74
|
+
(len(ensemble_table),),
|
|
75
|
+
generator=generator,
|
|
76
|
+
device=device,
|
|
77
|
+
).tolist()
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
def _tables_by_option(
|
|
81
|
+
self,
|
|
82
|
+
ensemble_table: EnsembleTable,
|
|
83
|
+
) -> Iterator[tuple[int, EnsembleTable]]:
|
|
84
|
+
member_ids_by_option: dict[int, list[int]] = {}
|
|
85
|
+
for member_id, option_id in enumerate(self._option_ids):
|
|
86
|
+
member_ids_by_option.setdefault(option_id, []).append(member_id)
|
|
87
|
+
# Select lazily, since selecting members may copy them.
|
|
88
|
+
for option_id, member_ids in sorted(member_ids_by_option.items()):
|
|
89
|
+
yield option_id, ensemble_table[member_ids]
|
|
90
|
+
|
|
91
|
+
def _gather_outputs(
|
|
92
|
+
self,
|
|
93
|
+
ensemble_table: EnsembleTable,
|
|
94
|
+
outputs: dict[int, EnsembleTable],
|
|
95
|
+
) -> EnsembleTable:
|
|
96
|
+
tables = []
|
|
97
|
+
member_ids = []
|
|
98
|
+
next_member_id_by_option: dict[int, int] = {}
|
|
99
|
+
for option_id in self._option_ids:
|
|
100
|
+
tables.append(outputs[option_id])
|
|
101
|
+
member_id = next_member_id_by_option.get(option_id, 0)
|
|
102
|
+
member_ids.append(member_id)
|
|
103
|
+
next_member_id_by_option[option_id] = member_id + 1
|
|
104
|
+
return ensemble_table.gather_members(
|
|
105
|
+
tables=tables,
|
|
106
|
+
member_ids=member_ids,
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
def _check_num_members(self, ensemble_table: EnsembleTable) -> None:
|
|
110
|
+
if len(self._option_ids) != len(ensemble_table):
|
|
111
|
+
raise RuntimeError(
|
|
112
|
+
f"{self.__class__.__name__!r} was fitted with "
|
|
113
|
+
f"{len(self._option_ids)} ensemble members, but got "
|
|
114
|
+
f"{len(ensemble_table)}."
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
def _fit_ensemble(
|
|
118
|
+
self,
|
|
119
|
+
ensemble_table: EnsembleTable,
|
|
120
|
+
*,
|
|
121
|
+
generator: torch.Generator | None = None,
|
|
122
|
+
) -> None:
|
|
123
|
+
self._option_ids = self._draw_option_ids(
|
|
124
|
+
ensemble_table,
|
|
125
|
+
generator=generator,
|
|
126
|
+
)
|
|
127
|
+
for option_id, table in self._tables_by_option(ensemble_table):
|
|
128
|
+
self.options[option_id].fit_ensemble(table, generator=generator)
|
|
129
|
+
|
|
130
|
+
def _fit_transform_ensemble(
|
|
131
|
+
self,
|
|
132
|
+
ensemble_table: EnsembleTable,
|
|
133
|
+
*,
|
|
134
|
+
generator: torch.Generator | None = None,
|
|
135
|
+
) -> EnsembleTable:
|
|
136
|
+
self._option_ids = self._draw_option_ids(
|
|
137
|
+
ensemble_table,
|
|
138
|
+
generator=generator,
|
|
139
|
+
)
|
|
140
|
+
outputs = {
|
|
141
|
+
option_id: self.options[option_id].fit_transform_ensemble(
|
|
142
|
+
table,
|
|
143
|
+
generator=generator,
|
|
144
|
+
)
|
|
145
|
+
for option_id, table in self._tables_by_option(ensemble_table)
|
|
146
|
+
}
|
|
147
|
+
return self._gather_outputs(ensemble_table, outputs)
|
|
148
|
+
|
|
149
|
+
def _transform_ensemble(
|
|
150
|
+
self,
|
|
151
|
+
ensemble_table: EnsembleTable,
|
|
152
|
+
) -> EnsembleTable:
|
|
153
|
+
self._check_num_members(ensemble_table)
|
|
154
|
+
outputs = {
|
|
155
|
+
option_id: self.options[option_id].transform_ensemble(table)
|
|
156
|
+
for option_id, table in self._tables_by_option(ensemble_table)
|
|
157
|
+
}
|
|
158
|
+
return self._gather_outputs(ensemble_table, outputs)
|
|
159
|
+
|
|
160
|
+
def _inverse_transform_ensemble(
|
|
161
|
+
self,
|
|
162
|
+
ensemble_table: EnsembleTable,
|
|
163
|
+
) -> EnsembleTable:
|
|
164
|
+
self._check_num_members(ensemble_table)
|
|
165
|
+
outputs = {}
|
|
166
|
+
for option_id, table in self._tables_by_option(ensemble_table):
|
|
167
|
+
processor = self.options[option_id]
|
|
168
|
+
if not isinstance(processor, EnsembleInvertibleMixin):
|
|
169
|
+
raise TypeError(
|
|
170
|
+
f"{processor.__class__.__name__!r} is not invertible."
|
|
171
|
+
)
|
|
172
|
+
outputs[option_id] = processor.inverse_transform_ensemble(table)
|
|
173
|
+
return self._gather_outputs(ensemble_table, outputs)
|
|
174
|
+
|
|
175
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
176
|
+
inner = ",\n".join(
|
|
177
|
+
option.__repr__(indent=indent + 2) for option in self.options
|
|
178
|
+
)
|
|
179
|
+
if self.method != "random":
|
|
180
|
+
inner += f",\n{' ' * (indent + 2)}method={self.method!r}"
|
|
181
|
+
return (
|
|
182
|
+
f"{' ' * indent}{self.__class__.__name__}(\n"
|
|
183
|
+
f"{inner},\n"
|
|
184
|
+
f"{' ' * indent})"
|
|
185
|
+
)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from sdm import Stype, StypeLike, TableTensor
|
|
5
|
+
from sdm.processing.base import Processor
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class DropStypes(Processor):
|
|
9
|
+
r"""Remove all columns for specific semantic types.
|
|
10
|
+
|
|
11
|
+
Args:
|
|
12
|
+
stypes: Semantic column types to remove. All other semantic types are
|
|
13
|
+
preserved unchanged.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
requires_fit = False
|
|
17
|
+
|
|
18
|
+
def __init__(self, *stypes: StypeLike) -> None:
|
|
19
|
+
super().__init__()
|
|
20
|
+
self._stypes = stypes
|
|
21
|
+
self.handles_stypes = frozenset(Stype(stype) for stype in stypes)
|
|
22
|
+
|
|
23
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
24
|
+
return table.drop_stypes(self.handles_stypes)
|
|
25
|
+
|
|
26
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
27
|
+
stypes = ", ".join(map(repr, self._stypes))
|
|
28
|
+
return f"{' ' * indent}{self.__class__.__name__}({stypes})"
|