structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,187 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import copy
5
+ from itertools import repeat
6
+ from typing import cast
7
+
8
+ import torch
9
+ from torch.nn import ModuleList
10
+
11
+ from sdm import EnsembleTable, Stype
12
+ from sdm.processing import (
13
+ EnsembleInvertibleMixin,
14
+ EnsembleProcessor,
15
+ InvertibleMixin,
16
+ Processor,
17
+ )
18
+
19
+
20
+ class EnsembleProcessorAdapter(EnsembleProcessor, EnsembleInvertibleMixin):
21
+ """Adapt an ordinary processor to ensemble-aware processing.
22
+
23
+ The adapter turns a :class:`~sdm.processing.base.Processor` into an
24
+ :class:`~sdm.processing.ensemble.EnsembleProcessor`. It copies and fits the
25
+ processor separately for each group of compatible tables in an
26
+ :class:`~sdm.EnsembleTable`.
27
+
28
+ The wrapped processor must preserve row and leading dimensions as required
29
+ by the :class:`~sdm.processing.base.Processor` contract. A processor that
30
+ changes the ensemble structure must implement
31
+ :class:`~sdm.processing.ensemble.EnsembleProcessor` directly. Inverse
32
+ transformation requires the wrapped processor to implement
33
+ :class:`~sdm.processing.base.InvertibleMixin`.
34
+
35
+ Args:
36
+ processor: Processor to fit separately for each ensemble table group.
37
+ """
38
+
39
+ def __init__(self, processor: Processor) -> None:
40
+ super().__init__()
41
+ self.processor = processor
42
+ self.requires_fit = processor.requires_fit
43
+
44
+ self._processors: ModuleList[Processor] = ModuleList()
45
+ self._fitted_locations: tuple[tuple[int, int], ...] = ()
46
+
47
+ def get_extra_state(
48
+ self,
49
+ ) -> tuple[int, tuple[tuple[int, int], ...]]:
50
+ r""":meta private:""" # noqa: D415
51
+ return len(self._processors), self._fitted_locations
52
+
53
+ def set_extra_state(self, state: object) -> None:
54
+ r""":meta private:""" # noqa: D415
55
+ num_processors, self._fitted_locations = cast(
56
+ tuple[int, tuple[tuple[int, int], ...]], state
57
+ )
58
+ self._processors = ModuleList(
59
+ [copy.deepcopy(self.processor) for _ in range(num_processors)]
60
+ )
61
+
62
+ @property
63
+ def handles_stypes(self) -> frozenset[Stype]:
64
+ r""":meta private:""" # noqa: D415
65
+ return self.processor.handles_stypes
66
+
67
+ def _fit_ensemble(
68
+ self,
69
+ ensemble_table: EnsembleTable,
70
+ *,
71
+ generator: torch.Generator | None = None,
72
+ ) -> None:
73
+ self._fitted_locations = ensemble_table._locations
74
+ self._processors = ModuleList()
75
+ for group_id, group in enumerate(ensemble_table._iter_groups()):
76
+ if group_id == 0:
77
+ processor = self.processor
78
+ else:
79
+ processor = copy.deepcopy(self.processor)
80
+ processor.fit(group, generator=generator)
81
+ self._processors.append(processor)
82
+
83
+ def _fit_transform_ensemble(
84
+ self,
85
+ ensemble_table: EnsembleTable,
86
+ *,
87
+ generator: torch.Generator | None = None,
88
+ ) -> EnsembleTable:
89
+ if not self.requires_fit:
90
+ return self._transform_ensemble(ensemble_table)
91
+
92
+ self._fitted_locations = ensemble_table._locations
93
+ outputs = []
94
+ self._processors = ModuleList()
95
+ for group_id, group in enumerate(ensemble_table._iter_groups()):
96
+ if group_id == 0:
97
+ processor = self.processor
98
+ else:
99
+ processor = copy.deepcopy(self.processor)
100
+ outputs.append(processor.fit_transform(group, generator=generator))
101
+ self._processors.append(processor)
102
+ return ensemble_table.replace_groups(outputs)
103
+
104
+ def _transform_ensemble(
105
+ self,
106
+ ensemble_table: EnsembleTable,
107
+ ) -> EnsembleTable:
108
+ if self.requires_fit:
109
+ processors = self._aligned_processors(ensemble_table)
110
+ else:
111
+ processors = repeat(self.processor, ensemble_table.num_groups)
112
+ outputs = [
113
+ processor.transform(group)
114
+ for group, processor in zip(
115
+ ensemble_table._iter_groups(), processors, strict=True
116
+ )
117
+ ]
118
+ return ensemble_table.replace_groups(outputs)
119
+
120
+ def _inverse_transform_ensemble(
121
+ self,
122
+ ensemble_table: EnsembleTable,
123
+ ) -> EnsembleTable:
124
+ if self.requires_fit:
125
+ processors = self._aligned_processors(ensemble_table)
126
+ else:
127
+ processors = repeat(self.processor, ensemble_table.num_groups)
128
+
129
+ outputs = []
130
+ for group, processor in zip(
131
+ ensemble_table._iter_groups(), processors, strict=True
132
+ ):
133
+ if not isinstance(processor, InvertibleMixin):
134
+ raise AttributeError(
135
+ f"{self.processor.__class__.__name__!r} object has no "
136
+ "attribute 'inverse_transform'"
137
+ )
138
+ outputs.append(processor.inverse_transform(group))
139
+ return ensemble_table.replace_groups(outputs)
140
+
141
+ def _aligned_processors(
142
+ self,
143
+ ensemble_table: EnsembleTable,
144
+ ) -> tuple[Processor, ...]:
145
+ if ensemble_table._locations == self._fitted_locations:
146
+ return tuple(self._processors)
147
+
148
+ # One fitted processor per transform ensemble group. Multiple
149
+ # transform groups can share the same fitted processor.
150
+ processors: list[Processor | None] = [None] * ensemble_table.num_groups
151
+ # Current and fitted batch of every member sharing a fitted processor.
152
+ batches_by_processor: dict[int, list[tuple[int, int, int]]] = {}
153
+ for member_id, (group_id, batch_id) in enumerate(
154
+ ensemble_table._locations
155
+ ):
156
+ fitted_group_id, fitted_batch_id = self._fitted_locations[
157
+ member_id
158
+ ]
159
+ processor = self._processors[fitted_group_id]
160
+ if processors[group_id] is None:
161
+ processors[group_id] = processor
162
+ elif processors[group_id] is not processor:
163
+ raise RuntimeError(
164
+ "Cannot apply fitted processor state to an ensemble "
165
+ "group containing members from different fitted groups"
166
+ )
167
+ batches_by_processor.setdefault(fitted_group_id, []).append(
168
+ (group_id, batch_id, fitted_batch_id)
169
+ )
170
+
171
+ # A processor fitted on a stacked tensor table stores per-batch state.
172
+ # It can only operate on stacked tensor tables with the same layout,
173
+ # but not on splits or reordered batches.
174
+ for batches in batches_by_processor.values():
175
+ if len({fitted for _, _, fitted in batches}) > 1 and (
176
+ len({group for group, _, _ in batches}) > 1
177
+ or any(current != fitted for _, current, fitted in batches)
178
+ ):
179
+ raise RuntimeError(
180
+ "Cannot apply position-dependent fitted processor state "
181
+ "after its ensemble group was split or reordered"
182
+ )
183
+
184
+ return tuple(cast(Processor, processor) for processor in processors)
185
+
186
+ def __repr__(self, *, indent: int = 0) -> str:
187
+ return self.processor.__repr__(indent=indent)
@@ -0,0 +1,18 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from sdm import Stype, TableTensor
5
+ from sdm.processing import InvertibleMixin, Processor
6
+
7
+
8
+ class Identity(Processor, InvertibleMixin):
9
+ r"""Return inputs unchanged."""
10
+
11
+ handles_stypes = frozenset(Stype)
12
+ requires_fit = False
13
+
14
+ def _transform(self, table: TableTensor) -> TableTensor:
15
+ return table
16
+
17
+ def _inverse_transform(self, table: TableTensor) -> TableTensor:
18
+ return table
@@ -0,0 +1,107 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from collections.abc import Mapping, Sequence
5
+ from typing import Literal
6
+
7
+ import torch
8
+
9
+ from sdm import EnsembleTable, Stype, StypeLike, TableTensor
10
+ from sdm.processing import EnsembleProcessor
11
+
12
+
13
+ class SelectColumns(EnsembleProcessor):
14
+ r"""Select a subset of columns for each semantic type.
15
+
16
+ Args:
17
+ max_columns: The maximum number of columns to keep per semantic type.
18
+ method: The column selection method.
19
+ ``"first"`` keeps the first columns according to their order within
20
+ each semantic block. ``"round_robin"`` assigns each ensemble
21
+ member the next consecutive chunk, wrapping to the first column
22
+ after the last. For five columns and ``max_columns=2``, members
23
+ 0, 1, and 2 receive columns (0, 1), (2, 3), and (4, 0).
24
+ """
25
+
26
+ handles_stypes = frozenset(Stype)
27
+ requires_fit = False
28
+
29
+ def __init__(
30
+ self,
31
+ max_columns: int,
32
+ method: Literal["first", "round_robin"] = "first",
33
+ ) -> None:
34
+ super().__init__()
35
+ if max_columns <= 0:
36
+ raise ValueError("max_columns must be positive")
37
+ self.max_columns = max_columns
38
+ self.method = method
39
+
40
+ def _transform_ensemble(
41
+ self,
42
+ ensemble_table: EnsembleTable,
43
+ ) -> EnsembleTable:
44
+ if all(
45
+ group.size(-1) <= self.max_columns
46
+ for group in ensemble_table._iter_groups()
47
+ ):
48
+ return ensemble_table
49
+
50
+ if self.method == "first":
51
+ groups = []
52
+ for group in ensemble_table._iter_groups():
53
+ columns: Mapping[StypeLike, Sequence[str]] = {
54
+ stype: column_names[: self.max_columns]
55
+ for stype, column_names in group.columns.items()
56
+ }
57
+ blocks = {
58
+ stype: block[..., : self.max_columns]
59
+ for stype, block in group.items()
60
+ }
61
+ groups.append(group.__class__(columns=columns, **blocks))
62
+ return ensemble_table.replace_groups(groups)
63
+
64
+ assert self.method == "round_robin"
65
+ tables: list[TableTensor] = []
66
+ for member_id in range(len(ensemble_table)):
67
+ table = ensemble_table[member_id]
68
+ columns: dict[StypeLike, tuple[str, ...]] = {}
69
+ blocks = {}
70
+ for stype, block in table.items():
71
+ column_names = table.columns[stype]
72
+ num_columns = len(column_names)
73
+ # Empty blocks are passed through unchanged.
74
+ if num_columns == 0:
75
+ columns[stype] = column_names
76
+ blocks[stype] = block
77
+ continue
78
+ chunk_size = min(self.max_columns, num_columns)
79
+ start = member_id * chunk_size % num_columns
80
+ first_count = min(chunk_size, num_columns - start)
81
+ wrap_count = chunk_size - first_count
82
+ # Consecutive chunk from ``start``, wrapping past the last
83
+ # column back to the first.
84
+ columns[stype] = (
85
+ column_names[start : start + first_count]
86
+ + column_names[:wrap_count]
87
+ )
88
+ if wrap_count == 0:
89
+ blocks[stype] = block[..., start : start + first_count]
90
+ continue
91
+ blocks[stype] = torch.cat(
92
+ [
93
+ block[..., start : start + first_count],
94
+ block[..., :wrap_count],
95
+ ],
96
+ dim=-1,
97
+ )
98
+ tables.append(
99
+ table.__class__(
100
+ columns=columns,
101
+ **blocks,
102
+ )
103
+ )
104
+ return ensemble_table.replace_tables(
105
+ tables=tables,
106
+ member_table_ids=range(len(tables)),
107
+ )
@@ -0,0 +1,158 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from collections.abc import Iterable, Iterator
5
+ from typing import Self, cast
6
+
7
+ import torch
8
+
9
+ from sdm import EnsembleTable, Stype, TableTensor
10
+ from sdm.processing import (
11
+ EnsembleInvertibleMixin,
12
+ EnsembleProcessor,
13
+ )
14
+
15
+
16
+ class Sequential(EnsembleProcessor, EnsembleInvertibleMixin):
17
+ r"""Apply processors and callables to a table in sequence.
18
+
19
+ Args:
20
+ args: Sequence of :class:`Processor` instances or callables.
21
+ """
22
+
23
+ def __init__(self, *args: object) -> None:
24
+ super().__init__()
25
+ self.requires_fit = False
26
+ self.extend(args)
27
+
28
+ @property
29
+ def handles_stypes(self) -> frozenset[Stype]:
30
+ r""":meta private:""" # noqa: D415
31
+ if len(self) == 0:
32
+ return frozenset(Stype)
33
+ return frozenset(
34
+ stype for child in self for stype in child.handles_stypes
35
+ )
36
+
37
+ def append(self, processor: object) -> Self:
38
+ r"""Append a processor or callable to this sequence.
39
+
40
+ Args:
41
+ processor: The processor to append.
42
+ """
43
+ processor = EnsembleProcessor.as_processor(processor)
44
+ if isinstance(processor, Sequential):
45
+ for child in processor.children():
46
+ self.add_module(str(len(self)), child)
47
+ else:
48
+ self.add_module(str(len(self)), processor)
49
+
50
+ self.requires_fit = any(child.requires_fit for child in self)
51
+ self._fitted = False
52
+ return self
53
+
54
+ def prepend(self, processor: object) -> Self:
55
+ r"""Prepend a processor or callable to this sequence.
56
+
57
+ Args:
58
+ processor: The processor to prepend.
59
+ """
60
+ processor = EnsembleProcessor.as_processor(processor)
61
+
62
+ if isinstance(processor, Sequential):
63
+ processors = tuple(processor.children())
64
+ else:
65
+ processors = (processor,)
66
+
67
+ processors = (*processors, *self.children())
68
+
69
+ self._modules.clear()
70
+ for child in processors:
71
+ self.add_module(str(len(self)), child)
72
+
73
+ self.requires_fit = any(child.requires_fit for child in self)
74
+ self._fitted = False
75
+ return self
76
+
77
+ def extend(self, processors: Iterable[object]) -> Self:
78
+ r"""Append multiple processors or callables to this sequence.
79
+
80
+ Args:
81
+ processors: The processors to append.
82
+ """
83
+ for processor in processors:
84
+ self.append(processor)
85
+ return self
86
+
87
+ def _transform(self, table: TableTensor) -> TableTensor:
88
+ out = table
89
+ for child in self:
90
+ out = child.transform(out)
91
+ return out
92
+
93
+ def _fit_ensemble(
94
+ self,
95
+ ensemble_table: EnsembleTable,
96
+ *,
97
+ generator: torch.Generator | None = None,
98
+ ) -> None:
99
+ out = ensemble_table
100
+ for i, child in enumerate(self):
101
+ if i < len(self) - 1:
102
+ out = child.fit_transform_ensemble(out, generator=generator)
103
+ else:
104
+ child.fit_ensemble(out, generator=generator)
105
+
106
+ def _fit_transform_ensemble(
107
+ self,
108
+ ensemble_table: EnsembleTable,
109
+ *,
110
+ generator: torch.Generator | None = None,
111
+ ) -> EnsembleTable:
112
+ out = ensemble_table
113
+ for child in self:
114
+ out = child.fit_transform_ensemble(out, generator=generator)
115
+ return out
116
+
117
+ def _transform_ensemble(
118
+ self,
119
+ ensemble_table: EnsembleTable,
120
+ ) -> EnsembleTable:
121
+ out = ensemble_table
122
+ for child in self:
123
+ out = child.transform_ensemble(out)
124
+ return out
125
+
126
+ def _inverse_transform_ensemble(
127
+ self,
128
+ ensemble_table: EnsembleTable,
129
+ ) -> EnsembleTable:
130
+ out = ensemble_table
131
+ for child in reversed(list(self)):
132
+ if not isinstance(child, EnsembleInvertibleMixin):
133
+ raise AttributeError(
134
+ f"{child.__class__.__name__!r} object has no "
135
+ "attribute 'inverse_transform_ensemble'"
136
+ )
137
+ out = child.inverse_transform_ensemble(out)
138
+ return out
139
+
140
+ def __iter__(self) -> Iterator[EnsembleProcessor]:
141
+ return cast(Iterator[EnsembleProcessor], self.children())
142
+
143
+ def __len__(self) -> int:
144
+ return len(self._modules)
145
+
146
+ def __iadd__(self, other: object) -> Self:
147
+ self.append(EnsembleProcessor.as_processor(other))
148
+ return self
149
+
150
+ def __repr__(self, *, indent: int = 0) -> str:
151
+ if len(self) == 0:
152
+ return super().__repr__(indent=indent)
153
+ reprs = "".join(
154
+ [f"{child.__repr__(indent=indent + 2)},\n" for child in self]
155
+ )
156
+ return (
157
+ f"{' ' * indent}{self.__class__.__name__}(\n{reprs}{' ' * indent})"
158
+ )
@@ -0,0 +1,195 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from typing import Literal, cast
5
+
6
+ import torch
7
+ from torch import Tensor
8
+
9
+ from sdm import EnsembleTable, Stype, TableTensor
10
+ from sdm.nn._buffer import BufferList
11
+ from sdm.processing import EnsembleInvertibleMixin, EnsembleProcessor
12
+
13
+
14
+ class ShuffleColumns(EnsembleProcessor, EnsembleInvertibleMixin):
15
+ """Permute numerical feature columns and their names.
16
+
17
+ Convert non-numerical feature stypes before this step, for example with
18
+ :class:`~sdm.processing.ToNumerical`.
19
+
20
+ Args:
21
+ method: Permutation strategy. ``"random"`` draws independent
22
+ permutations, and ``"latin"`` draws coupled Latin
23
+ permutations.
24
+ """
25
+
26
+ handles_stypes = frozenset({Stype.numerical})
27
+ requires_fit = True
28
+
29
+ def __init__(
30
+ self,
31
+ method: Literal["random", "latin"] = "random",
32
+ ) -> None:
33
+ super().__init__()
34
+ self.method = method
35
+ self._permutations: BufferList[Tensor] = BufferList()
36
+ self._host_permutations: tuple[tuple[int, ...], ...] = ()
37
+
38
+ def get_extra_state(self) -> tuple[tuple[int, ...], ...]:
39
+ r""":meta private:""" # noqa: D415
40
+ return self._host_permutations
41
+
42
+ def set_extra_state(self, state: object) -> None:
43
+ r""":meta private:""" # noqa: D415
44
+ self._host_permutations = cast(tuple[tuple[int, ...], ...], state)
45
+
46
+ def _fit_ensemble(
47
+ self,
48
+ ensemble_table: EnsembleTable,
49
+ *,
50
+ generator: torch.Generator | None = None,
51
+ ) -> None:
52
+ device = next(ensemble_table._iter_groups()).device
53
+ widths = [
54
+ ensemble_table[member_id].numerical.size(-1)
55
+ for member_id in range(len(ensemble_table))
56
+ ]
57
+ permutations = []
58
+ if self.method == "latin":
59
+ max_features = max(widths)
60
+ base_rank = torch.randperm(
61
+ max_features,
62
+ generator=generator,
63
+ device=device,
64
+ )
65
+ row_rank = torch.randperm(
66
+ max_features,
67
+ generator=generator,
68
+ device=device,
69
+ )
70
+ for member_id, n_features in enumerate(widths):
71
+ if n_features <= 1:
72
+ permutation = torch.arange(n_features, device=device)
73
+ else:
74
+ base = base_rank
75
+ rows = row_rank
76
+ if n_features < max_features:
77
+ base = base[base < n_features]
78
+ rows = rows[rows < n_features]
79
+ pattern = member_id % n_features
80
+ permutation = base[(pattern - rows) % n_features]
81
+ permutations.append(permutation)
82
+ else:
83
+ assert self.method == "random"
84
+ for n_features in widths:
85
+ if n_features <= 1:
86
+ permutation = torch.arange(n_features, device=device)
87
+ else:
88
+ permutation = torch.randperm(
89
+ n_features,
90
+ generator=generator,
91
+ device=device,
92
+ )
93
+ permutations.append(permutation)
94
+
95
+ self._permutations = BufferList(permutations)
96
+ # TODO: Add to EnsembleTable directly.
97
+ member_ids_by_group: list[list[int]] = [
98
+ [] for _ in range(ensemble_table.num_groups)
99
+ ]
100
+ for member_id, (group_id, _) in enumerate(ensemble_table._locations):
101
+ member_ids_by_group[group_id].append(member_id)
102
+ host_permutations = [()] * len(permutations)
103
+ for member_ids in member_ids_by_group:
104
+ host_rows = torch.stack(
105
+ [permutations[member_id] for member_id in member_ids]
106
+ ).tolist()
107
+ for member_id, host_row in zip(member_ids, host_rows, strict=True):
108
+ host_permutations[member_id] = tuple(host_row)
109
+ self._host_permutations = tuple(host_permutations)
110
+
111
+ def _transform_ensemble(
112
+ self,
113
+ ensemble_table: EnsembleTable,
114
+ ) -> EnsembleTable:
115
+ if len(self._permutations) != len(ensemble_table):
116
+ raise RuntimeError(
117
+ f"{self.__class__.__name__!r} was fitted with "
118
+ f"{len(self._permutations)} ensemble members, but got "
119
+ f"{len(ensemble_table)}."
120
+ )
121
+ tables: list[TableTensor] = []
122
+ for member_id in range(len(ensemble_table)):
123
+ table = ensemble_table[member_id]
124
+ host_permutation = self._host_permutations[member_id]
125
+ shuffled = table.__class__(
126
+ columns={
127
+ Stype.numerical: tuple(
128
+ table.columns[Stype.numerical][i]
129
+ for i in host_permutation
130
+ )
131
+ },
132
+ numerical=table.numerical.index_select(
133
+ -1, self._permutations[member_id]
134
+ ),
135
+ )
136
+ tables.append(
137
+ cast(
138
+ TableTensor,
139
+ torch.cat(
140
+ (table.drop_stypes(Stype.numerical), shuffled),
141
+ dim=-1,
142
+ ),
143
+ )
144
+ )
145
+ return EnsembleTable.from_tables(
146
+ tables=tables,
147
+ member_table_ids=range(len(tables)),
148
+ )
149
+
150
+ def _inverse_transform_ensemble(
151
+ self,
152
+ ensemble_table: EnsembleTable,
153
+ ) -> EnsembleTable:
154
+ if len(self._permutations) != len(ensemble_table):
155
+ raise RuntimeError(
156
+ f"{self.__class__.__name__!r} was fitted with "
157
+ f"{len(self._permutations)} ensemble members, but got "
158
+ f"{len(ensemble_table)}."
159
+ )
160
+ tables: list[TableTensor] = []
161
+ for member_id in range(len(ensemble_table)):
162
+ table = ensemble_table[member_id]
163
+ host_permutation = self._host_permutations[member_id]
164
+ inverse_host_permutation = [0] * len(host_permutation)
165
+ for destination, source in enumerate(host_permutation):
166
+ inverse_host_permutation[source] = destination
167
+ shuffled = table.__class__(
168
+ columns={
169
+ Stype.numerical: tuple(
170
+ table.columns[Stype.numerical][i]
171
+ for i in inverse_host_permutation
172
+ )
173
+ },
174
+ numerical=table.numerical.index_select(
175
+ -1, self._permutations[member_id].argsort()
176
+ ),
177
+ )
178
+ tables.append(
179
+ cast(
180
+ TableTensor,
181
+ torch.cat(
182
+ (table.drop_stypes(Stype.numerical), shuffled),
183
+ dim=-1,
184
+ ),
185
+ )
186
+ )
187
+ return EnsembleTable.from_tables(
188
+ tables=tables,
189
+ member_table_ids=range(len(tables)),
190
+ )
191
+
192
+ def __repr__(self, *, indent: int = 0) -> str:
193
+ return (
194
+ f"{' ' * indent}{self.__class__.__name__}(method={self.method!r})"
195
+ )