structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/nn/softplus.py ADDED
@@ -0,0 +1,65 @@
1
+ # Portions derived from TabFM:
2
+ # Copyright 2026 Google LLC
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # https://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
17
+ # SPDX-License-Identifier: Apache-2.0
18
+
19
+ import torch
20
+ import torch.nn.functional as F
21
+ from torch import Tensor
22
+
23
+
24
+ class SoftplusScale(torch.nn.Module):
25
+ r"""Apply a learned positive scale to the final input dimension.
26
+
27
+ Each channel is multiplied by a positive factor parametrized as
28
+ ``multiplier * softplus(weight)``.
29
+
30
+ Args:
31
+ channels: The number of input and output channels.
32
+ multiplier: The fixed multiplier applied to every learned factor.
33
+ weight_init: The initial values of ``weight``.
34
+ device: The device.
35
+ dtype: The parameter dtype.
36
+ """
37
+
38
+ def __init__(
39
+ self,
40
+ channels: int,
41
+ multiplier: float = 1.0,
42
+ weight_init: float = 0.0,
43
+ device: torch.device | str | None = None,
44
+ dtype: torch.dtype | None = None,
45
+ ) -> None:
46
+ super().__init__()
47
+
48
+ self.multiplier = multiplier
49
+ self.weight = torch.nn.Parameter(
50
+ data=torch.full(
51
+ size=(channels,),
52
+ fill_value=weight_init,
53
+ device=device,
54
+ dtype=dtype,
55
+ )
56
+ )
57
+
58
+ def forward(self, tensor: Tensor) -> Tensor:
59
+ r"""The forward pass.
60
+
61
+ Args:
62
+ tensor: The input tensor.
63
+ """
64
+ scale = self.multiplier * F.softplus(self.weight.float())
65
+ return tensor * scale.to(tensor.dtype)
@@ -0,0 +1,96 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Processors for structured data tables."""
5
+
6
+ from sdm.processing.base import Processor, InvertibleMixin
7
+ from sdm.processing.ensemble import (
8
+ EnsembleProcessor,
9
+ EnsembleInvertibleMixin,
10
+ )
11
+ from sdm.processing.common import (
12
+ Identity,
13
+ Callable,
14
+ Sequential,
15
+ DropStypes,
16
+ StypeDispatch,
17
+ TaskDispatch,
18
+ TableDispatch,
19
+ EnsembleProcessorAdapter,
20
+ Choice,
21
+ ToNumerical,
22
+ ShuffleColumns,
23
+ SelectColumns,
24
+ )
25
+ from sdm.processing.text import TFIDF, SentenceTransformer
26
+ from sdm.processing.numerical import (
27
+ Cast,
28
+ Clip,
29
+ ClipQuantiles,
30
+ ClipSigma,
31
+ ClipSoft,
32
+ ImputeMean,
33
+ PowerTransform,
34
+ QuantileTransform,
35
+ RankGaussian,
36
+ Standardize,
37
+ RobustScale,
38
+ FlipSign,
39
+ DropConstantColumns,
40
+ PCA,
41
+ RandomProjection,
42
+ )
43
+ from sdm.processing.categorical import (
44
+ AlignCategories,
45
+ ShuffleCategories,
46
+ ImputeMode,
47
+ AddCategoryCounts,
48
+ )
49
+ from sdm.processing.datetime import AddCalendarFields
50
+ from sdm.processing.output import AverageEstimators, Softmax, SortQuantiles
51
+ from sdm.processing.recipe import Recipe
52
+
53
+ __all__ = [
54
+ "Processor",
55
+ "InvertibleMixin",
56
+ "EnsembleInvertibleMixin",
57
+ "EnsembleProcessorAdapter",
58
+ "Identity",
59
+ "Callable",
60
+ "Sequential",
61
+ "DropStypes",
62
+ "StypeDispatch",
63
+ "TaskDispatch",
64
+ "TableDispatch",
65
+ "EnsembleProcessor",
66
+ "Choice",
67
+ "ToNumerical",
68
+ "ShuffleColumns",
69
+ "SelectColumns",
70
+ "TFIDF",
71
+ "SentenceTransformer",
72
+ "Cast",
73
+ "Clip",
74
+ "ClipQuantiles",
75
+ "ClipSigma",
76
+ "ClipSoft",
77
+ "ImputeMean",
78
+ "PowerTransform",
79
+ "QuantileTransform",
80
+ "RankGaussian",
81
+ "Standardize",
82
+ "RobustScale",
83
+ "FlipSign",
84
+ "DropConstantColumns",
85
+ "PCA",
86
+ "RandomProjection",
87
+ "AlignCategories",
88
+ "ShuffleCategories",
89
+ "ImputeMode",
90
+ "AddCategoryCounts",
91
+ "AddCalendarFields",
92
+ "AverageEstimators",
93
+ "Softmax",
94
+ "SortQuantiles",
95
+ "Recipe",
96
+ ]
sdm/processing/base.py ADDED
@@ -0,0 +1,254 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from __future__ import annotations
5
+
6
+ import abc
7
+ from collections.abc import Sequence
8
+ from typing import TYPE_CHECKING, Any, Self
9
+
10
+ import torch
11
+ from torch import Tensor
12
+
13
+ from sdm import Stype, TableTensor
14
+
15
+ if TYPE_CHECKING:
16
+ from sdm.processing import Sequential
17
+
18
+
19
+ class Processor(torch.nn.Module, abc.ABC):
20
+ r"""Base processor for tensor-aware table transformations.
21
+
22
+ A :class:`Processor` defines a reusable transformation on
23
+ :class:`~sdm.tensor.TableTensor` for feature, target and output
24
+ preprocessing.
25
+ A :class:`Processor` learns any required state via :meth:`fit`, and applies
26
+ the transformation via :meth:`transform`. Implementations preserve batch
27
+ dimensions, and preserve rows unless documented otherwise. Batch
28
+ dimensions are processed independently.
29
+
30
+ :meth:`fit`, :meth:`transform`, and :meth:`fit_transform` are no-ops for
31
+ stypes outside of :attr:`handles_stypes`.
32
+ """
33
+
34
+ #: Semantic types this processor operates on.
35
+ handles_stypes: frozenset[Stype]
36
+
37
+ #: Whether this processor requires fitting.
38
+ requires_fit: bool
39
+
40
+ _fitted_state: Tensor
41
+
42
+ def __init__(self) -> None:
43
+ super().__init__()
44
+ self.register_buffer("_fitted_state", torch.tensor(False))
45
+ self.__fitted = False
46
+
47
+ @property
48
+ def _fitted(self) -> bool:
49
+ return self.__fitted
50
+
51
+ @_fitted.setter
52
+ def _fitted(self, value: bool) -> None:
53
+ self.__fitted = value
54
+ self._fitted_state.fill_(value)
55
+
56
+ @property
57
+ def is_fitted(self) -> bool:
58
+ """Whether the processor has all state required for transformation."""
59
+ return not self.requires_fit or self._fitted
60
+
61
+ @staticmethod
62
+ def as_processor(processor: object) -> Processor:
63
+ r"""Normalize a processor-like object to a :class:`Processor`.
64
+
65
+ Args:
66
+ processor: A processor-like object. A :class:`Processor` is
67
+ returned as-is, a callable is wrapped as a stateless processor,
68
+ and a sequence of processor-like objects is normalized to
69
+ :class:`~sdm.processing.common.Sequential`.
70
+ """
71
+ from sdm.processing import Callable, Sequential # noqa: PLC0415
72
+
73
+ if isinstance(processor, Processor):
74
+ return processor
75
+ if callable(processor):
76
+ return Callable(processor) # type: ignore
77
+ if isinstance(processor, Sequence) and not isinstance(processor, str):
78
+ return Sequential(*processor)
79
+ raise TypeError(
80
+ f"Input must be a 'Processor', callable, or sequence of them "
81
+ f"(got '{type(processor).__name__}')"
82
+ )
83
+
84
+ def _load_from_state_dict(
85
+ self,
86
+ state_dict: dict[str, Any],
87
+ prefix: str,
88
+ local_metadata: dict[str, Any],
89
+ strict: bool,
90
+ missing_keys: list[str],
91
+ unexpected_keys: list[str],
92
+ error_msgs: list[str],
93
+ ) -> None:
94
+ # Resize uninitialized buffers before PyTorch copies saved values.
95
+ for name, buffer in self._buffers.items():
96
+ state = state_dict.get(f"{prefix}{name}")
97
+ if (
98
+ buffer is not None
99
+ and isinstance(state, Tensor)
100
+ and (
101
+ buffer.shape != state.shape or buffer.dtype != state.dtype
102
+ )
103
+ ):
104
+ self._buffers[name] = buffer.new_empty(
105
+ state.shape,
106
+ dtype=state.dtype,
107
+ )
108
+ super()._load_from_state_dict(
109
+ state_dict,
110
+ prefix,
111
+ local_metadata,
112
+ strict,
113
+ missing_keys,
114
+ unexpected_keys,
115
+ error_msgs,
116
+ )
117
+ self.__fitted = bool(self._fitted_state)
118
+
119
+ def _set_fitted(self, device: torch.device) -> None:
120
+ self._fitted_state = self._fitted_state.to(device=device)
121
+ self._fitted = True
122
+
123
+ def _check_is_fitted(self) -> None:
124
+ if not self.is_fitted:
125
+ raise RuntimeError(
126
+ f"{self.__class__.__name__!r} is not fitted; "
127
+ "call 'fit()' before."
128
+ )
129
+
130
+ def _fit(
131
+ self,
132
+ table: TableTensor,
133
+ *,
134
+ generator: torch.Generator | None = None,
135
+ ) -> None:
136
+ raise NotImplementedError
137
+
138
+ @abc.abstractmethod
139
+ def _transform(self, table: TableTensor) -> TableTensor:
140
+ pass
141
+
142
+ def _fit_transform(
143
+ self,
144
+ table: TableTensor,
145
+ *,
146
+ generator: torch.Generator | None = None,
147
+ ) -> TableTensor:
148
+ if self.requires_fit:
149
+ self._fit(table, generator=generator)
150
+ return self._transform(table)
151
+
152
+ def fit(
153
+ self,
154
+ table: TableTensor,
155
+ *,
156
+ generator: torch.Generator | None = None,
157
+ ) -> Self:
158
+ r"""Fit the processor.
159
+
160
+ Args:
161
+ table: The table used to compute the processor state.
162
+ generator: Pseudorandom number generator used for sampling.
163
+ """
164
+ if not table.active_stypes & self.handles_stypes:
165
+ return self
166
+ if self.requires_fit:
167
+ self._fit(table, generator=generator)
168
+ self._set_fitted(table.device)
169
+ return self
170
+
171
+ def transform(self, table: TableTensor) -> TableTensor:
172
+ r"""Transform ``table``.
173
+
174
+ Args:
175
+ table: The table to transform.
176
+
177
+ Returns:
178
+ The transformed table.
179
+ """
180
+ if not table.active_stypes & self.handles_stypes:
181
+ return table
182
+ self._check_is_fitted()
183
+ return self._transform(table)
184
+
185
+ def forward(self, table: TableTensor) -> TableTensor:
186
+ r"""Alias of :meth:`transform`."""
187
+ return self.transform(table)
188
+
189
+ def fit_transform(
190
+ self,
191
+ table: TableTensor,
192
+ *,
193
+ generator: torch.Generator | None = None,
194
+ ) -> TableTensor:
195
+ r"""Fit the processor and transform ``table``.
196
+
197
+ Args:
198
+ table: The table to fit on and transform.
199
+ generator: Pseudorandom number generator used for sampling.
200
+
201
+ Returns:
202
+ The transformed table.
203
+ """
204
+ if not table.active_stypes & self.handles_stypes:
205
+ return table
206
+ out = self._fit_transform(table, generator=generator)
207
+ if self.requires_fit:
208
+ self._set_fitted(table.device)
209
+ return out
210
+
211
+ def __add__(self, other: object) -> Sequential:
212
+ from sdm.processing import Sequential # noqa: PLC0415
213
+
214
+ try:
215
+ other = Processor.as_processor(other)
216
+ except TypeError:
217
+ return NotImplemented
218
+ return Sequential(self, other)
219
+
220
+ def __radd__(self, other: object) -> Sequential:
221
+ from sdm.processing import Sequential # noqa: PLC0415
222
+
223
+ try:
224
+ other = Processor.as_processor(other)
225
+ except TypeError:
226
+ return NotImplemented
227
+ return Sequential(other, self)
228
+
229
+ def __repr__(self, *, indent: int = 0) -> str:
230
+ return f"{' ' * indent}{self.__class__.__name__}()"
231
+
232
+
233
+ class InvertibleMixin(abc.ABC):
234
+ r"""Extend a :class:`Processor` by an inverse transformation."""
235
+
236
+ @abc.abstractmethod
237
+ def _inverse_transform(self, table: TableTensor) -> TableTensor: ...
238
+
239
+ def inverse_transform(self, table: TableTensor) -> TableTensor:
240
+ r"""Apply the inverse transformation to ``table``.
241
+
242
+ Args:
243
+ table: The table in transformed representation.
244
+
245
+ Returns:
246
+ The table restored to the representation before
247
+ :meth:`~Processor.transform`.
248
+ """
249
+ self._check_is_fitted()
250
+ return self._inverse_transform(table)
251
+
252
+ if TYPE_CHECKING:
253
+ # Provided at runtime by `Processor` via the MRO.
254
+ def _check_is_fitted(self) -> None: ...
@@ -0,0 +1,16 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Categorical preprocessing transforms."""
5
+
6
+ from sdm.processing.categorical.align import AlignCategories
7
+ from sdm.processing.categorical.shuffle import ShuffleCategories
8
+ from sdm.processing.categorical.impute import ImputeMode
9
+ from sdm.processing.categorical.count import AddCategoryCounts
10
+
11
+ __all__ = [
12
+ "AlignCategories",
13
+ "ShuffleCategories",
14
+ "ImputeMode",
15
+ "AddCategoryCounts",
16
+ ]
@@ -0,0 +1,60 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import os
5
+
6
+ from torch import Tensor
7
+
8
+ from sdm import Stype, TableTensor
9
+ from sdm.nn._buffer import BufferList
10
+
11
+
12
+ def _check_categorical_codes(table: TableTensor) -> None:
13
+ """Raise if a categorical code exceeds its column's category vocabulary.
14
+
15
+ Negative codes encode missing values and are allowed.
16
+
17
+ Args:
18
+ table: The table whose categorical codes are validated.
19
+ """
20
+ if os.getenv("SDM_ENABLE_CATEGORY_CHECKS", "1") == "0":
21
+ return
22
+ columns = table.columns[Stype.categorical]
23
+ for index, category in enumerate(table.categorical.categories):
24
+ codes = table.categorical[..., index]
25
+ if (codes >= category.numel()).any():
26
+ raise ValueError(
27
+ f"Categorical column {columns[index]!r} contains a code "
28
+ "outside its category vocabulary."
29
+ )
30
+
31
+
32
+ def _check_categories(
33
+ table: TableTensor,
34
+ categories: BufferList[Tensor],
35
+ ) -> None:
36
+ """Raise if a category vocabulary differs from the fitted one.
37
+
38
+ Args:
39
+ table: The table whose category vocabularies are validated.
40
+ categories: The fitted category vocabulary of every column.
41
+ """
42
+ if os.getenv("SDM_ENABLE_CATEGORY_CHECKS", "1") == "0":
43
+ return
44
+ columns = table.columns[Stype.categorical]
45
+ if len(table.categorical.categories) != len(categories):
46
+ raise ValueError(
47
+ f"Expected {len(categories)} fitted categorical "
48
+ f"columns (got {len(columns)})."
49
+ )
50
+ for index, (actual, expected) in enumerate(
51
+ zip(table.categorical.categories, categories)
52
+ ):
53
+ expected = expected.to(device=actual.device)
54
+ if not actual.equal(expected):
55
+ raise ValueError(
56
+ "Expected the category vocabulary for categorical column "
57
+ f"{columns[index]!r} to match the fitted values and order. "
58
+ "Use 'AlignCategories' before this processor for "
59
+ "independently tensorized inputs."
60
+ )