structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,325 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import math
5
+
6
+ import torch
7
+ from torch import Tensor
8
+
9
+ from sdm import Stype, TableTensor
10
+ from sdm.processing import InvertibleMixin, Processor
11
+ from sdm.processing.numerical._stats import (
12
+ _constant_feature_mask,
13
+ _isfinite,
14
+ )
15
+
16
+ # Keep GPU execution batched; adaptive per-column stopping would resynchronize.
17
+ # For float32 overflow-safe bounds, 44 golden steps reaches ~1.48e-8.
18
+ _YEOJOHNSON_OPTIMIZATION_STEPS = 44
19
+
20
+
21
+ def _yeojohnson_transform(
22
+ inp: Tensor,
23
+ lambdas: Tensor,
24
+ *,
25
+ magnitude_log: Tensor | None = None,
26
+ positive: Tensor | None = None,
27
+ exponents: Tensor | None = None,
28
+ out: Tensor | None = None,
29
+ ) -> Tensor:
30
+ eps = torch.finfo(inp.dtype).eps
31
+ two_minus_lambda = 2 - lambdas
32
+ if magnitude_log is None:
33
+ magnitude_log = inp.abs().log1p_()
34
+ if positive is None:
35
+ positive = inp >= 0
36
+ exponents = torch.where(positive, lambdas, two_minus_lambda, out=exponents)
37
+ out = torch.mul(exponents, magnitude_log, out=out)
38
+ out.expm1_().div_(exponents)
39
+ zero_exponent = torch.where(
40
+ positive, lambdas.abs() < eps, two_minus_lambda.abs() < eps
41
+ )
42
+ torch.where(zero_exponent, magnitude_log, out, out=out)
43
+ return out.copysign_(inp)
44
+
45
+
46
+ def _yeojohnson_inverse_transform(inp: Tensor, lambdas: Tensor) -> Tensor:
47
+ positive = inp >= 0
48
+ eps = torch.finfo(inp.dtype).eps
49
+
50
+ positive_power = ((inp * lambdas).log1p() / lambdas).expm1()
51
+ positive_log = inp.expm1()
52
+ positive_out = torch.where(
53
+ lambdas.abs() < eps,
54
+ positive_log,
55
+ positive_power,
56
+ )
57
+
58
+ two_minus_lambda = 2 - lambdas
59
+ negative_power = -(
60
+ (-two_minus_lambda * inp).log1p() / two_minus_lambda
61
+ ).expm1()
62
+ negative_log = -(-inp).expm1()
63
+ negative_out = torch.where(
64
+ two_minus_lambda.abs() < eps,
65
+ negative_log,
66
+ negative_power,
67
+ )
68
+
69
+ return torch.where(positive, positive_out, negative_out)
70
+
71
+
72
+ def _yeojohnson_bounds(inp: Tensor) -> tuple[Tensor, Tensor]:
73
+ missing = inp.isnan()
74
+ max_abs = inp.abs().nan_to_num_(nan=0.0).amax(dim=-2, keepdim=True)
75
+ log1p_max_x = (20 * max_abs).log1p()
76
+ log1p_max_x = torch.where(
77
+ max_abs == 0,
78
+ torch.ones_like(log1p_max_x),
79
+ log1p_max_x,
80
+ )
81
+ finfo = torch.finfo(inp.dtype)
82
+ log_eps = math.log(finfo.eps)
83
+ log_tiny_float = (math.log(finfo.tiny) - log_eps) / 2
84
+ log_max_float = (math.log(finfo.max) + log_eps) / 2
85
+
86
+ lower_bound = log_tiny_float / log1p_max_x
87
+ upper_bound = log_max_float / log1p_max_x
88
+ positive_lower = lower_bound
89
+ positive_upper = upper_bound
90
+
91
+ negative = inp < 0
92
+ all_negative = (negative | missing).all(dim=-2, keepdim=True)
93
+ any_negative = negative.any(dim=-2, keepdim=True)
94
+
95
+ mixed_lower = torch.maximum(2 - positive_upper, positive_lower)
96
+ mixed_upper = torch.minimum(2 - mixed_lower, positive_upper)
97
+ lower_bound = torch.where(any_negative, mixed_lower, positive_lower)
98
+ upper_bound = torch.where(any_negative, mixed_upper, positive_upper)
99
+
100
+ negative_lower = 2 - positive_upper
101
+ negative_upper = 2 - positive_lower
102
+ lower_bound = torch.where(all_negative, negative_lower, lower_bound)
103
+ upper_bound = torch.where(all_negative, negative_upper, upper_bound)
104
+
105
+ zero = max_abs == 0
106
+ return (
107
+ torch.where(zero, torch.ones_like(lower_bound), lower_bound),
108
+ torch.where(zero, torch.ones_like(upper_bound), upper_bound),
109
+ )
110
+
111
+
112
+ def _yeojohnson_log_likelihood(
113
+ inp: Tensor,
114
+ lambdas: Tensor,
115
+ magnitude_log: Tensor,
116
+ positive: Tensor,
117
+ log_jacobian: Tensor,
118
+ count: Tensor,
119
+ exponents: Tensor,
120
+ transformed: Tensor,
121
+ ) -> Tensor:
122
+ transformed = _yeojohnson_transform(
123
+ inp=inp,
124
+ lambdas=lambdas,
125
+ magnitude_log=magnitude_log,
126
+ positive=positive,
127
+ exponents=exponents,
128
+ out=transformed,
129
+ )
130
+ mean = transformed.nansum(dim=-2, keepdim=True).div_(count)
131
+ variance = transformed.sub_(mean).square_().nansum(dim=-2, keepdim=True)
132
+ variance /= count
133
+ tiny = torch.finfo(inp.dtype).tiny
134
+ valid = variance.isfinite() & (variance >= tiny)
135
+ loglike = variance.log_().mul_(-count / 2)
136
+ loglike.add_((lambdas - 1) * log_jacobian)
137
+ return loglike.masked_fill_(~valid, -math.inf)
138
+
139
+
140
+ def _optimize_lambdas(
141
+ inp: Tensor,
142
+ constant_features: Tensor,
143
+ *,
144
+ count: Tensor,
145
+ ) -> Tensor:
146
+ left, right = _yeojohnson_bounds(inp)
147
+ left = left.masked_fill(constant_features, 1.0)
148
+ right = right.masked_fill(constant_features, 1.0)
149
+
150
+ # Reuse full-table workspaces throughout the golden-section search.
151
+ magnitude_log = inp.abs().log1p_()
152
+ positive = inp >= 0
153
+ log_jacobian = magnitude_log.copysign(inp).nansum(dim=-2, keepdim=True)
154
+ exponents = torch.empty_like(inp)
155
+ transformed = torch.empty_like(inp)
156
+
157
+ invphi = (math.sqrt(5) - 1) / 2
158
+ span = (right - left).mul_(invphi)
159
+ c = right - span
160
+ d = left + span
161
+ fc = _yeojohnson_log_likelihood(
162
+ inp=inp,
163
+ lambdas=c,
164
+ magnitude_log=magnitude_log,
165
+ positive=positive,
166
+ log_jacobian=log_jacobian,
167
+ count=count,
168
+ exponents=exponents,
169
+ transformed=transformed,
170
+ )
171
+ fd = _yeojohnson_log_likelihood(
172
+ inp=inp,
173
+ lambdas=d,
174
+ magnitude_log=magnitude_log,
175
+ positive=positive,
176
+ log_jacobian=log_jacobian,
177
+ count=count,
178
+ exponents=exponents,
179
+ transformed=transformed,
180
+ )
181
+ c_next = torch.empty_like(c)
182
+ d_next = torch.empty_like(d)
183
+ new_point = torch.empty_like(c)
184
+ choose_right = torch.empty_like(c, dtype=torch.bool)
185
+
186
+ for _ in range(_YEOJOHNSON_OPTIMIZATION_STEPS):
187
+ choose_right = torch.lt(fc, fd, out=choose_right)
188
+ # Keep the search fully vectorized: each feature independently
189
+ # chooses its next interval without per-column Python branching.
190
+ left = torch.where(choose_right, c, left, out=left)
191
+ right = torch.where(choose_right, right, d, out=right)
192
+ span = torch.sub(right, left, out=span).mul_(invphi)
193
+ c_next = torch.sub(right, span, out=c_next)
194
+ c_next = torch.where(choose_right, d, c_next, out=c_next)
195
+ d_next = torch.add(left, span, out=d_next)
196
+ d_next = torch.where(choose_right, d_next, c, out=d_next)
197
+ new_point = torch.where(choose_right, d_next, c_next, out=new_point)
198
+ new_score = _yeojohnson_log_likelihood(
199
+ inp=inp,
200
+ lambdas=new_point,
201
+ magnitude_log=magnitude_log,
202
+ positive=positive,
203
+ log_jacobian=log_jacobian,
204
+ count=count,
205
+ exponents=exponents,
206
+ transformed=transformed,
207
+ )
208
+ fc = torch.where(choose_right, new_score, fc, out=fc)
209
+ fd = torch.where(choose_right, fd, new_score, out=fd)
210
+ fc, fd = fd, fc
211
+ c, c_next = c_next, c
212
+ d, d_next = d_next, d
213
+
214
+ lambdas = (left + right).div_(2)
215
+ return lambdas.masked_fill_(constant_features, 1.0)
216
+
217
+
218
+ class PowerTransform(Processor, InvertibleMixin):
219
+ """Apply a feature-wise Yeo-Johnson power transform.
220
+
221
+ NaN and infinite values are left out of the fitted statistics. NaN values
222
+ are preserved during the transform.
223
+
224
+ Args:
225
+ standardize: If ``True``, zero-mean and unit-variance the transformed
226
+ features using statistics fitted after the power transform.
227
+ """
228
+
229
+ handles_stypes = frozenset({Stype.numerical})
230
+ requires_fit = True
231
+
232
+ def __init__(
233
+ self,
234
+ *,
235
+ standardize: bool = True,
236
+ ) -> None:
237
+ super().__init__()
238
+ self.standardize = standardize
239
+ self.register_buffer("lambdas", torch.empty(0))
240
+ self.register_buffer("max", torch.empty(0))
241
+ self.register_buffer("mean", torch.empty(0))
242
+ self.register_buffer("scale", torch.empty(0))
243
+
244
+ def _optimize_lambdas(
245
+ self,
246
+ inp: Tensor,
247
+ constant_features: Tensor,
248
+ *,
249
+ count: Tensor,
250
+ ) -> Tensor:
251
+ return _optimize_lambdas(inp, constant_features, count=count)
252
+
253
+ def _fit(
254
+ self,
255
+ table: TableTensor,
256
+ *,
257
+ generator: torch.Generator | None = None,
258
+ ) -> None:
259
+ finite = _isfinite(table.numerical)
260
+ finite_or_nan = table.numerical.masked_fill(~finite, torch.nan)
261
+ count = finite.sum(dim=-2, keepdim=True).clamp_(min=1)
262
+
263
+ mean = finite_or_nan.nansum(-2, keepdim=True).div_(count)
264
+ var = finite_or_nan.sub(mean).square_().nansum(-2, keepdim=True)
265
+ var /= count
266
+ constant_features = _constant_feature_mask(
267
+ var,
268
+ mean,
269
+ num_samples=count,
270
+ )
271
+ del var
272
+
273
+ # ``mean`` never exceeds the column maximum, and is zero for an
274
+ # entirely missing column, whose max must be zero.
275
+ self.max = torch.where(finite, table.numerical, mean).amax(
276
+ dim=-2,
277
+ keepdim=True,
278
+ )
279
+ del finite
280
+
281
+ self.lambdas = self._optimize_lambdas(
282
+ finite_or_nan,
283
+ constant_features,
284
+ count=count,
285
+ )
286
+
287
+ if self.standardize:
288
+ transformed = _yeojohnson_transform(finite_or_nan, self.lambdas)
289
+ del finite_or_nan
290
+ mean = transformed.nansum(dim=-2, keepdim=True).div_(count)
291
+ var = transformed.sub_(mean).square_().nansum(-2, keepdim=True)
292
+ var /= count
293
+ scale = var.sqrt()
294
+ scale[_constant_feature_mask(var, mean, count)] = 1.0
295
+ self.mean = mean
296
+ self.scale = scale
297
+ else:
298
+ self.mean = torch.zeros_like(self.lambdas)
299
+ self.scale = torch.ones_like(self.lambdas)
300
+
301
+ def _transform(self, table: TableTensor) -> TableTensor:
302
+ """Transform ``table`` with fitted Yeo-Johnson parameters."""
303
+ transformed = _yeojohnson_transform(table.numerical, self.lambdas)
304
+ numerical = transformed.sub_(self.mean).div_(self.scale)
305
+ # The fitted lambdas only keep the fitted range representable, so a
306
+ # query far outside it can overflow.
307
+ bound = torch.finfo(numerical.dtype).max
308
+ return table.replace_blocks(
309
+ numerical=numerical.clamp_(min=-bound, max=bound)
310
+ )
311
+
312
+ def _inverse_transform(self, table: TableTensor) -> TableTensor:
313
+ unscaled = table.numerical * self.scale + self.mean
314
+ inverse = _yeojohnson_inverse_transform(unscaled, self.lambdas)
315
+
316
+ # Above the fitted upper bound the inverse diverges, either to
317
+ # infinity or, past the asymptote, to NaN.
318
+ diverged = ~_isfinite(inverse) & ~unscaled.isnan()
319
+ return table.replace_blocks(
320
+ numerical=torch.where(
321
+ diverged,
322
+ torch.fmin(inverse, self.max),
323
+ inverse,
324
+ )
325
+ )
@@ -0,0 +1,306 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from typing import Literal
5
+
6
+ import torch
7
+ from torch import Tensor
8
+
9
+ from sdm import Stype, TableTensor
10
+ from sdm.processing import InvertibleMixin, Processor
11
+ from sdm.processing.numerical._stats import _ndtri
12
+
13
+ BOUNDS_THRESH = 1e-7
14
+ _MAX_NUM_COLS = 32
15
+
16
+
17
+ def _torch_interp(x: Tensor, xp: Tensor, fp: Tensor) -> Tensor:
18
+ n = xp.numel()
19
+ if n == 1:
20
+ return fp[0].expand_as(x)
21
+
22
+ idx = torch.searchsorted(xp, x, right=True).clamp(1, n - 1)
23
+
24
+ x0, x1 = xp[idx - 1], xp[idx]
25
+ y0, y1 = fp[idx - 1], fp[idx]
26
+
27
+ denom = x1 - x0
28
+ weight = torch.where(denom != 0, (x - x0) / denom, 0.0)
29
+ result = torch.lerp(y0, y1, weight)
30
+
31
+ result = torch.where(x <= xp[0], fp[0], result)
32
+ return torch.where(x >= xp[-1], fp[-1], result)
33
+
34
+
35
+ def _batched_interp(
36
+ values: Tensor,
37
+ boundaries: Tensor,
38
+ references: Tensor,
39
+ ) -> Tensor:
40
+ # Batched 1D linear interpolation over independent feature grids:
41
+ # values ``[F, N]`` with boundaries/references as ``[Q]`` or ``[F, Q]``.
42
+ n = boundaries.shape[-1]
43
+ if n == 1:
44
+ if references.dim() == 1:
45
+ return references[0].expand_as(values)
46
+ return references[:, :1].expand_as(values)
47
+
48
+ idx = torch.searchsorted(boundaries, values, right=True).clamp(1, n - 1)
49
+
50
+ if boundaries.dim() == 1:
51
+ x0 = boundaries[idx - 1]
52
+ x1 = boundaries[idx]
53
+ lower_boundary = boundaries[0]
54
+ upper_boundary = boundaries[-1]
55
+ else:
56
+ x0 = boundaries.gather(1, idx - 1)
57
+ x1 = boundaries.gather(1, idx)
58
+ lower_boundary = boundaries[:, :1]
59
+ upper_boundary = boundaries[:, -1:]
60
+
61
+ if references.dim() == 1:
62
+ y0 = references[idx - 1]
63
+ y1 = references[idx]
64
+ lower = references[0]
65
+ upper = references[-1]
66
+ else:
67
+ y0 = references.gather(1, idx - 1)
68
+ y1 = references.gather(1, idx)
69
+ lower = references[:, :1]
70
+ upper = references[:, -1:]
71
+
72
+ denom = x1 - x0
73
+ weight = torch.where(denom != 0, (values - x0) / denom, 0.0)
74
+ result = torch.lerp(y0, y1, weight)
75
+
76
+ result = torch.where(values <= lower_boundary, lower, result)
77
+ return torch.where(values >= upper_boundary, upper, result)
78
+
79
+
80
+ class QuantileTransform(Processor, InvertibleMixin):
81
+ """Map numerical columns through their empirical quantiles.
82
+
83
+ QuantileTransform grids are capped by the number of fitted rows and, when
84
+ ``subsample`` is set, by ``20%`` of the subsample size to keep dense grids
85
+ tractable.
86
+
87
+ The subsample rows are drawn from the ``generator`` passed to ``fit()``;
88
+ without one, they are drawn from the data device's global generator.
89
+
90
+ Args:
91
+ n_quantiles: Maximum number of quantiles to compute.
92
+ subsample: Maximum number of rows to use for quantile computation.
93
+ output_distribution: Distribution to map the empirical quantiles to.
94
+ """
95
+
96
+ handles_stypes = frozenset({Stype.numerical})
97
+ requires_fit = True
98
+
99
+ _quantiles: Tensor
100
+ _references: Tensor
101
+
102
+ def __init__(
103
+ self,
104
+ *,
105
+ n_quantiles: int = 1000,
106
+ subsample: int | None = 10_000,
107
+ output_distribution: Literal["uniform", "normal"] = "uniform",
108
+ ) -> None:
109
+ super().__init__()
110
+ if n_quantiles <= 0:
111
+ raise ValueError("n_quantiles must be positive.")
112
+ if subsample is not None and subsample <= 0:
113
+ raise ValueError("subsample must be positive or None.")
114
+ self._n_quantiles = n_quantiles
115
+ self.subsample = subsample
116
+ self.output_distribution = output_distribution
117
+ self.register_buffer(
118
+ "_quantiles",
119
+ torch.empty(0),
120
+ )
121
+ self.register_buffer(
122
+ "_references",
123
+ torch.empty(0),
124
+ )
125
+
126
+ def _subsample_indices(
127
+ self,
128
+ inp: Tensor,
129
+ generator: torch.Generator | None,
130
+ ) -> Tensor:
131
+ return torch.randperm(
132
+ inp.size(-2),
133
+ generator=generator,
134
+ device=inp.device,
135
+ )[: self.subsample]
136
+
137
+ def _fit(
138
+ self,
139
+ table: TableTensor,
140
+ *,
141
+ generator: torch.Generator | None = None,
142
+ ) -> None:
143
+ numerical = table.numerical
144
+ n_samples = numerical.size(-2)
145
+ quantile_limit = n_samples
146
+ if self.subsample is not None:
147
+ # Keep quantiles well below the subsample size; very dense
148
+ # percentile grids are slow to fit and add little resolution.
149
+ quantile_limit = min(
150
+ quantile_limit,
151
+ int(self.subsample * 0.2),
152
+ )
153
+ n_quantiles = max(
154
+ 1,
155
+ min(self._n_quantiles, quantile_limit),
156
+ )
157
+ references = torch.linspace(
158
+ 0,
159
+ 1,
160
+ n_quantiles,
161
+ device=numerical.device,
162
+ dtype=numerical.dtype,
163
+ )
164
+
165
+ if self.subsample is not None and self.subsample < n_samples:
166
+ indices = self._subsample_indices(numerical, generator)
167
+ input_sample = numerical.index_select(-2, indices)
168
+ else:
169
+ input_sample = numerical
170
+
171
+ sample_size = input_sample.size(-2)
172
+ sample_columns = (
173
+ input_sample.movedim(-1, -2).reshape(-1, sample_size).contiguous()
174
+ )
175
+ quantiles = torch.quantile(
176
+ sample_columns,
177
+ references,
178
+ dim=-1,
179
+ )
180
+ quantiles = (
181
+ quantiles.movedim(0, -1)
182
+ .reshape(
183
+ *numerical.shape[:-2],
184
+ numerical.size(-1),
185
+ n_quantiles,
186
+ )
187
+ .movedim(-1, -2)
188
+ )
189
+ self._quantiles = quantiles
190
+ self._references = references
191
+
192
+ def _transform(self, table: TableTensor) -> TableTensor:
193
+ numerical = table.numerical
194
+ n_samples = numerical.size(-2)
195
+ n_features = numerical.size(-1)
196
+ input_columns = (
197
+ numerical.movedim(-1, -2).reshape(-1, n_samples).contiguous()
198
+ )
199
+ quantile_columns = (
200
+ self._quantiles.movedim(-1, -2)
201
+ .reshape(-1, self._references.numel())
202
+ .contiguous()
203
+ )
204
+ transformed_columns = torch.empty_like(input_columns)
205
+
206
+ for start in range(0, input_columns.size(0), _MAX_NUM_COLS):
207
+ end = min(start + _MAX_NUM_COLS, input_columns.size(0))
208
+ input_chunk = input_columns[start:end]
209
+ quantile_chunk = quantile_columns[start:end]
210
+ lower_bound_x = quantile_chunk[:, :1]
211
+ upper_bound_x = quantile_chunk[:, -1:]
212
+ if self.output_distribution == "normal":
213
+ bounds_thresh = input_chunk.new_tensor(BOUNDS_THRESH)
214
+ lower_bounds_idx = input_chunk - bounds_thresh < lower_bound_x
215
+ upper_bounds_idx = input_chunk + bounds_thresh > upper_bound_x
216
+ elif self.output_distribution == "uniform":
217
+ lower_bounds_idx = input_chunk == lower_bound_x
218
+ upper_bounds_idx = input_chunk == upper_bound_x
219
+ else:
220
+ raise ValueError(
221
+ "output_distribution must be 'uniform' or 'normal'."
222
+ )
223
+
224
+ forward = _batched_interp(
225
+ input_chunk,
226
+ quantile_chunk,
227
+ self._references,
228
+ )
229
+ backward = _batched_interp(
230
+ -input_chunk,
231
+ -quantile_chunk.flip(1),
232
+ -self._references.flip(0),
233
+ )
234
+ output = 0.5 * (forward - backward)
235
+
236
+ output = torch.where(upper_bounds_idx, 1.0, output)
237
+ output = torch.where(lower_bounds_idx, 0.0, output)
238
+
239
+ if self.output_distribution == "normal":
240
+ eps = input_chunk.new_tensor(
241
+ BOUNDS_THRESH - torch.finfo(torch.float64).eps
242
+ )
243
+ output = _ndtri(output)
244
+ clip_min = _ndtri(eps)
245
+ clip_max = _ndtri(1.0 - eps)
246
+ output = output.clamp(clip_min, clip_max)
247
+
248
+ transformed_columns[start:end] = output
249
+
250
+ transformed = transformed_columns.reshape(
251
+ *numerical.shape[:-2],
252
+ n_features,
253
+ n_samples,
254
+ ).movedim(-1, -2)
255
+ return table.replace_blocks(numerical=transformed)
256
+
257
+ def _inverse_transform(self, table: TableTensor) -> TableTensor:
258
+ numerical = table.numerical
259
+ n_samples = numerical.size(-2)
260
+ n_features = numerical.size(-1)
261
+ input_columns = (
262
+ numerical.movedim(-1, -2).reshape(-1, n_samples).contiguous()
263
+ )
264
+ quantile_columns = (
265
+ self._quantiles.movedim(-1, -2)
266
+ .reshape(-1, self._references.numel())
267
+ .contiguous()
268
+ )
269
+ inverse_columns = torch.empty_like(input_columns)
270
+
271
+ for start in range(0, input_columns.size(0), _MAX_NUM_COLS):
272
+ end = min(start + _MAX_NUM_COLS, input_columns.size(0))
273
+ input_chunk = input_columns[start:end]
274
+ quantile_chunk = quantile_columns[start:end]
275
+ lower_bound_y = quantile_chunk[:, :1]
276
+ upper_bound_y = quantile_chunk[:, -1:]
277
+ if self.output_distribution == "normal":
278
+ input_chunk = torch.special.ndtr(input_chunk)
279
+
280
+ if self.output_distribution == "normal":
281
+ bounds_thresh = input_chunk.new_tensor(BOUNDS_THRESH)
282
+ lower_bounds_idx = input_chunk - bounds_thresh < 0.0
283
+ upper_bounds_idx = input_chunk + bounds_thresh > 1.0
284
+ elif self.output_distribution == "uniform":
285
+ lower_bounds_idx = input_chunk == 0.0
286
+ upper_bounds_idx = input_chunk == 1.0
287
+ else:
288
+ raise ValueError(
289
+ "output_distribution must be 'uniform' or 'normal'."
290
+ )
291
+
292
+ output = _batched_interp(
293
+ input_chunk,
294
+ self._references,
295
+ quantile_chunk,
296
+ )
297
+ output = torch.where(upper_bounds_idx, upper_bound_y, output)
298
+ output = torch.where(lower_bounds_idx, lower_bound_y, output)
299
+ inverse_columns[start:end] = output
300
+
301
+ inverse = inverse_columns.reshape(
302
+ *numerical.shape[:-2],
303
+ n_features,
304
+ n_samples,
305
+ ).movedim(-1, -2)
306
+ return table.replace_blocks(numerical=inverse)
@@ -0,0 +1,63 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import torch
5
+
6
+ from sdm import Stype, TableTensor
7
+ from sdm.processing import Processor
8
+
9
+
10
+ class ClipQuantiles(Processor):
11
+ """Clamp feature columns to fitted quantile bounds.
12
+
13
+ Values outside the fitted bounds are discarded, so this processor is not
14
+ invertible. Quantile transform bounds are fitted independently for each
15
+ feature column.
16
+
17
+ Args:
18
+ q_low: Lower quantile in ``[0, 1]`` used as the per-column lower bound.
19
+ q_high: Upper quantile in ``[0, 1]`` used as the per-column upper
20
+ bound. Must satisfy ``0 <= q_low <= q_high <= 1``.
21
+ """
22
+
23
+ handles_stypes = frozenset({Stype.numerical})
24
+ requires_fit = True
25
+
26
+ def __init__(
27
+ self,
28
+ *,
29
+ q_low: float = 0.0,
30
+ q_high: float = 1.0,
31
+ ) -> None:
32
+ super().__init__()
33
+ if q_low > q_high:
34
+ raise ValueError("q_low and q_high must satisfy q_low <= q_high.")
35
+ self.q_low = q_low
36
+ self.q_high = q_high
37
+ self.register_buffer("lower_bound", torch.empty(0))
38
+ self.register_buffer("upper_bound", torch.empty(0))
39
+
40
+ def _fit(
41
+ self,
42
+ table: TableTensor,
43
+ *,
44
+ generator: torch.Generator | None = None,
45
+ ) -> None:
46
+ numerical = table.numerical
47
+ quantiles = numerical.new_tensor([self.q_low, self.q_high])
48
+ q_low, q_high = torch.quantile(
49
+ numerical,
50
+ quantiles,
51
+ dim=-2,
52
+ keepdim=True,
53
+ )
54
+ self.lower_bound = q_low
55
+ self.upper_bound = q_high
56
+
57
+ def _transform(self, table: TableTensor) -> TableTensor:
58
+ """Clamp ``table`` to the fitted lower and upper bounds."""
59
+ numerical = table.numerical.clamp(
60
+ min=self.lower_bound,
61
+ max=self.upper_bound,
62
+ )
63
+ return table.replace_blocks(numerical=numerical)