structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from typing import Literal, cast
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
|
|
8
|
+
from sdm import EnsembleTable, Stype, TableTensor
|
|
9
|
+
from sdm.nn._buffer import BufferList
|
|
10
|
+
from sdm.processing import EnsembleProcessor
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class RandomProjection(EnsembleProcessor):
|
|
14
|
+
r"""Randomly project numerical columns.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
channels: Number of output numerical columns.
|
|
18
|
+
init: The projection weight initialization.
|
|
19
|
+
``"normal"`` samples Gaussian weights scaled by
|
|
20
|
+
``1 / sqrt(channels)``.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
24
|
+
requires_fit = True
|
|
25
|
+
|
|
26
|
+
def __init__(
|
|
27
|
+
self,
|
|
28
|
+
channels: int,
|
|
29
|
+
*,
|
|
30
|
+
init: Literal["normal"] = "normal",
|
|
31
|
+
) -> None:
|
|
32
|
+
super().__init__()
|
|
33
|
+
self.channels = channels
|
|
34
|
+
self.init = init
|
|
35
|
+
self._weights: BufferList[torch.Tensor] = BufferList()
|
|
36
|
+
|
|
37
|
+
def _fit_ensemble(
|
|
38
|
+
self,
|
|
39
|
+
ensemble_table: EnsembleTable,
|
|
40
|
+
*,
|
|
41
|
+
generator: torch.Generator | None = None,
|
|
42
|
+
) -> None:
|
|
43
|
+
weights = []
|
|
44
|
+
for i, group in enumerate(ensemble_table._iter_groups()):
|
|
45
|
+
num_members = ensemble_table.num_members_in_group(i)
|
|
46
|
+
weight = group.numerical.new_empty(
|
|
47
|
+
(
|
|
48
|
+
num_members,
|
|
49
|
+
*group.size()[1:-2],
|
|
50
|
+
self.channels,
|
|
51
|
+
group.numerical.size(-1),
|
|
52
|
+
)
|
|
53
|
+
)
|
|
54
|
+
assert self.init == "normal"
|
|
55
|
+
weight.normal_(std=self.channels**-0.5, generator=generator)
|
|
56
|
+
weights.append(weight)
|
|
57
|
+
self._weights = BufferList(weights)
|
|
58
|
+
|
|
59
|
+
def _transform_ensemble(
|
|
60
|
+
self,
|
|
61
|
+
ensemble_table: EnsembleTable,
|
|
62
|
+
) -> EnsembleTable:
|
|
63
|
+
groups = []
|
|
64
|
+
for i in range(ensemble_table.num_groups):
|
|
65
|
+
group = ensemble_table.expanded_group(i)
|
|
66
|
+
projected = TableTensor(
|
|
67
|
+
columns={
|
|
68
|
+
Stype.numerical: [f"rp_{i}" for i in range(self.channels)]
|
|
69
|
+
},
|
|
70
|
+
numerical=group.numerical @ self._weights[i].transpose(-1, -2),
|
|
71
|
+
)
|
|
72
|
+
group = cast(
|
|
73
|
+
TableTensor,
|
|
74
|
+
torch.cat(
|
|
75
|
+
[group.drop_stypes(Stype.numerical), projected], dim=-1
|
|
76
|
+
),
|
|
77
|
+
)
|
|
78
|
+
groups.append(group)
|
|
79
|
+
|
|
80
|
+
locations = []
|
|
81
|
+
group_position = [0] * ensemble_table.num_groups
|
|
82
|
+
for group_id, _ in ensemble_table._locations:
|
|
83
|
+
locations.append((group_id, group_position[group_id]))
|
|
84
|
+
group_position[group_id] += 1
|
|
85
|
+
|
|
86
|
+
return EnsembleTable(groups=groups, locations=locations)
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
|
|
6
|
+
from sdm import Stype, TableTensor
|
|
7
|
+
from sdm._memory import split_size
|
|
8
|
+
from sdm.processing import Processor
|
|
9
|
+
from sdm.processing.numerical._stats import _isfinite, _ndtri
|
|
10
|
+
from sdm.processing.numerical.quantile import _batched_interp
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class RankGaussian(Processor):
|
|
14
|
+
"""Map interpolated empirical mid-ranks to standard normal quantiles.
|
|
15
|
+
|
|
16
|
+
Each fitted value receives probability ``(L + R) / (2 * N)``, where
|
|
17
|
+
``L`` and ``R`` count finite fitted values strictly below and at or below
|
|
18
|
+
it, and ``N`` is the number of finite fitted values. Ties share a rank.
|
|
19
|
+
Query probabilities interpolate between retained fitted value-rank pairs
|
|
20
|
+
and clamp to the endpoint probabilities.
|
|
21
|
+
|
|
22
|
+
When ``max_knots`` is set, each column retains at most that many pairs,
|
|
23
|
+
selected at evenly spaced normal quantiles between the fitted extremes.
|
|
24
|
+
|
|
25
|
+
NaN and infinite values are ignored during fitting. NaNs are preserved
|
|
26
|
+
during transformation. Constant columns map to zero; columns without
|
|
27
|
+
finite fitted values produce NaNs.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
max_knots: Maximum number of knots per column, at least ``2``. If
|
|
31
|
+
``None``, retain all fitted values.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
35
|
+
requires_fit = True
|
|
36
|
+
|
|
37
|
+
def __init__(self, *, max_knots: int | None = None) -> None:
|
|
38
|
+
super().__init__()
|
|
39
|
+
if max_knots is not None and max_knots < 2:
|
|
40
|
+
raise ValueError("max_knots must be at least 2.")
|
|
41
|
+
self.max_knots = max_knots
|
|
42
|
+
self.register_buffer("_values", torch.empty(0))
|
|
43
|
+
self.register_buffer("_probabilities", torch.empty(0))
|
|
44
|
+
|
|
45
|
+
def _fit(
|
|
46
|
+
self,
|
|
47
|
+
table: TableTensor,
|
|
48
|
+
*,
|
|
49
|
+
generator: torch.Generator | None = None,
|
|
50
|
+
) -> None:
|
|
51
|
+
numerical = table.numerical
|
|
52
|
+
dtype = torch.promote_types(numerical.dtype, torch.float32)
|
|
53
|
+
# Columns are fitted on their own, so chunks of columns bound the
|
|
54
|
+
# sorting and ranking temporaries of about ten values per cell.
|
|
55
|
+
size = split_size(
|
|
56
|
+
num_items=numerical.size(-1),
|
|
57
|
+
item_bytes=numerical[..., :1].numel() * 10 * dtype.itemsize,
|
|
58
|
+
device=numerical.device,
|
|
59
|
+
)
|
|
60
|
+
knots = []
|
|
61
|
+
for chunk in numerical.split(size, dim=-1):
|
|
62
|
+
num_rows = chunk.size(-2)
|
|
63
|
+
max_knots = num_rows if self.max_knots is None else self.max_knots
|
|
64
|
+
columns = chunk.to(dtype).movedim(-1, -2) # [..., C, N]
|
|
65
|
+
finite = _isfinite(columns)
|
|
66
|
+
count = finite.sum(dim=-1, keepdim=True) # [..., C, 1]
|
|
67
|
+
values = columns.masked_fill(~finite, torch.inf)
|
|
68
|
+
values = values.sort(dim=-1).values.contiguous()
|
|
69
|
+
left = torch.searchsorted(values, values, right=False)
|
|
70
|
+
right = torch.searchsorted(values, values, right=True)
|
|
71
|
+
probabilities = (left + right).to(values.dtype) / (
|
|
72
|
+
2 * count.clamp_min(1)
|
|
73
|
+
)
|
|
74
|
+
# Rounding to one would give an infinite normal quantile.
|
|
75
|
+
probabilities.clamp_(max=1 - torch.finfo(dtype).eps)
|
|
76
|
+
last = (count - 1).clamp_min(0)
|
|
77
|
+
rows = torch.arange(num_rows, device=values.device)
|
|
78
|
+
|
|
79
|
+
if num_rows <= max_knots:
|
|
80
|
+
positions = rows.minimum(last)
|
|
81
|
+
else:
|
|
82
|
+
# Index among distinct column values; a value starts a new
|
|
83
|
+
# one at its first occurrence.
|
|
84
|
+
distinct = (left == rows).cumsum(dim=-1).sub_(1)
|
|
85
|
+
num_distinct = distinct.gather(-1, last) + 1
|
|
86
|
+
# Knots at the first occurrence of each distinct value, padded
|
|
87
|
+
# with the largest one.
|
|
88
|
+
steps = torch.arange(max_knots, device=values.device)
|
|
89
|
+
positions = torch.searchsorted(
|
|
90
|
+
distinct,
|
|
91
|
+
steps.minimum(num_distinct - 1),
|
|
92
|
+
)
|
|
93
|
+
# With more distinct values than knots, select rows whose
|
|
94
|
+
# mid-ranks are closest to normal quantiles spaced evenly
|
|
95
|
+
# between the extremes. Mid-ranks never decrease along rows.
|
|
96
|
+
lower = _ndtri(probabilities[..., :1])
|
|
97
|
+
upper = _ndtri(probabilities.gather(-1, last))
|
|
98
|
+
quantiles = torch.special.ndtr(
|
|
99
|
+
lower.lerp(
|
|
100
|
+
upper,
|
|
101
|
+
steps.to(probabilities.dtype) / (max_knots - 1),
|
|
102
|
+
)
|
|
103
|
+
)
|
|
104
|
+
above = torch.searchsorted(probabilities, quantiles).minimum(
|
|
105
|
+
last
|
|
106
|
+
)
|
|
107
|
+
below = (above - 1).clamp_(min=0)
|
|
108
|
+
closer = (quantiles - probabilities.gather(-1, below)) < (
|
|
109
|
+
probabilities.gather(-1, above) - quantiles
|
|
110
|
+
)
|
|
111
|
+
spaced = torch.where(closer, below, above)
|
|
112
|
+
spaced[..., :1] = 0
|
|
113
|
+
spaced[..., -1:] = last
|
|
114
|
+
positions = torch.where(
|
|
115
|
+
num_distinct <= max_knots,
|
|
116
|
+
positions,
|
|
117
|
+
spaced,
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
missing = count == 0
|
|
121
|
+
knots.append(
|
|
122
|
+
(
|
|
123
|
+
values.gather(-1, positions).masked_fill_(
|
|
124
|
+
missing, torch.nan
|
|
125
|
+
),
|
|
126
|
+
probabilities.gather(-1, positions).masked_fill_(
|
|
127
|
+
missing,
|
|
128
|
+
torch.nan,
|
|
129
|
+
),
|
|
130
|
+
)
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
values, probabilities = zip(*knots, strict=True)
|
|
134
|
+
self._values = torch.cat(values, dim=-2)
|
|
135
|
+
self._probabilities = torch.cat(probabilities, dim=-2)
|
|
136
|
+
|
|
137
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
138
|
+
numerical = table.numerical
|
|
139
|
+
columns = numerical.movedim(-1, -2) # [..., C, R]
|
|
140
|
+
values = self._values.flatten(end_dim=-2)
|
|
141
|
+
probabilities = self._probabilities.flatten(end_dim=-2)
|
|
142
|
+
output = torch.empty_like(numerical)
|
|
143
|
+
# Rows transform on their own, so chunks of rows bound the
|
|
144
|
+
# interpolation temporaries of about a dozen values per cell.
|
|
145
|
+
size = split_size(
|
|
146
|
+
num_items=columns.size(-1),
|
|
147
|
+
item_bytes=columns[..., :1].numel() * 12 * values.itemsize,
|
|
148
|
+
device=numerical.device,
|
|
149
|
+
)
|
|
150
|
+
for rows, out in zip(
|
|
151
|
+
columns.split(size, dim=-1),
|
|
152
|
+
output.split(size, dim=-2),
|
|
153
|
+
strict=True,
|
|
154
|
+
):
|
|
155
|
+
quantiles = _batched_interp(
|
|
156
|
+
rows.flatten(end_dim=-2).to(values.dtype).contiguous(),
|
|
157
|
+
values,
|
|
158
|
+
probabilities,
|
|
159
|
+
)
|
|
160
|
+
normal = _ndtri(quantiles).reshape(rows.shape)
|
|
161
|
+
out.copy_(normal.masked_fill_(rows.isnan(), torch.nan).mT)
|
|
162
|
+
return table.replace_blocks(numerical=output)
|
|
163
|
+
|
|
164
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
165
|
+
return (
|
|
166
|
+
f"{' ' * indent}{self.__class__.__name__}("
|
|
167
|
+
f"max_knots={self.max_knots})"
|
|
168
|
+
)
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
|
|
6
|
+
from sdm import Stype, TableTensor
|
|
7
|
+
from sdm._memory import split_size
|
|
8
|
+
from sdm.processing import InvertibleMixin, Processor
|
|
9
|
+
from sdm.processing.numerical._stats import _isfinite
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class RobustScale(Processor, InvertibleMixin):
|
|
13
|
+
"""Center and scale each feature column with median and quantile range.
|
|
14
|
+
|
|
15
|
+
Columns whose lower and upper quantile coincide use a unit scale to keep
|
|
16
|
+
the transform finite and invertible. NaN and infinite values are ignored
|
|
17
|
+
when fitting statistics and preserved during the transform.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
quantile_range: Percentile pair ``(low, high)`` in ``[0, 100]``
|
|
21
|
+
used as the scale.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
25
|
+
requires_fit = True
|
|
26
|
+
|
|
27
|
+
def __init__(
|
|
28
|
+
self,
|
|
29
|
+
*,
|
|
30
|
+
quantile_range: tuple[float, float] = (25.0, 75.0),
|
|
31
|
+
) -> None:
|
|
32
|
+
super().__init__()
|
|
33
|
+
low, high = quantile_range
|
|
34
|
+
if not 0 <= low <= high <= 100:
|
|
35
|
+
raise ValueError(
|
|
36
|
+
"quantile_range must satisfy 0 <= low <= high <= 100."
|
|
37
|
+
)
|
|
38
|
+
self.quantile_range = (low, high)
|
|
39
|
+
self.register_buffer("median", torch.empty(0))
|
|
40
|
+
self.register_buffer("scale", torch.empty(0))
|
|
41
|
+
|
|
42
|
+
def _fit(
|
|
43
|
+
self,
|
|
44
|
+
table: TableTensor,
|
|
45
|
+
*,
|
|
46
|
+
generator: torch.Generator | None = None,
|
|
47
|
+
) -> None:
|
|
48
|
+
numerical = table.numerical
|
|
49
|
+
finite_or_nan = numerical.masked_fill(~_isfinite(numerical), torch.nan)
|
|
50
|
+
q_low, q_high = (value / 100.0 for value in self.quantile_range)
|
|
51
|
+
# 'nanquantile' requires single or double precision input.
|
|
52
|
+
quantile_input = finite_or_nan.to(
|
|
53
|
+
dtype=torch.promote_types(numerical.dtype, torch.float32),
|
|
54
|
+
)
|
|
55
|
+
q = quantile_input.new_tensor([q_low, 0.5, q_high])
|
|
56
|
+
# 'nanquantile' allocates several copies of its input, including
|
|
57
|
+
# int64 sort indices, which chunks of the independent columns bound.
|
|
58
|
+
size = split_size(
|
|
59
|
+
num_items=quantile_input.size(-1),
|
|
60
|
+
item_bytes=quantile_input[..., :1].numel()
|
|
61
|
+
* 4
|
|
62
|
+
* torch.int64.itemsize,
|
|
63
|
+
device=quantile_input.device,
|
|
64
|
+
)
|
|
65
|
+
lower, median, upper = torch.cat(
|
|
66
|
+
[
|
|
67
|
+
chunk.nanquantile(q, dim=-2, keepdim=True)
|
|
68
|
+
for chunk in quantile_input.split(size, dim=-1)
|
|
69
|
+
],
|
|
70
|
+
dim=-1,
|
|
71
|
+
)
|
|
72
|
+
self.median = median.to(dtype=numerical.dtype)
|
|
73
|
+
scale = torch.where(lower == upper, 1.0, upper - lower)
|
|
74
|
+
self.scale = scale.to(dtype=numerical.dtype)
|
|
75
|
+
|
|
76
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
77
|
+
numerical = (table.numerical - self.median).div_(self.scale)
|
|
78
|
+
return table.replace_blocks(numerical=numerical)
|
|
79
|
+
|
|
80
|
+
def _inverse_transform(self, table: TableTensor) -> TableTensor:
|
|
81
|
+
numerical = (table.numerical * self.scale).add_(self.median)
|
|
82
|
+
return table.replace_blocks(numerical=numerical)
|
|
83
|
+
|
|
84
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
85
|
+
return (
|
|
86
|
+
f"{' ' * indent}{self.__class__.__name__}("
|
|
87
|
+
f"quantile_range={self.quantile_range})"
|
|
88
|
+
)
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import math
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
|
|
8
|
+
from sdm import Stype, TableTensor
|
|
9
|
+
from sdm._memory import split_size
|
|
10
|
+
from sdm.processing import Processor
|
|
11
|
+
from sdm.processing.numerical._stats import _isfinite
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ClipSigma(Processor):
|
|
15
|
+
"""Two-stage z-score outlier clipping with soft logarithmic bounds.
|
|
16
|
+
|
|
17
|
+
The first pass masks values outside the initial z-score bounds, then the
|
|
18
|
+
second pass refits bounds on the remaining values. The transform applies
|
|
19
|
+
logarithmic soft clipping instead of hard truncation. NaN and infinite
|
|
20
|
+
values are ignored when fitting statistics and preserved during the
|
|
21
|
+
transform.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
threshold: Positive z-score multiplier setting how many standard
|
|
25
|
+
deviations from the mean mark the soft clipping bounds.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
29
|
+
requires_fit = True
|
|
30
|
+
|
|
31
|
+
def __init__(
|
|
32
|
+
self,
|
|
33
|
+
*,
|
|
34
|
+
threshold: float = 4.0,
|
|
35
|
+
) -> None:
|
|
36
|
+
super().__init__()
|
|
37
|
+
if threshold <= 0:
|
|
38
|
+
raise ValueError("threshold must be positive.")
|
|
39
|
+
self.threshold = threshold
|
|
40
|
+
self.register_buffer("lower_bound", torch.empty(0))
|
|
41
|
+
self.register_buffer("upper_bound", torch.empty(0))
|
|
42
|
+
|
|
43
|
+
def _fit(
|
|
44
|
+
self,
|
|
45
|
+
table: TableTensor,
|
|
46
|
+
*,
|
|
47
|
+
generator: torch.Generator | None = None,
|
|
48
|
+
) -> None:
|
|
49
|
+
|
|
50
|
+
numerical = table.numerical
|
|
51
|
+
assert not numerical.requires_grad
|
|
52
|
+
finite = _isfinite(numerical)
|
|
53
|
+
count_finite = finite.sum(dim=-2, keepdim=True)
|
|
54
|
+
finite_or_nan = numerical.masked_fill(~finite, torch.nan)
|
|
55
|
+
|
|
56
|
+
# Compute finite mean and standard deviation:
|
|
57
|
+
mean = finite_or_nan.nansum(-2, keepdim=True).div_(count_finite)
|
|
58
|
+
mean.masked_fill_(mean.isnan(), 0.0)
|
|
59
|
+
|
|
60
|
+
var = finite_or_nan.sub_(mean).square_().nansum(-2, keepdim=True)
|
|
61
|
+
del finite_or_nan
|
|
62
|
+
var /= (count_finite - 1).clamp_(min=1)
|
|
63
|
+
std = var.sqrt().clamp(min=1e-6)
|
|
64
|
+
|
|
65
|
+
# Find values within range (non-finite values are never kept):
|
|
66
|
+
lower = mean - self.threshold * std
|
|
67
|
+
upper = mean + self.threshold * std
|
|
68
|
+
keep = (numerical >= lower).logical_and_(numerical <= upper)
|
|
69
|
+
keep &= finite
|
|
70
|
+
del finite
|
|
71
|
+
count = keep.sum(dim=-2, keepdim=True)
|
|
72
|
+
|
|
73
|
+
# Compute mean and standard deviation of kept values:
|
|
74
|
+
kept_mean = torch.where(keep, numerical, 0.0).sum(-2, keepdim=True)
|
|
75
|
+
kept_mean /= count.clamp(min=1)
|
|
76
|
+
|
|
77
|
+
centered = numerical.sub(kept_mean).masked_fill_(~keep, 0.0)
|
|
78
|
+
denominator = (count - 1).clamp(min=1)
|
|
79
|
+
kept_var = centered.square_().sum(-2, keepdim=True) / denominator
|
|
80
|
+
del centered
|
|
81
|
+
kept_std = kept_var.sqrt().clamp(min=1e-6)
|
|
82
|
+
|
|
83
|
+
has_kept = count > 0
|
|
84
|
+
mean = torch.where(has_kept, kept_mean, mean)
|
|
85
|
+
std = torch.where(has_kept, kept_std, std)
|
|
86
|
+
|
|
87
|
+
self.lower_bound = mean - self.threshold * std
|
|
88
|
+
self.upper_bound = mean + self.threshold * std
|
|
89
|
+
|
|
90
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
91
|
+
numerical = table.numerical
|
|
92
|
+
dtype = torch.promote_types(numerical.dtype, self.lower_bound.dtype)
|
|
93
|
+
if torch.is_grad_enabled():
|
|
94
|
+
log_abs = numerical.abs().log1p().to(dtype)
|
|
95
|
+
clipped = torch.maximum(self.lower_bound - log_abs, numerical)
|
|
96
|
+
clipped = torch.minimum(log_abs + self.upper_bound, clipped)
|
|
97
|
+
return table.replace_blocks(numerical=clipped)
|
|
98
|
+
|
|
99
|
+
out = torch.empty_like(numerical, dtype=dtype)
|
|
100
|
+
# Chunks of rows bound the temporary besides the output.
|
|
101
|
+
size = split_size(
|
|
102
|
+
num_items=numerical.size(-2),
|
|
103
|
+
item_bytes=math.prod(out.size()[:-2])
|
|
104
|
+
* out.size(-1)
|
|
105
|
+
* out.element_size(),
|
|
106
|
+
device=out.device,
|
|
107
|
+
)
|
|
108
|
+
for inp, clipped in zip(
|
|
109
|
+
numerical.split(size, dim=-2),
|
|
110
|
+
out.split(size, dim=-2),
|
|
111
|
+
strict=True,
|
|
112
|
+
):
|
|
113
|
+
log_abs = inp.abs().log1p_().to(dtype)
|
|
114
|
+
torch.sub(self.lower_bound, log_abs, out=clipped)
|
|
115
|
+
torch.maximum(clipped, inp, out=clipped)
|
|
116
|
+
torch.minimum(log_abs.add_(self.upper_bound), clipped, out=clipped)
|
|
117
|
+
return table.replace_blocks(numerical=out)
|
|
118
|
+
|
|
119
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
120
|
+
return (
|
|
121
|
+
f"{' ' * indent}{self.__class__.__name__}("
|
|
122
|
+
f"threshold={self.threshold})"
|
|
123
|
+
)
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
|
|
6
|
+
from sdm import Stype, TableTensor
|
|
7
|
+
from sdm.processing import InvertibleMixin, Processor
|
|
8
|
+
from sdm.processing.numerical._stats import (
|
|
9
|
+
_constant_feature_mask,
|
|
10
|
+
_isfinite,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Standardize(Processor, InvertibleMixin):
|
|
15
|
+
"""Center and scale each feature column.
|
|
16
|
+
|
|
17
|
+
Constant columns use a unit scale to keep the transform finite and
|
|
18
|
+
invertible. NaN and infinite values are ignored when fitting statistics
|
|
19
|
+
and preserved during the transform.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
eps: Value added to each fitted standard deviation.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
26
|
+
requires_fit = True
|
|
27
|
+
|
|
28
|
+
def __init__(self, *, eps: float = 0.0) -> None:
|
|
29
|
+
super().__init__()
|
|
30
|
+
if eps < 0:
|
|
31
|
+
raise ValueError("epsilon must be non-negative.")
|
|
32
|
+
self.eps = eps
|
|
33
|
+
self.register_buffer("mean", torch.empty(0))
|
|
34
|
+
self.register_buffer("scale", torch.empty(0))
|
|
35
|
+
|
|
36
|
+
def _fit(
|
|
37
|
+
self,
|
|
38
|
+
table: TableTensor,
|
|
39
|
+
*,
|
|
40
|
+
generator: torch.Generator | None = None,
|
|
41
|
+
) -> None:
|
|
42
|
+
|
|
43
|
+
finite = _isfinite(table.numerical)
|
|
44
|
+
count = finite.sum(dim=-2, keepdim=True)
|
|
45
|
+
finite_or_nan = table.numerical.masked_fill(~finite, torch.nan)
|
|
46
|
+
|
|
47
|
+
self.mean = finite_or_nan.nansum(-2, keepdim=True).div_(count)
|
|
48
|
+
self.mean.masked_fill_(self.mean.isnan(), 0.0)
|
|
49
|
+
|
|
50
|
+
var = finite_or_nan.sub_(self.mean).square_().nansum(-2, keepdim=True)
|
|
51
|
+
var /= count
|
|
52
|
+
var.masked_fill_(var.isnan(), 0.0)
|
|
53
|
+
|
|
54
|
+
self.scale = var.sqrt()
|
|
55
|
+
if self.eps == 0:
|
|
56
|
+
mask = _constant_feature_mask(var, self.mean, num_samples=count)
|
|
57
|
+
self.scale[mask] = 1.0
|
|
58
|
+
else:
|
|
59
|
+
self.scale += self.eps
|
|
60
|
+
|
|
61
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
62
|
+
numerical = (table.numerical - self.mean).div_(self.scale)
|
|
63
|
+
return table.replace_blocks(numerical=numerical)
|
|
64
|
+
|
|
65
|
+
def _inverse_transform(self, table: TableTensor) -> TableTensor:
|
|
66
|
+
dtype = table.numerical.dtype
|
|
67
|
+
numerical = (table.numerical * self.scale).add_(self.mean)
|
|
68
|
+
return table.replace_blocks(numerical=numerical.to(dtype=dtype))
|
|
69
|
+
|
|
70
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
71
|
+
return f"{' ' * indent}{self.__class__.__name__}(eps={self.eps})"
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Output postprocessing transforms."""
|
|
5
|
+
|
|
6
|
+
from sdm.processing.output.reduce import AverageEstimators
|
|
7
|
+
from sdm.processing.output.softmax import Softmax
|
|
8
|
+
from sdm.processing.output.sort import SortQuantiles
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"AverageEstimators",
|
|
12
|
+
"Softmax",
|
|
13
|
+
"SortQuantiles",
|
|
14
|
+
]
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from typing import cast
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
from torch import Tensor
|
|
8
|
+
|
|
9
|
+
from sdm import EnsembleTable, Stype, TableTensor
|
|
10
|
+
from sdm.processing import EnsembleProcessor
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class AverageEstimators(EnsembleProcessor):
|
|
14
|
+
"""Reduce the leading ensemble dimension of model outputs.
|
|
15
|
+
|
|
16
|
+
Inputs must be a numerical output table with shape ``[E, ...]``, where
|
|
17
|
+
``E`` is the non-empty leading ensemble dimension.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
trim_fraction: Fraction of estimators discarded from each tail after
|
|
21
|
+
sorting each output coordinate, in ``[0, 0.5)``. This can make the
|
|
22
|
+
aggregation less sensitive to outlying estimator predictions.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
26
|
+
requires_fit = False
|
|
27
|
+
|
|
28
|
+
def __init__(
|
|
29
|
+
self,
|
|
30
|
+
*,
|
|
31
|
+
trim_fraction: float = 0.0,
|
|
32
|
+
) -> None:
|
|
33
|
+
super().__init__()
|
|
34
|
+
if not 0.0 <= trim_fraction < 0.5:
|
|
35
|
+
raise ValueError("'trim_fraction' must be in [0, 0.5)")
|
|
36
|
+
self.trim_fraction = trim_fraction
|
|
37
|
+
|
|
38
|
+
def _transform_ensemble(
|
|
39
|
+
self,
|
|
40
|
+
ensemble_table: EnsembleTable,
|
|
41
|
+
) -> EnsembleTable:
|
|
42
|
+
|
|
43
|
+
if len(ensemble_table) == 1:
|
|
44
|
+
return ensemble_table
|
|
45
|
+
|
|
46
|
+
groups: list[Tensor] = [
|
|
47
|
+
ensemble_table.expanded_group(group_id)
|
|
48
|
+
for group_id in range(ensemble_table.num_groups)
|
|
49
|
+
]
|
|
50
|
+
table = torch.cat(groups, dim=0) if len(groups) > 1 else groups[0]
|
|
51
|
+
table = cast(TableTensor, table)
|
|
52
|
+
|
|
53
|
+
out = table.numerical
|
|
54
|
+
cut = int(self.trim_fraction * table.size(0))
|
|
55
|
+
if cut > 0:
|
|
56
|
+
out = out.sort(dim=0).values
|
|
57
|
+
out = out[cut : out.size(0) - cut]
|
|
58
|
+
out = out.mean(dim=0)
|
|
59
|
+
|
|
60
|
+
return EnsembleTable.from_table(
|
|
61
|
+
table=TableTensor(
|
|
62
|
+
columns={Stype.numerical: table.columns[Stype.numerical]},
|
|
63
|
+
numerical=out,
|
|
64
|
+
),
|
|
65
|
+
num_members=1,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
69
|
+
if table.dim() < 3:
|
|
70
|
+
raise ValueError(
|
|
71
|
+
f"Expected a leading ensemble dimension (got {table.dim()}D)"
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
ensemble_table = EnsembleTable(
|
|
75
|
+
groups=(table,),
|
|
76
|
+
locations=tuple((0, i) for i in range(table.size(0))),
|
|
77
|
+
)
|
|
78
|
+
return self._transform_ensemble(ensemble_table)[0]
|
|
79
|
+
|
|
80
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
81
|
+
if self.trim_fraction == 0:
|
|
82
|
+
return super().__repr__(indent=indent)
|
|
83
|
+
return (
|
|
84
|
+
f"{' ' * indent}{self.__class__.__name__}("
|
|
85
|
+
f"trim_fraction={self.trim_fraction})"
|
|
86
|
+
)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
|
|
6
|
+
from sdm import Stype, TableTensor
|
|
7
|
+
from sdm.processing import Processor
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Softmax(Processor):
|
|
11
|
+
"""Apply softmax to logits after temperature scaling.
|
|
12
|
+
|
|
13
|
+
Softmax acts on the final class dimension and preserves all leading
|
|
14
|
+
dimensions, so it supports both stacked and reduced estimator outputs.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
temperature: Positive divisor applied to logits before softmax;
|
|
18
|
+
higher values produce a softer distribution.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
22
|
+
requires_fit = False
|
|
23
|
+
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
*,
|
|
27
|
+
temperature: float = 1.0,
|
|
28
|
+
) -> None:
|
|
29
|
+
super().__init__()
|
|
30
|
+
if temperature <= 0:
|
|
31
|
+
raise ValueError("temperature must be positive.")
|
|
32
|
+
self.temperature = temperature
|
|
33
|
+
|
|
34
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
35
|
+
"""Return ``softmax(table / temperature)`` over the last dimension."""
|
|
36
|
+
numerical = torch.softmax(
|
|
37
|
+
table.numerical / self.temperature,
|
|
38
|
+
dim=-1,
|
|
39
|
+
)
|
|
40
|
+
return table.replace_blocks(numerical=numerical)
|
|
41
|
+
|
|
42
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
43
|
+
if self.temperature == 1.0:
|
|
44
|
+
return super().__repr__(indent=indent)
|
|
45
|
+
return (
|
|
46
|
+
f"{' ' * indent}{self.__class__.__name__}("
|
|
47
|
+
f"temperature={self.temperature})"
|
|
48
|
+
)
|