structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/nn/softplus.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Portions derived from TabFM:
|
|
2
|
+
# Copyright 2026 Google LLC
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
17
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
18
|
+
|
|
19
|
+
import torch
|
|
20
|
+
import torch.nn.functional as F
|
|
21
|
+
from torch import Tensor
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class SoftplusScale(torch.nn.Module):
|
|
25
|
+
r"""Apply a learned positive scale to the final input dimension.
|
|
26
|
+
|
|
27
|
+
Each channel is multiplied by a positive factor parametrized as
|
|
28
|
+
``multiplier * softplus(weight)``.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
channels: The number of input and output channels.
|
|
32
|
+
multiplier: The fixed multiplier applied to every learned factor.
|
|
33
|
+
weight_init: The initial values of ``weight``.
|
|
34
|
+
device: The device.
|
|
35
|
+
dtype: The parameter dtype.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(
|
|
39
|
+
self,
|
|
40
|
+
channels: int,
|
|
41
|
+
multiplier: float = 1.0,
|
|
42
|
+
weight_init: float = 0.0,
|
|
43
|
+
device: torch.device | str | None = None,
|
|
44
|
+
dtype: torch.dtype | None = None,
|
|
45
|
+
) -> None:
|
|
46
|
+
super().__init__()
|
|
47
|
+
|
|
48
|
+
self.multiplier = multiplier
|
|
49
|
+
self.weight = torch.nn.Parameter(
|
|
50
|
+
data=torch.full(
|
|
51
|
+
size=(channels,),
|
|
52
|
+
fill_value=weight_init,
|
|
53
|
+
device=device,
|
|
54
|
+
dtype=dtype,
|
|
55
|
+
)
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
def forward(self, tensor: Tensor) -> Tensor:
|
|
59
|
+
r"""The forward pass.
|
|
60
|
+
|
|
61
|
+
Args:
|
|
62
|
+
tensor: The input tensor.
|
|
63
|
+
"""
|
|
64
|
+
scale = self.multiplier * F.softplus(self.weight.float())
|
|
65
|
+
return tensor * scale.to(tensor.dtype)
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Processors for structured data tables."""
|
|
5
|
+
|
|
6
|
+
from sdm.processing.base import Processor, InvertibleMixin
|
|
7
|
+
from sdm.processing.ensemble import (
|
|
8
|
+
EnsembleProcessor,
|
|
9
|
+
EnsembleInvertibleMixin,
|
|
10
|
+
)
|
|
11
|
+
from sdm.processing.common import (
|
|
12
|
+
Identity,
|
|
13
|
+
Callable,
|
|
14
|
+
Sequential,
|
|
15
|
+
DropStypes,
|
|
16
|
+
StypeDispatch,
|
|
17
|
+
TaskDispatch,
|
|
18
|
+
TableDispatch,
|
|
19
|
+
EnsembleProcessorAdapter,
|
|
20
|
+
Choice,
|
|
21
|
+
ToNumerical,
|
|
22
|
+
ShuffleColumns,
|
|
23
|
+
SelectColumns,
|
|
24
|
+
)
|
|
25
|
+
from sdm.processing.text import TFIDF, SentenceTransformer
|
|
26
|
+
from sdm.processing.numerical import (
|
|
27
|
+
Cast,
|
|
28
|
+
Clip,
|
|
29
|
+
ClipQuantiles,
|
|
30
|
+
ClipSigma,
|
|
31
|
+
ClipSoft,
|
|
32
|
+
ImputeMean,
|
|
33
|
+
PowerTransform,
|
|
34
|
+
QuantileTransform,
|
|
35
|
+
RankGaussian,
|
|
36
|
+
Standardize,
|
|
37
|
+
RobustScale,
|
|
38
|
+
FlipSign,
|
|
39
|
+
DropConstantColumns,
|
|
40
|
+
PCA,
|
|
41
|
+
RandomProjection,
|
|
42
|
+
)
|
|
43
|
+
from sdm.processing.categorical import (
|
|
44
|
+
AlignCategories,
|
|
45
|
+
ShuffleCategories,
|
|
46
|
+
ImputeMode,
|
|
47
|
+
AddCategoryCounts,
|
|
48
|
+
)
|
|
49
|
+
from sdm.processing.datetime import AddCalendarFields
|
|
50
|
+
from sdm.processing.output import AverageEstimators, Softmax, SortQuantiles
|
|
51
|
+
from sdm.processing.recipe import Recipe
|
|
52
|
+
|
|
53
|
+
__all__ = [
|
|
54
|
+
"Processor",
|
|
55
|
+
"InvertibleMixin",
|
|
56
|
+
"EnsembleInvertibleMixin",
|
|
57
|
+
"EnsembleProcessorAdapter",
|
|
58
|
+
"Identity",
|
|
59
|
+
"Callable",
|
|
60
|
+
"Sequential",
|
|
61
|
+
"DropStypes",
|
|
62
|
+
"StypeDispatch",
|
|
63
|
+
"TaskDispatch",
|
|
64
|
+
"TableDispatch",
|
|
65
|
+
"EnsembleProcessor",
|
|
66
|
+
"Choice",
|
|
67
|
+
"ToNumerical",
|
|
68
|
+
"ShuffleColumns",
|
|
69
|
+
"SelectColumns",
|
|
70
|
+
"TFIDF",
|
|
71
|
+
"SentenceTransformer",
|
|
72
|
+
"Cast",
|
|
73
|
+
"Clip",
|
|
74
|
+
"ClipQuantiles",
|
|
75
|
+
"ClipSigma",
|
|
76
|
+
"ClipSoft",
|
|
77
|
+
"ImputeMean",
|
|
78
|
+
"PowerTransform",
|
|
79
|
+
"QuantileTransform",
|
|
80
|
+
"RankGaussian",
|
|
81
|
+
"Standardize",
|
|
82
|
+
"RobustScale",
|
|
83
|
+
"FlipSign",
|
|
84
|
+
"DropConstantColumns",
|
|
85
|
+
"PCA",
|
|
86
|
+
"RandomProjection",
|
|
87
|
+
"AlignCategories",
|
|
88
|
+
"ShuffleCategories",
|
|
89
|
+
"ImputeMode",
|
|
90
|
+
"AddCategoryCounts",
|
|
91
|
+
"AddCalendarFields",
|
|
92
|
+
"AverageEstimators",
|
|
93
|
+
"Softmax",
|
|
94
|
+
"SortQuantiles",
|
|
95
|
+
"Recipe",
|
|
96
|
+
]
|
sdm/processing/base.py
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import abc
|
|
7
|
+
from collections.abc import Sequence
|
|
8
|
+
from typing import TYPE_CHECKING, Any, Self
|
|
9
|
+
|
|
10
|
+
import torch
|
|
11
|
+
from torch import Tensor
|
|
12
|
+
|
|
13
|
+
from sdm import Stype, TableTensor
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING:
|
|
16
|
+
from sdm.processing import Sequential
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Processor(torch.nn.Module, abc.ABC):
|
|
20
|
+
r"""Base processor for tensor-aware table transformations.
|
|
21
|
+
|
|
22
|
+
A :class:`Processor` defines a reusable transformation on
|
|
23
|
+
:class:`~sdm.tensor.TableTensor` for feature, target and output
|
|
24
|
+
preprocessing.
|
|
25
|
+
A :class:`Processor` learns any required state via :meth:`fit`, and applies
|
|
26
|
+
the transformation via :meth:`transform`. Implementations preserve batch
|
|
27
|
+
dimensions, and preserve rows unless documented otherwise. Batch
|
|
28
|
+
dimensions are processed independently.
|
|
29
|
+
|
|
30
|
+
:meth:`fit`, :meth:`transform`, and :meth:`fit_transform` are no-ops for
|
|
31
|
+
stypes outside of :attr:`handles_stypes`.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
#: Semantic types this processor operates on.
|
|
35
|
+
handles_stypes: frozenset[Stype]
|
|
36
|
+
|
|
37
|
+
#: Whether this processor requires fitting.
|
|
38
|
+
requires_fit: bool
|
|
39
|
+
|
|
40
|
+
_fitted_state: Tensor
|
|
41
|
+
|
|
42
|
+
def __init__(self) -> None:
|
|
43
|
+
super().__init__()
|
|
44
|
+
self.register_buffer("_fitted_state", torch.tensor(False))
|
|
45
|
+
self.__fitted = False
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def _fitted(self) -> bool:
|
|
49
|
+
return self.__fitted
|
|
50
|
+
|
|
51
|
+
@_fitted.setter
|
|
52
|
+
def _fitted(self, value: bool) -> None:
|
|
53
|
+
self.__fitted = value
|
|
54
|
+
self._fitted_state.fill_(value)
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def is_fitted(self) -> bool:
|
|
58
|
+
"""Whether the processor has all state required for transformation."""
|
|
59
|
+
return not self.requires_fit or self._fitted
|
|
60
|
+
|
|
61
|
+
@staticmethod
|
|
62
|
+
def as_processor(processor: object) -> Processor:
|
|
63
|
+
r"""Normalize a processor-like object to a :class:`Processor`.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
processor: A processor-like object. A :class:`Processor` is
|
|
67
|
+
returned as-is, a callable is wrapped as a stateless processor,
|
|
68
|
+
and a sequence of processor-like objects is normalized to
|
|
69
|
+
:class:`~sdm.processing.common.Sequential`.
|
|
70
|
+
"""
|
|
71
|
+
from sdm.processing import Callable, Sequential # noqa: PLC0415
|
|
72
|
+
|
|
73
|
+
if isinstance(processor, Processor):
|
|
74
|
+
return processor
|
|
75
|
+
if callable(processor):
|
|
76
|
+
return Callable(processor) # type: ignore
|
|
77
|
+
if isinstance(processor, Sequence) and not isinstance(processor, str):
|
|
78
|
+
return Sequential(*processor)
|
|
79
|
+
raise TypeError(
|
|
80
|
+
f"Input must be a 'Processor', callable, or sequence of them "
|
|
81
|
+
f"(got '{type(processor).__name__}')"
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
def _load_from_state_dict(
|
|
85
|
+
self,
|
|
86
|
+
state_dict: dict[str, Any],
|
|
87
|
+
prefix: str,
|
|
88
|
+
local_metadata: dict[str, Any],
|
|
89
|
+
strict: bool,
|
|
90
|
+
missing_keys: list[str],
|
|
91
|
+
unexpected_keys: list[str],
|
|
92
|
+
error_msgs: list[str],
|
|
93
|
+
) -> None:
|
|
94
|
+
# Resize uninitialized buffers before PyTorch copies saved values.
|
|
95
|
+
for name, buffer in self._buffers.items():
|
|
96
|
+
state = state_dict.get(f"{prefix}{name}")
|
|
97
|
+
if (
|
|
98
|
+
buffer is not None
|
|
99
|
+
and isinstance(state, Tensor)
|
|
100
|
+
and (
|
|
101
|
+
buffer.shape != state.shape or buffer.dtype != state.dtype
|
|
102
|
+
)
|
|
103
|
+
):
|
|
104
|
+
self._buffers[name] = buffer.new_empty(
|
|
105
|
+
state.shape,
|
|
106
|
+
dtype=state.dtype,
|
|
107
|
+
)
|
|
108
|
+
super()._load_from_state_dict(
|
|
109
|
+
state_dict,
|
|
110
|
+
prefix,
|
|
111
|
+
local_metadata,
|
|
112
|
+
strict,
|
|
113
|
+
missing_keys,
|
|
114
|
+
unexpected_keys,
|
|
115
|
+
error_msgs,
|
|
116
|
+
)
|
|
117
|
+
self.__fitted = bool(self._fitted_state)
|
|
118
|
+
|
|
119
|
+
def _set_fitted(self, device: torch.device) -> None:
|
|
120
|
+
self._fitted_state = self._fitted_state.to(device=device)
|
|
121
|
+
self._fitted = True
|
|
122
|
+
|
|
123
|
+
def _check_is_fitted(self) -> None:
|
|
124
|
+
if not self.is_fitted:
|
|
125
|
+
raise RuntimeError(
|
|
126
|
+
f"{self.__class__.__name__!r} is not fitted; "
|
|
127
|
+
"call 'fit()' before."
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
def _fit(
|
|
131
|
+
self,
|
|
132
|
+
table: TableTensor,
|
|
133
|
+
*,
|
|
134
|
+
generator: torch.Generator | None = None,
|
|
135
|
+
) -> None:
|
|
136
|
+
raise NotImplementedError
|
|
137
|
+
|
|
138
|
+
@abc.abstractmethod
|
|
139
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
140
|
+
pass
|
|
141
|
+
|
|
142
|
+
def _fit_transform(
|
|
143
|
+
self,
|
|
144
|
+
table: TableTensor,
|
|
145
|
+
*,
|
|
146
|
+
generator: torch.Generator | None = None,
|
|
147
|
+
) -> TableTensor:
|
|
148
|
+
if self.requires_fit:
|
|
149
|
+
self._fit(table, generator=generator)
|
|
150
|
+
return self._transform(table)
|
|
151
|
+
|
|
152
|
+
def fit(
|
|
153
|
+
self,
|
|
154
|
+
table: TableTensor,
|
|
155
|
+
*,
|
|
156
|
+
generator: torch.Generator | None = None,
|
|
157
|
+
) -> Self:
|
|
158
|
+
r"""Fit the processor.
|
|
159
|
+
|
|
160
|
+
Args:
|
|
161
|
+
table: The table used to compute the processor state.
|
|
162
|
+
generator: Pseudorandom number generator used for sampling.
|
|
163
|
+
"""
|
|
164
|
+
if not table.active_stypes & self.handles_stypes:
|
|
165
|
+
return self
|
|
166
|
+
if self.requires_fit:
|
|
167
|
+
self._fit(table, generator=generator)
|
|
168
|
+
self._set_fitted(table.device)
|
|
169
|
+
return self
|
|
170
|
+
|
|
171
|
+
def transform(self, table: TableTensor) -> TableTensor:
|
|
172
|
+
r"""Transform ``table``.
|
|
173
|
+
|
|
174
|
+
Args:
|
|
175
|
+
table: The table to transform.
|
|
176
|
+
|
|
177
|
+
Returns:
|
|
178
|
+
The transformed table.
|
|
179
|
+
"""
|
|
180
|
+
if not table.active_stypes & self.handles_stypes:
|
|
181
|
+
return table
|
|
182
|
+
self._check_is_fitted()
|
|
183
|
+
return self._transform(table)
|
|
184
|
+
|
|
185
|
+
def forward(self, table: TableTensor) -> TableTensor:
|
|
186
|
+
r"""Alias of :meth:`transform`."""
|
|
187
|
+
return self.transform(table)
|
|
188
|
+
|
|
189
|
+
def fit_transform(
|
|
190
|
+
self,
|
|
191
|
+
table: TableTensor,
|
|
192
|
+
*,
|
|
193
|
+
generator: torch.Generator | None = None,
|
|
194
|
+
) -> TableTensor:
|
|
195
|
+
r"""Fit the processor and transform ``table``.
|
|
196
|
+
|
|
197
|
+
Args:
|
|
198
|
+
table: The table to fit on and transform.
|
|
199
|
+
generator: Pseudorandom number generator used for sampling.
|
|
200
|
+
|
|
201
|
+
Returns:
|
|
202
|
+
The transformed table.
|
|
203
|
+
"""
|
|
204
|
+
if not table.active_stypes & self.handles_stypes:
|
|
205
|
+
return table
|
|
206
|
+
out = self._fit_transform(table, generator=generator)
|
|
207
|
+
if self.requires_fit:
|
|
208
|
+
self._set_fitted(table.device)
|
|
209
|
+
return out
|
|
210
|
+
|
|
211
|
+
def __add__(self, other: object) -> Sequential:
|
|
212
|
+
from sdm.processing import Sequential # noqa: PLC0415
|
|
213
|
+
|
|
214
|
+
try:
|
|
215
|
+
other = Processor.as_processor(other)
|
|
216
|
+
except TypeError:
|
|
217
|
+
return NotImplemented
|
|
218
|
+
return Sequential(self, other)
|
|
219
|
+
|
|
220
|
+
def __radd__(self, other: object) -> Sequential:
|
|
221
|
+
from sdm.processing import Sequential # noqa: PLC0415
|
|
222
|
+
|
|
223
|
+
try:
|
|
224
|
+
other = Processor.as_processor(other)
|
|
225
|
+
except TypeError:
|
|
226
|
+
return NotImplemented
|
|
227
|
+
return Sequential(other, self)
|
|
228
|
+
|
|
229
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
230
|
+
return f"{' ' * indent}{self.__class__.__name__}()"
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
class InvertibleMixin(abc.ABC):
|
|
234
|
+
r"""Extend a :class:`Processor` by an inverse transformation."""
|
|
235
|
+
|
|
236
|
+
@abc.abstractmethod
|
|
237
|
+
def _inverse_transform(self, table: TableTensor) -> TableTensor: ...
|
|
238
|
+
|
|
239
|
+
def inverse_transform(self, table: TableTensor) -> TableTensor:
|
|
240
|
+
r"""Apply the inverse transformation to ``table``.
|
|
241
|
+
|
|
242
|
+
Args:
|
|
243
|
+
table: The table in transformed representation.
|
|
244
|
+
|
|
245
|
+
Returns:
|
|
246
|
+
The table restored to the representation before
|
|
247
|
+
:meth:`~Processor.transform`.
|
|
248
|
+
"""
|
|
249
|
+
self._check_is_fitted()
|
|
250
|
+
return self._inverse_transform(table)
|
|
251
|
+
|
|
252
|
+
if TYPE_CHECKING:
|
|
253
|
+
# Provided at runtime by `Processor` via the MRO.
|
|
254
|
+
def _check_is_fitted(self) -> None: ...
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Categorical preprocessing transforms."""
|
|
5
|
+
|
|
6
|
+
from sdm.processing.categorical.align import AlignCategories
|
|
7
|
+
from sdm.processing.categorical.shuffle import ShuffleCategories
|
|
8
|
+
from sdm.processing.categorical.impute import ImputeMode
|
|
9
|
+
from sdm.processing.categorical.count import AddCategoryCounts
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"AlignCategories",
|
|
13
|
+
"ShuffleCategories",
|
|
14
|
+
"ImputeMode",
|
|
15
|
+
"AddCategoryCounts",
|
|
16
|
+
]
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
|
|
6
|
+
from torch import Tensor
|
|
7
|
+
|
|
8
|
+
from sdm import Stype, TableTensor
|
|
9
|
+
from sdm.nn._buffer import BufferList
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _check_categorical_codes(table: TableTensor) -> None:
|
|
13
|
+
"""Raise if a categorical code exceeds its column's category vocabulary.
|
|
14
|
+
|
|
15
|
+
Negative codes encode missing values and are allowed.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
table: The table whose categorical codes are validated.
|
|
19
|
+
"""
|
|
20
|
+
if os.getenv("SDM_ENABLE_CATEGORY_CHECKS", "1") == "0":
|
|
21
|
+
return
|
|
22
|
+
columns = table.columns[Stype.categorical]
|
|
23
|
+
for index, category in enumerate(table.categorical.categories):
|
|
24
|
+
codes = table.categorical[..., index]
|
|
25
|
+
if (codes >= category.numel()).any():
|
|
26
|
+
raise ValueError(
|
|
27
|
+
f"Categorical column {columns[index]!r} contains a code "
|
|
28
|
+
"outside its category vocabulary."
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _check_categories(
|
|
33
|
+
table: TableTensor,
|
|
34
|
+
categories: BufferList[Tensor],
|
|
35
|
+
) -> None:
|
|
36
|
+
"""Raise if a category vocabulary differs from the fitted one.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
table: The table whose category vocabularies are validated.
|
|
40
|
+
categories: The fitted category vocabulary of every column.
|
|
41
|
+
"""
|
|
42
|
+
if os.getenv("SDM_ENABLE_CATEGORY_CHECKS", "1") == "0":
|
|
43
|
+
return
|
|
44
|
+
columns = table.columns[Stype.categorical]
|
|
45
|
+
if len(table.categorical.categories) != len(categories):
|
|
46
|
+
raise ValueError(
|
|
47
|
+
f"Expected {len(categories)} fitted categorical "
|
|
48
|
+
f"columns (got {len(columns)})."
|
|
49
|
+
)
|
|
50
|
+
for index, (actual, expected) in enumerate(
|
|
51
|
+
zip(table.categorical.categories, categories)
|
|
52
|
+
):
|
|
53
|
+
expected = expected.to(device=actual.device)
|
|
54
|
+
if not actual.equal(expected):
|
|
55
|
+
raise ValueError(
|
|
56
|
+
"Expected the category vocabulary for categorical column "
|
|
57
|
+
f"{columns[index]!r} to match the fitted values and order. "
|
|
58
|
+
"Use 'AlignCategories' before this processor for "
|
|
59
|
+
"independently tensorized inputs."
|
|
60
|
+
)
|