structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,540 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from collections.abc import Sequence
|
|
5
|
+
from typing import Literal, cast
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
from torch import Tensor
|
|
9
|
+
|
|
10
|
+
from sdm import (
|
|
11
|
+
CategoricalTensor,
|
|
12
|
+
ColumnarTensor,
|
|
13
|
+
EnsembleTable,
|
|
14
|
+
StringTensor,
|
|
15
|
+
Stype,
|
|
16
|
+
TableTensor,
|
|
17
|
+
)
|
|
18
|
+
from sdm.nn._buffer import BufferList
|
|
19
|
+
from sdm.processing import EnsembleProcessor
|
|
20
|
+
from sdm.relational.join import join_index
|
|
21
|
+
|
|
22
|
+
_UNSIGNED_DTYPES = frozenset({torch.uint16, torch.uint32, torch.uint64})
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class AlignCategories(EnsembleProcessor):
|
|
26
|
+
"""Align categorical columns to categories observed during fitting.
|
|
27
|
+
|
|
28
|
+
A categorical column stores each value as an integer code indexing an
|
|
29
|
+
ordered list of categories. Separately created tables can use different
|
|
30
|
+
codes for the same value. Fitting learns the category list for each column,
|
|
31
|
+
and transforming remaps another table to use it. Missing values and
|
|
32
|
+
categories not retained during fitting receive code ``-1``.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
sort_by: How to order fitted categories.
|
|
36
|
+
``"code"`` keeps observed categories in original order.
|
|
37
|
+
``"frequency"`` orders observed categories by descending frequency.
|
|
38
|
+
``"value"`` orders observed categories by ascending value.
|
|
39
|
+
min_frequency: Minimum number of observations required to retain a
|
|
40
|
+
category. Values of rarer categories receive code ``-1``.
|
|
41
|
+
Must be positive.
|
|
42
|
+
|
|
43
|
+
>>> import pandas as pd
|
|
44
|
+
>>> import sdm
|
|
45
|
+
>>> from sdm.processing import AlignCategories
|
|
46
|
+
>>> table1 = sdm.TableTensor.from_pandas(
|
|
47
|
+
... pd.DataFrame({"color": ["red", "blue", "red"]}),
|
|
48
|
+
... stypes={"color": "categorical"},
|
|
49
|
+
... )
|
|
50
|
+
>>> table2 = sdm.TableTensor.from_pandas(
|
|
51
|
+
... pd.DataFrame({"color": ["blue", "green", None]}),
|
|
52
|
+
... stypes={"color": "categorical"},
|
|
53
|
+
... )
|
|
54
|
+
>>> table1.categorical.categories[0].tolist()
|
|
55
|
+
['red', 'blue']
|
|
56
|
+
>>> table2.categorical.categories[0].tolist()
|
|
57
|
+
['blue', 'green']
|
|
58
|
+
>>> table2.categorical.code[:, 0].tolist()
|
|
59
|
+
[0, 1, -1]
|
|
60
|
+
>>> processor = AlignCategories().fit(table1)
|
|
61
|
+
>>> table2 = processor.transform(table2)
|
|
62
|
+
>>> table2.categorical.categories[0].tolist()
|
|
63
|
+
['red', 'blue']
|
|
64
|
+
>>> table2.categorical.code[:, 0].tolist()
|
|
65
|
+
[1, -1, -1]
|
|
66
|
+
|
|
67
|
+
Here, ``"blue"`` changes from code ``0`` to ``1``, while unseen
|
|
68
|
+
``"green"`` and the missing value use ``-1``.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
handles_stypes = frozenset({Stype.categorical})
|
|
72
|
+
requires_fit = True
|
|
73
|
+
|
|
74
|
+
def __init__(
|
|
75
|
+
self,
|
|
76
|
+
sort_by: Literal["code", "frequency", "value"] = "code",
|
|
77
|
+
*,
|
|
78
|
+
min_frequency: int = 1,
|
|
79
|
+
) -> None:
|
|
80
|
+
super().__init__()
|
|
81
|
+
if min_frequency <= 0:
|
|
82
|
+
raise ValueError("min_frequency must be positive")
|
|
83
|
+
self.sort_by = sort_by
|
|
84
|
+
self.min_frequency = min_frequency
|
|
85
|
+
self._categories: BufferList[BufferList[Tensor]] = BufferList()
|
|
86
|
+
|
|
87
|
+
def _fit_column(
|
|
88
|
+
self,
|
|
89
|
+
input_categories: Tensor,
|
|
90
|
+
codes: Tensor,
|
|
91
|
+
*,
|
|
92
|
+
align_codes: bool,
|
|
93
|
+
) -> tuple[tuple[Tensor, ...], Tensor | None]:
|
|
94
|
+
batch_size = codes.size(0)
|
|
95
|
+
if input_categories.numel() == 0:
|
|
96
|
+
aligned_codes = torch.full_like(codes, -1) if align_codes else None
|
|
97
|
+
return (input_categories,) * batch_size, aligned_codes
|
|
98
|
+
|
|
99
|
+
mask = codes >= 0
|
|
100
|
+
indices = codes.clamp_min(0).long()
|
|
101
|
+
# Count categories independently per batch: [B, N] -> [B, K].
|
|
102
|
+
counts = codes.new_zeros((batch_size, input_categories.numel()))
|
|
103
|
+
counts.scatter_add_(1, indices, mask.to(codes.dtype))
|
|
104
|
+
|
|
105
|
+
if self.sort_by == "frequency":
|
|
106
|
+
order = counts.argsort(dim=1, descending=True, stable=True)
|
|
107
|
+
ordered_categories = input_categories
|
|
108
|
+
elif self.sort_by == "value":
|
|
109
|
+
if (
|
|
110
|
+
input_categories.is_cuda
|
|
111
|
+
and input_categories.dtype in _UNSIGNED_DTYPES
|
|
112
|
+
):
|
|
113
|
+
key = input_categories.to(torch.int64)
|
|
114
|
+
if input_categories.dtype == torch.uint64:
|
|
115
|
+
# Map unsigned integer order onto signed integer order.
|
|
116
|
+
key = key.bitwise_xor(torch.iinfo(torch.int64).min)
|
|
117
|
+
perm = key.argsort()
|
|
118
|
+
ordered_categories = input_categories.index_select(0, perm)
|
|
119
|
+
else:
|
|
120
|
+
ordered_categories, perm = input_categories.sort()
|
|
121
|
+
order = perm.expand(batch_size, -1)
|
|
122
|
+
else:
|
|
123
|
+
assert self.sort_by == "code"
|
|
124
|
+
order = torch.arange(
|
|
125
|
+
input_categories.numel(),
|
|
126
|
+
device=codes.device,
|
|
127
|
+
).expand(batch_size, -1)
|
|
128
|
+
ordered_categories = input_categories
|
|
129
|
+
|
|
130
|
+
observed = counts.gather(1, order) >= self.min_frequency
|
|
131
|
+
# Only ragged vocabularies require per-batch materialization.
|
|
132
|
+
fitted_categories = []
|
|
133
|
+
for batch_index in range(batch_size):
|
|
134
|
+
selected_indices = order[batch_index, observed[batch_index]]
|
|
135
|
+
if (
|
|
136
|
+
selected_indices.numel() == input_categories.numel()
|
|
137
|
+
and self.sort_by != "frequency"
|
|
138
|
+
):
|
|
139
|
+
fitted_categories.append(ordered_categories)
|
|
140
|
+
elif (
|
|
141
|
+
input_categories.dtype in _UNSIGNED_DTYPES
|
|
142
|
+
and input_categories.is_cpu
|
|
143
|
+
):
|
|
144
|
+
# PyTorch CPU index_select is not implemented for these dtypes.
|
|
145
|
+
fitted_categories.append(input_categories[selected_indices])
|
|
146
|
+
else:
|
|
147
|
+
fitted_categories.append(
|
|
148
|
+
input_categories.index_select(0, selected_indices)
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
if not align_codes:
|
|
152
|
+
return tuple(fitted_categories), None
|
|
153
|
+
|
|
154
|
+
rank = observed.cumsum(dim=1, dtype=codes.dtype) - 1
|
|
155
|
+
lookup = torch.full_like(counts, -1)
|
|
156
|
+
lookup.scatter_(1, order, torch.where(observed, rank, -1))
|
|
157
|
+
aligned_codes = torch.where(mask, lookup.gather(1, indices), -1)
|
|
158
|
+
return tuple(fitted_categories), aligned_codes
|
|
159
|
+
|
|
160
|
+
def _fit_columns(
|
|
161
|
+
self,
|
|
162
|
+
table: TableTensor,
|
|
163
|
+
*,
|
|
164
|
+
align_codes: bool,
|
|
165
|
+
) -> tuple[tuple[tuple[Tensor, ...], ...], Tensor | None]:
|
|
166
|
+
codes = table.categorical.code
|
|
167
|
+
if codes.dim() == 2:
|
|
168
|
+
codes = codes.unsqueeze(0)
|
|
169
|
+
aligned_codes = torch.full_like(codes, -1) if align_codes else None
|
|
170
|
+
|
|
171
|
+
# Accumulate ragged vocabularies in batch-major order: [batch][column].
|
|
172
|
+
categories_by_batch: list[list[Tensor]] = [
|
|
173
|
+
[] for _ in range(codes.size(0))
|
|
174
|
+
]
|
|
175
|
+
for column_index, input_categories in enumerate(
|
|
176
|
+
table.categorical.categories
|
|
177
|
+
):
|
|
178
|
+
fitted_categories, aligned_column_codes = self._fit_column(
|
|
179
|
+
input_categories=input_categories,
|
|
180
|
+
codes=codes[..., column_index],
|
|
181
|
+
align_codes=align_codes,
|
|
182
|
+
)
|
|
183
|
+
if aligned_codes is not None:
|
|
184
|
+
assert aligned_column_codes is not None
|
|
185
|
+
aligned_codes[..., column_index] = aligned_column_codes
|
|
186
|
+
for batch_categories, column_categories in zip(
|
|
187
|
+
categories_by_batch,
|
|
188
|
+
fitted_categories,
|
|
189
|
+
strict=True,
|
|
190
|
+
):
|
|
191
|
+
batch_categories.append(column_categories)
|
|
192
|
+
|
|
193
|
+
fitted_categories = tuple(
|
|
194
|
+
tuple(batch_categories) for batch_categories in categories_by_batch
|
|
195
|
+
)
|
|
196
|
+
return fitted_categories, aligned_codes
|
|
197
|
+
|
|
198
|
+
def _fit_and_align(
|
|
199
|
+
self,
|
|
200
|
+
table: TableTensor,
|
|
201
|
+
) -> tuple[tuple[tuple[Tensor, ...], ...], tuple[TableTensor, ...]]:
|
|
202
|
+
single_table = table.categorical.code.dim() == 2
|
|
203
|
+
fitted_categories, aligned_codes = self._fit_columns(
|
|
204
|
+
table,
|
|
205
|
+
align_codes=True,
|
|
206
|
+
)
|
|
207
|
+
assert aligned_codes is not None
|
|
208
|
+
aligned_tables = tuple(
|
|
209
|
+
(table if single_table else table[batch_index]).replace_blocks(
|
|
210
|
+
categorical=CategoricalTensor(
|
|
211
|
+
aligned_codes[batch_index],
|
|
212
|
+
categories=batch_categories,
|
|
213
|
+
),
|
|
214
|
+
)
|
|
215
|
+
for batch_index, batch_categories in enumerate(fitted_categories)
|
|
216
|
+
)
|
|
217
|
+
return fitted_categories, aligned_tables
|
|
218
|
+
|
|
219
|
+
def _fit(
|
|
220
|
+
self,
|
|
221
|
+
table: TableTensor,
|
|
222
|
+
*,
|
|
223
|
+
generator: torch.Generator | None = None,
|
|
224
|
+
) -> None:
|
|
225
|
+
fitted_categories, _ = self._fit_columns(
|
|
226
|
+
table,
|
|
227
|
+
align_codes=False,
|
|
228
|
+
)
|
|
229
|
+
self._categories = BufferList(
|
|
230
|
+
BufferList(categories) for categories in fitted_categories
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
def _fit_transform(
|
|
234
|
+
self,
|
|
235
|
+
table: TableTensor,
|
|
236
|
+
*,
|
|
237
|
+
generator: torch.Generator | None = None,
|
|
238
|
+
) -> TableTensor:
|
|
239
|
+
fitted_categories, aligned_tables = self._fit_and_align(table)
|
|
240
|
+
self._categories = BufferList(
|
|
241
|
+
BufferList(categories) for categories in fitted_categories
|
|
242
|
+
)
|
|
243
|
+
return aligned_tables[0]
|
|
244
|
+
|
|
245
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
246
|
+
return self._align_to_categories(
|
|
247
|
+
table,
|
|
248
|
+
cast(Sequence[Sequence[Tensor]], self._categories),
|
|
249
|
+
)[0]
|
|
250
|
+
|
|
251
|
+
@staticmethod
|
|
252
|
+
def _member_table_ids(
|
|
253
|
+
ensemble_table: EnsembleTable,
|
|
254
|
+
) -> tuple[int, ...]:
|
|
255
|
+
group_offsets = []
|
|
256
|
+
offset = 0
|
|
257
|
+
for group in ensemble_table._iter_groups():
|
|
258
|
+
group_offsets.append(offset)
|
|
259
|
+
offset += group.size(0)
|
|
260
|
+
|
|
261
|
+
# TODO: Replace this private access with a public EnsembleTable
|
|
262
|
+
# operation returning flattened tables and member table IDs.
|
|
263
|
+
return tuple(
|
|
264
|
+
group_offsets[group_index] + position
|
|
265
|
+
for group_index, position in ensemble_table._locations
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
def _fit_ensemble(
|
|
269
|
+
self,
|
|
270
|
+
ensemble_table: EnsembleTable,
|
|
271
|
+
*,
|
|
272
|
+
generator: torch.Generator | None = None,
|
|
273
|
+
) -> None:
|
|
274
|
+
fitted_categories = []
|
|
275
|
+
for group in ensemble_table._iter_groups():
|
|
276
|
+
group_categories, _ = self._fit_columns(
|
|
277
|
+
group,
|
|
278
|
+
align_codes=False,
|
|
279
|
+
)
|
|
280
|
+
fitted_categories.extend(group_categories)
|
|
281
|
+
self._categories = BufferList(
|
|
282
|
+
BufferList(categories) for categories in fitted_categories
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
def _fit_transform_ensemble(
|
|
286
|
+
self,
|
|
287
|
+
ensemble_table: EnsembleTable,
|
|
288
|
+
*,
|
|
289
|
+
generator: torch.Generator | None = None,
|
|
290
|
+
) -> EnsembleTable:
|
|
291
|
+
fitted_categories = []
|
|
292
|
+
aligned_tables = []
|
|
293
|
+
for group in ensemble_table._iter_groups():
|
|
294
|
+
group_categories, group_tables = self._fit_and_align(group)
|
|
295
|
+
fitted_categories.extend(group_categories)
|
|
296
|
+
aligned_tables.extend(group_tables)
|
|
297
|
+
|
|
298
|
+
member_table_ids = self._member_table_ids(ensemble_table)
|
|
299
|
+
output = ensemble_table.replace_tables(
|
|
300
|
+
tables=aligned_tables,
|
|
301
|
+
member_table_ids=member_table_ids,
|
|
302
|
+
)
|
|
303
|
+
self._categories = BufferList(
|
|
304
|
+
BufferList(categories) for categories in fitted_categories
|
|
305
|
+
)
|
|
306
|
+
return output
|
|
307
|
+
|
|
308
|
+
def _transform_ensemble(
|
|
309
|
+
self,
|
|
310
|
+
ensemble_table: EnsembleTable,
|
|
311
|
+
) -> EnsembleTable:
|
|
312
|
+
table_ids = self._member_table_ids(ensemble_table)
|
|
313
|
+
aligned_tables = []
|
|
314
|
+
offset = 0
|
|
315
|
+
for group in ensemble_table._iter_groups():
|
|
316
|
+
end = offset + group.size(0)
|
|
317
|
+
aligned_tables.extend(
|
|
318
|
+
self._align_to_categories(
|
|
319
|
+
group,
|
|
320
|
+
tuple(
|
|
321
|
+
cast(Sequence[Tensor], self._categories[index])
|
|
322
|
+
for index in range(offset, end)
|
|
323
|
+
),
|
|
324
|
+
)
|
|
325
|
+
)
|
|
326
|
+
offset = end
|
|
327
|
+
|
|
328
|
+
return ensemble_table.replace_tables(
|
|
329
|
+
tables=aligned_tables,
|
|
330
|
+
member_table_ids=table_ids,
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
@staticmethod
|
|
334
|
+
def _string_category_lookups(
|
|
335
|
+
input_categories: tuple[StringTensor, ...],
|
|
336
|
+
fitted_categories: tuple[Tensor, ...],
|
|
337
|
+
codes: Tensor,
|
|
338
|
+
) -> tuple[Tensor, ...]:
|
|
339
|
+
if not input_categories:
|
|
340
|
+
return ()
|
|
341
|
+
|
|
342
|
+
# Pair IDs for each input category: (total_input_categories,).
|
|
343
|
+
left_pair = torch.cat(
|
|
344
|
+
[
|
|
345
|
+
codes.new_full((categories.numel(),), pair_index)
|
|
346
|
+
for pair_index, categories in enumerate(input_categories)
|
|
347
|
+
]
|
|
348
|
+
)
|
|
349
|
+
# Pair IDs for each fitted category: (total_fitted_categories,).
|
|
350
|
+
right_pair = torch.cat(
|
|
351
|
+
[
|
|
352
|
+
codes.new_full((categories.numel(),), pair_index)
|
|
353
|
+
for pair_index, categories in enumerate(fitted_categories)
|
|
354
|
+
]
|
|
355
|
+
)
|
|
356
|
+
left_index, right_index = join_index(
|
|
357
|
+
left_table=TableTensor(
|
|
358
|
+
columns={"id": ("pair", "value")},
|
|
359
|
+
id=ColumnarTensor(
|
|
360
|
+
(left_pair, torch.cat(input_categories)),
|
|
361
|
+
),
|
|
362
|
+
),
|
|
363
|
+
right_table=TableTensor(
|
|
364
|
+
columns={"id": ("pair", "value")},
|
|
365
|
+
id=ColumnarTensor(
|
|
366
|
+
(right_pair, torch.cat(fitted_categories)),
|
|
367
|
+
),
|
|
368
|
+
),
|
|
369
|
+
left_keys=["pair", "value"],
|
|
370
|
+
right_keys=["pair", "value"],
|
|
371
|
+
dtype=codes.dtype,
|
|
372
|
+
)
|
|
373
|
+
fitted_codes = torch.cat(
|
|
374
|
+
[
|
|
375
|
+
torch.arange(
|
|
376
|
+
categories.numel(),
|
|
377
|
+
dtype=codes.dtype,
|
|
378
|
+
device=codes.device,
|
|
379
|
+
)
|
|
380
|
+
for categories in fitted_categories
|
|
381
|
+
]
|
|
382
|
+
)
|
|
383
|
+
# Fitted code for each input category: (total_input_categories,).
|
|
384
|
+
lookup = codes.new_full(
|
|
385
|
+
(sum(categories.numel() for categories in input_categories),),
|
|
386
|
+
-1,
|
|
387
|
+
)
|
|
388
|
+
lookup[left_index] = fitted_codes[right_index]
|
|
389
|
+
return lookup.split(
|
|
390
|
+
[categories.numel() for categories in input_categories]
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
@staticmethod
|
|
394
|
+
def _category_lookup(
|
|
395
|
+
input_categories: Tensor,
|
|
396
|
+
fitted_categories: Tensor,
|
|
397
|
+
codes: Tensor,
|
|
398
|
+
) -> Tensor:
|
|
399
|
+
lookup = codes.new_full((input_categories.numel(),), -1)
|
|
400
|
+
if fitted_categories.numel() == 0:
|
|
401
|
+
return lookup
|
|
402
|
+
|
|
403
|
+
comparable_categories = input_categories
|
|
404
|
+
sorted_categories = fitted_categories
|
|
405
|
+
if (
|
|
406
|
+
comparable_categories.dtype == torch.bool
|
|
407
|
+
or comparable_categories.dtype in _UNSIGNED_DTYPES
|
|
408
|
+
):
|
|
409
|
+
comparable_categories = comparable_categories.to(torch.int64)
|
|
410
|
+
sorted_categories = sorted_categories.to(torch.int64)
|
|
411
|
+
|
|
412
|
+
sorted_categories, perm = sorted_categories.sort()
|
|
413
|
+
position = torch.searchsorted(
|
|
414
|
+
sorted_categories,
|
|
415
|
+
comparable_categories,
|
|
416
|
+
)
|
|
417
|
+
position = position.clamp(max=sorted_categories.numel() - 1)
|
|
418
|
+
match = sorted_categories[position] == comparable_categories
|
|
419
|
+
left_index = match.nonzero().view(-1)
|
|
420
|
+
right_index = perm[position[left_index]]
|
|
421
|
+
lookup[left_index] = right_index.to(codes.dtype)
|
|
422
|
+
return lookup
|
|
423
|
+
|
|
424
|
+
def _align_to_categories(
|
|
425
|
+
self,
|
|
426
|
+
table: TableTensor,
|
|
427
|
+
fitted_categories: Sequence[Sequence[Tensor]],
|
|
428
|
+
) -> tuple[TableTensor, ...]:
|
|
429
|
+
codes = table.categorical.code
|
|
430
|
+
single_table = codes.dim() == 2
|
|
431
|
+
if single_table:
|
|
432
|
+
codes = codes.unsqueeze(0)
|
|
433
|
+
aligned_codes = torch.full_like(codes, -1)
|
|
434
|
+
lookups_by_column: list[dict[int, Tensor]] = [
|
|
435
|
+
{} for _ in table.categorical.categories
|
|
436
|
+
]
|
|
437
|
+
string_requests: list[tuple[int, int, StringTensor, Tensor]] = []
|
|
438
|
+
for column_index, input_categories in enumerate(
|
|
439
|
+
table.categorical.categories
|
|
440
|
+
):
|
|
441
|
+
if input_categories.numel() == 0:
|
|
442
|
+
continue
|
|
443
|
+
|
|
444
|
+
lookup_by_categories = lookups_by_column[column_index]
|
|
445
|
+
pending_categories: set[int] = set()
|
|
446
|
+
for batch_categories in fitted_categories:
|
|
447
|
+
column_categories = batch_categories[column_index]
|
|
448
|
+
identity = id(column_categories)
|
|
449
|
+
lookup = lookup_by_categories.get(identity)
|
|
450
|
+
if lookup is not None or identity in pending_categories:
|
|
451
|
+
continue
|
|
452
|
+
if column_categories.numel() == 0:
|
|
453
|
+
lookup_by_categories[identity] = codes.new_full(
|
|
454
|
+
(input_categories.numel(),),
|
|
455
|
+
-1,
|
|
456
|
+
)
|
|
457
|
+
continue
|
|
458
|
+
if isinstance(input_categories, StringTensor) != isinstance(
|
|
459
|
+
column_categories, StringTensor
|
|
460
|
+
):
|
|
461
|
+
raise NotImplementedError(
|
|
462
|
+
"Cannot align string categories with non-string "
|
|
463
|
+
"categories"
|
|
464
|
+
)
|
|
465
|
+
if isinstance(input_categories, StringTensor):
|
|
466
|
+
pending_categories.add(identity)
|
|
467
|
+
string_requests.append(
|
|
468
|
+
(
|
|
469
|
+
column_index,
|
|
470
|
+
identity,
|
|
471
|
+
input_categories,
|
|
472
|
+
column_categories,
|
|
473
|
+
)
|
|
474
|
+
)
|
|
475
|
+
continue
|
|
476
|
+
lookup_by_categories[identity] = self._category_lookup(
|
|
477
|
+
input_categories,
|
|
478
|
+
column_categories,
|
|
479
|
+
codes,
|
|
480
|
+
)
|
|
481
|
+
|
|
482
|
+
string_lookups = self._string_category_lookups(
|
|
483
|
+
input_categories=tuple(
|
|
484
|
+
input_categories
|
|
485
|
+
for _, _, input_categories, _ in string_requests
|
|
486
|
+
),
|
|
487
|
+
fitted_categories=tuple(
|
|
488
|
+
fitted_categories
|
|
489
|
+
for _, _, _, fitted_categories in string_requests
|
|
490
|
+
),
|
|
491
|
+
codes=codes,
|
|
492
|
+
)
|
|
493
|
+
for (column_index, identity, _, _), lookup in zip(
|
|
494
|
+
string_requests,
|
|
495
|
+
string_lookups,
|
|
496
|
+
strict=True,
|
|
497
|
+
):
|
|
498
|
+
lookups_by_column[column_index][identity] = lookup
|
|
499
|
+
|
|
500
|
+
for column_index, input_categories in enumerate(
|
|
501
|
+
table.categorical.categories
|
|
502
|
+
):
|
|
503
|
+
if input_categories.numel() == 0:
|
|
504
|
+
continue
|
|
505
|
+
lookup_by_categories = lookups_by_column[column_index]
|
|
506
|
+
lookups = [
|
|
507
|
+
lookup_by_categories[id(batch_categories[column_index])]
|
|
508
|
+
for batch_categories in fitted_categories
|
|
509
|
+
]
|
|
510
|
+
column_codes = codes[..., column_index]
|
|
511
|
+
mask = column_codes >= 0
|
|
512
|
+
lookup = torch.stack(lookups)
|
|
513
|
+
indices = column_codes.clamp_min(0).long()
|
|
514
|
+
aligned_codes[..., column_index] = torch.where(
|
|
515
|
+
mask,
|
|
516
|
+
lookup.gather(1, indices),
|
|
517
|
+
-1,
|
|
518
|
+
)
|
|
519
|
+
|
|
520
|
+
return tuple(
|
|
521
|
+
(table if single_table else table[batch_index]).replace_blocks(
|
|
522
|
+
categorical=CategoricalTensor(
|
|
523
|
+
aligned_codes[batch_index],
|
|
524
|
+
categories=batch_categories,
|
|
525
|
+
),
|
|
526
|
+
)
|
|
527
|
+
for batch_index, batch_categories in enumerate(fitted_categories)
|
|
528
|
+
)
|
|
529
|
+
|
|
530
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
531
|
+
arguments = []
|
|
532
|
+
if self.sort_by != "code":
|
|
533
|
+
arguments.append(f"sort_by={self.sort_by!r}")
|
|
534
|
+
if self.min_frequency != 1:
|
|
535
|
+
arguments.append(f"min_frequency={self.min_frequency!r}")
|
|
536
|
+
if not arguments:
|
|
537
|
+
return super().__repr__(indent=indent)
|
|
538
|
+
return (
|
|
539
|
+
f"{' ' * indent}{self.__class__.__name__}({', '.join(arguments)})"
|
|
540
|
+
)
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from typing import cast
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
from torch import Tensor
|
|
8
|
+
|
|
9
|
+
from sdm import Stype, TableTensor
|
|
10
|
+
from sdm.nn._buffer import BufferList
|
|
11
|
+
from sdm.processing import Processor
|
|
12
|
+
from sdm.processing.categorical._categorical import (
|
|
13
|
+
_check_categorical_codes,
|
|
14
|
+
_check_categories,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class AddCategoryCounts(Processor):
|
|
19
|
+
"""Add the log row count of each categorical value.
|
|
20
|
+
|
|
21
|
+
Each eligible column receives one numerical column ``<column>__count``
|
|
22
|
+
holding ``log1p`` of the number of fitted rows sharing the row's code.
|
|
23
|
+
Negative codes (missing or unknown values) share one count.
|
|
24
|
+
|
|
25
|
+
Transform inputs must use the fitted per-column category vocabularies.
|
|
26
|
+
Use :class:`~sdm.processing.AlignCategories` before this processor when
|
|
27
|
+
training and transform inputs were tensorized independently.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
min_cardinality: A count column is added when its fitted vocabulary
|
|
31
|
+
size exceeds this value. Missing values do not contribute to
|
|
32
|
+
vocabulary size. Defaults to 0.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
handles_stypes = frozenset({Stype.categorical})
|
|
36
|
+
requires_fit = True
|
|
37
|
+
|
|
38
|
+
def __init__(self, min_cardinality: int = 0) -> None:
|
|
39
|
+
super().__init__()
|
|
40
|
+
self.min_cardinality = min_cardinality
|
|
41
|
+
self._columns: tuple[str, ...] = ()
|
|
42
|
+
self._categories: BufferList[Tensor] = BufferList()
|
|
43
|
+
self.register_buffer("_log_counts", torch.empty(0))
|
|
44
|
+
self.register_buffer(
|
|
45
|
+
"_num_categories",
|
|
46
|
+
torch.empty(0, dtype=torch.long),
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
def get_extra_state(self) -> tuple[str, ...]:
|
|
50
|
+
r""":meta private:""" # noqa: D415
|
|
51
|
+
return self._columns
|
|
52
|
+
|
|
53
|
+
def set_extra_state(self, state: object) -> None:
|
|
54
|
+
r""":meta private:""" # noqa: D415
|
|
55
|
+
self._columns = cast(tuple[str, ...], state)
|
|
56
|
+
|
|
57
|
+
def _fit(
|
|
58
|
+
self,
|
|
59
|
+
table: TableTensor,
|
|
60
|
+
*,
|
|
61
|
+
generator: torch.Generator | None = None,
|
|
62
|
+
) -> None:
|
|
63
|
+
_check_categorical_codes(table)
|
|
64
|
+
self._columns = tuple(
|
|
65
|
+
column
|
|
66
|
+
for column, categories in zip(
|
|
67
|
+
table.columns[Stype.categorical], table.categorical.categories
|
|
68
|
+
)
|
|
69
|
+
if categories.numel() > self.min_cardinality
|
|
70
|
+
)
|
|
71
|
+
categorical = table.select_columns(self._columns).categorical
|
|
72
|
+
codes = categorical.code # [*batch, num_rows, num_columns]
|
|
73
|
+
sizes = [categories.numel() for categories in categorical.categories]
|
|
74
|
+
num_categories = codes.new_tensor(sizes, dtype=torch.long)
|
|
75
|
+
# Negative codes accumulate in the slot behind their column's
|
|
76
|
+
# vocabulary: [*batch, num_columns, max(sizes) + 1].
|
|
77
|
+
slots = torch.where(codes >= 0, codes.long(), num_categories)
|
|
78
|
+
slots = slots.transpose(-2, -1)
|
|
79
|
+
counts = torch.zeros(
|
|
80
|
+
(*codes.shape[:-2], len(sizes), max(sizes, default=0) + 1),
|
|
81
|
+
dtype=torch.long,
|
|
82
|
+
device=codes.device,
|
|
83
|
+
)
|
|
84
|
+
counts.scatter_add_(-1, slots, torch.ones_like(slots))
|
|
85
|
+
|
|
86
|
+
dtype = table.numerical.dtype
|
|
87
|
+
count_dtype = (
|
|
88
|
+
torch.float32
|
|
89
|
+
if dtype in {torch.float16, torch.bfloat16}
|
|
90
|
+
else dtype
|
|
91
|
+
)
|
|
92
|
+
self._log_counts = counts.to(count_dtype).log1p().to(dtype)
|
|
93
|
+
self._num_categories = num_categories
|
|
94
|
+
self._categories = BufferList(table.categorical.categories)
|
|
95
|
+
|
|
96
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
97
|
+
_check_categories(table, self._categories)
|
|
98
|
+
_check_categorical_codes(table)
|
|
99
|
+
if not self._columns:
|
|
100
|
+
return table
|
|
101
|
+
codes = table.select_columns(self._columns).categorical.code
|
|
102
|
+
slots = torch.where(codes >= 0, codes.long(), self._num_categories)
|
|
103
|
+
counts = self._log_counts.gather(-1, slots.transpose(-2, -1))
|
|
104
|
+
|
|
105
|
+
out_table = TableTensor(
|
|
106
|
+
columns={
|
|
107
|
+
Stype.numerical: tuple(
|
|
108
|
+
f"{column}__count" for column in self._columns
|
|
109
|
+
),
|
|
110
|
+
},
|
|
111
|
+
numerical=counts.transpose(-2, -1).to(table.numerical.dtype),
|
|
112
|
+
)
|
|
113
|
+
return cast(TableTensor, torch.cat([table, out_table], dim=-1))
|
|
114
|
+
|
|
115
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
116
|
+
if self.min_cardinality == 0:
|
|
117
|
+
return super().__repr__(indent=indent)
|
|
118
|
+
return (
|
|
119
|
+
f"{' ' * indent}{self.__class__.__name__}"
|
|
120
|
+
f"(min_cardinality={self.min_cardinality!r})"
|
|
121
|
+
)
|