structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,540 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from collections.abc import Sequence
5
+ from typing import Literal, cast
6
+
7
+ import torch
8
+ from torch import Tensor
9
+
10
+ from sdm import (
11
+ CategoricalTensor,
12
+ ColumnarTensor,
13
+ EnsembleTable,
14
+ StringTensor,
15
+ Stype,
16
+ TableTensor,
17
+ )
18
+ from sdm.nn._buffer import BufferList
19
+ from sdm.processing import EnsembleProcessor
20
+ from sdm.relational.join import join_index
21
+
22
+ _UNSIGNED_DTYPES = frozenset({torch.uint16, torch.uint32, torch.uint64})
23
+
24
+
25
+ class AlignCategories(EnsembleProcessor):
26
+ """Align categorical columns to categories observed during fitting.
27
+
28
+ A categorical column stores each value as an integer code indexing an
29
+ ordered list of categories. Separately created tables can use different
30
+ codes for the same value. Fitting learns the category list for each column,
31
+ and transforming remaps another table to use it. Missing values and
32
+ categories not retained during fitting receive code ``-1``.
33
+
34
+ Args:
35
+ sort_by: How to order fitted categories.
36
+ ``"code"`` keeps observed categories in original order.
37
+ ``"frequency"`` orders observed categories by descending frequency.
38
+ ``"value"`` orders observed categories by ascending value.
39
+ min_frequency: Minimum number of observations required to retain a
40
+ category. Values of rarer categories receive code ``-1``.
41
+ Must be positive.
42
+
43
+ >>> import pandas as pd
44
+ >>> import sdm
45
+ >>> from sdm.processing import AlignCategories
46
+ >>> table1 = sdm.TableTensor.from_pandas(
47
+ ... pd.DataFrame({"color": ["red", "blue", "red"]}),
48
+ ... stypes={"color": "categorical"},
49
+ ... )
50
+ >>> table2 = sdm.TableTensor.from_pandas(
51
+ ... pd.DataFrame({"color": ["blue", "green", None]}),
52
+ ... stypes={"color": "categorical"},
53
+ ... )
54
+ >>> table1.categorical.categories[0].tolist()
55
+ ['red', 'blue']
56
+ >>> table2.categorical.categories[0].tolist()
57
+ ['blue', 'green']
58
+ >>> table2.categorical.code[:, 0].tolist()
59
+ [0, 1, -1]
60
+ >>> processor = AlignCategories().fit(table1)
61
+ >>> table2 = processor.transform(table2)
62
+ >>> table2.categorical.categories[0].tolist()
63
+ ['red', 'blue']
64
+ >>> table2.categorical.code[:, 0].tolist()
65
+ [1, -1, -1]
66
+
67
+ Here, ``"blue"`` changes from code ``0`` to ``1``, while unseen
68
+ ``"green"`` and the missing value use ``-1``.
69
+ """
70
+
71
+ handles_stypes = frozenset({Stype.categorical})
72
+ requires_fit = True
73
+
74
+ def __init__(
75
+ self,
76
+ sort_by: Literal["code", "frequency", "value"] = "code",
77
+ *,
78
+ min_frequency: int = 1,
79
+ ) -> None:
80
+ super().__init__()
81
+ if min_frequency <= 0:
82
+ raise ValueError("min_frequency must be positive")
83
+ self.sort_by = sort_by
84
+ self.min_frequency = min_frequency
85
+ self._categories: BufferList[BufferList[Tensor]] = BufferList()
86
+
87
+ def _fit_column(
88
+ self,
89
+ input_categories: Tensor,
90
+ codes: Tensor,
91
+ *,
92
+ align_codes: bool,
93
+ ) -> tuple[tuple[Tensor, ...], Tensor | None]:
94
+ batch_size = codes.size(0)
95
+ if input_categories.numel() == 0:
96
+ aligned_codes = torch.full_like(codes, -1) if align_codes else None
97
+ return (input_categories,) * batch_size, aligned_codes
98
+
99
+ mask = codes >= 0
100
+ indices = codes.clamp_min(0).long()
101
+ # Count categories independently per batch: [B, N] -> [B, K].
102
+ counts = codes.new_zeros((batch_size, input_categories.numel()))
103
+ counts.scatter_add_(1, indices, mask.to(codes.dtype))
104
+
105
+ if self.sort_by == "frequency":
106
+ order = counts.argsort(dim=1, descending=True, stable=True)
107
+ ordered_categories = input_categories
108
+ elif self.sort_by == "value":
109
+ if (
110
+ input_categories.is_cuda
111
+ and input_categories.dtype in _UNSIGNED_DTYPES
112
+ ):
113
+ key = input_categories.to(torch.int64)
114
+ if input_categories.dtype == torch.uint64:
115
+ # Map unsigned integer order onto signed integer order.
116
+ key = key.bitwise_xor(torch.iinfo(torch.int64).min)
117
+ perm = key.argsort()
118
+ ordered_categories = input_categories.index_select(0, perm)
119
+ else:
120
+ ordered_categories, perm = input_categories.sort()
121
+ order = perm.expand(batch_size, -1)
122
+ else:
123
+ assert self.sort_by == "code"
124
+ order = torch.arange(
125
+ input_categories.numel(),
126
+ device=codes.device,
127
+ ).expand(batch_size, -1)
128
+ ordered_categories = input_categories
129
+
130
+ observed = counts.gather(1, order) >= self.min_frequency
131
+ # Only ragged vocabularies require per-batch materialization.
132
+ fitted_categories = []
133
+ for batch_index in range(batch_size):
134
+ selected_indices = order[batch_index, observed[batch_index]]
135
+ if (
136
+ selected_indices.numel() == input_categories.numel()
137
+ and self.sort_by != "frequency"
138
+ ):
139
+ fitted_categories.append(ordered_categories)
140
+ elif (
141
+ input_categories.dtype in _UNSIGNED_DTYPES
142
+ and input_categories.is_cpu
143
+ ):
144
+ # PyTorch CPU index_select is not implemented for these dtypes.
145
+ fitted_categories.append(input_categories[selected_indices])
146
+ else:
147
+ fitted_categories.append(
148
+ input_categories.index_select(0, selected_indices)
149
+ )
150
+
151
+ if not align_codes:
152
+ return tuple(fitted_categories), None
153
+
154
+ rank = observed.cumsum(dim=1, dtype=codes.dtype) - 1
155
+ lookup = torch.full_like(counts, -1)
156
+ lookup.scatter_(1, order, torch.where(observed, rank, -1))
157
+ aligned_codes = torch.where(mask, lookup.gather(1, indices), -1)
158
+ return tuple(fitted_categories), aligned_codes
159
+
160
+ def _fit_columns(
161
+ self,
162
+ table: TableTensor,
163
+ *,
164
+ align_codes: bool,
165
+ ) -> tuple[tuple[tuple[Tensor, ...], ...], Tensor | None]:
166
+ codes = table.categorical.code
167
+ if codes.dim() == 2:
168
+ codes = codes.unsqueeze(0)
169
+ aligned_codes = torch.full_like(codes, -1) if align_codes else None
170
+
171
+ # Accumulate ragged vocabularies in batch-major order: [batch][column].
172
+ categories_by_batch: list[list[Tensor]] = [
173
+ [] for _ in range(codes.size(0))
174
+ ]
175
+ for column_index, input_categories in enumerate(
176
+ table.categorical.categories
177
+ ):
178
+ fitted_categories, aligned_column_codes = self._fit_column(
179
+ input_categories=input_categories,
180
+ codes=codes[..., column_index],
181
+ align_codes=align_codes,
182
+ )
183
+ if aligned_codes is not None:
184
+ assert aligned_column_codes is not None
185
+ aligned_codes[..., column_index] = aligned_column_codes
186
+ for batch_categories, column_categories in zip(
187
+ categories_by_batch,
188
+ fitted_categories,
189
+ strict=True,
190
+ ):
191
+ batch_categories.append(column_categories)
192
+
193
+ fitted_categories = tuple(
194
+ tuple(batch_categories) for batch_categories in categories_by_batch
195
+ )
196
+ return fitted_categories, aligned_codes
197
+
198
+ def _fit_and_align(
199
+ self,
200
+ table: TableTensor,
201
+ ) -> tuple[tuple[tuple[Tensor, ...], ...], tuple[TableTensor, ...]]:
202
+ single_table = table.categorical.code.dim() == 2
203
+ fitted_categories, aligned_codes = self._fit_columns(
204
+ table,
205
+ align_codes=True,
206
+ )
207
+ assert aligned_codes is not None
208
+ aligned_tables = tuple(
209
+ (table if single_table else table[batch_index]).replace_blocks(
210
+ categorical=CategoricalTensor(
211
+ aligned_codes[batch_index],
212
+ categories=batch_categories,
213
+ ),
214
+ )
215
+ for batch_index, batch_categories in enumerate(fitted_categories)
216
+ )
217
+ return fitted_categories, aligned_tables
218
+
219
+ def _fit(
220
+ self,
221
+ table: TableTensor,
222
+ *,
223
+ generator: torch.Generator | None = None,
224
+ ) -> None:
225
+ fitted_categories, _ = self._fit_columns(
226
+ table,
227
+ align_codes=False,
228
+ )
229
+ self._categories = BufferList(
230
+ BufferList(categories) for categories in fitted_categories
231
+ )
232
+
233
+ def _fit_transform(
234
+ self,
235
+ table: TableTensor,
236
+ *,
237
+ generator: torch.Generator | None = None,
238
+ ) -> TableTensor:
239
+ fitted_categories, aligned_tables = self._fit_and_align(table)
240
+ self._categories = BufferList(
241
+ BufferList(categories) for categories in fitted_categories
242
+ )
243
+ return aligned_tables[0]
244
+
245
+ def _transform(self, table: TableTensor) -> TableTensor:
246
+ return self._align_to_categories(
247
+ table,
248
+ cast(Sequence[Sequence[Tensor]], self._categories),
249
+ )[0]
250
+
251
+ @staticmethod
252
+ def _member_table_ids(
253
+ ensemble_table: EnsembleTable,
254
+ ) -> tuple[int, ...]:
255
+ group_offsets = []
256
+ offset = 0
257
+ for group in ensemble_table._iter_groups():
258
+ group_offsets.append(offset)
259
+ offset += group.size(0)
260
+
261
+ # TODO: Replace this private access with a public EnsembleTable
262
+ # operation returning flattened tables and member table IDs.
263
+ return tuple(
264
+ group_offsets[group_index] + position
265
+ for group_index, position in ensemble_table._locations
266
+ )
267
+
268
+ def _fit_ensemble(
269
+ self,
270
+ ensemble_table: EnsembleTable,
271
+ *,
272
+ generator: torch.Generator | None = None,
273
+ ) -> None:
274
+ fitted_categories = []
275
+ for group in ensemble_table._iter_groups():
276
+ group_categories, _ = self._fit_columns(
277
+ group,
278
+ align_codes=False,
279
+ )
280
+ fitted_categories.extend(group_categories)
281
+ self._categories = BufferList(
282
+ BufferList(categories) for categories in fitted_categories
283
+ )
284
+
285
+ def _fit_transform_ensemble(
286
+ self,
287
+ ensemble_table: EnsembleTable,
288
+ *,
289
+ generator: torch.Generator | None = None,
290
+ ) -> EnsembleTable:
291
+ fitted_categories = []
292
+ aligned_tables = []
293
+ for group in ensemble_table._iter_groups():
294
+ group_categories, group_tables = self._fit_and_align(group)
295
+ fitted_categories.extend(group_categories)
296
+ aligned_tables.extend(group_tables)
297
+
298
+ member_table_ids = self._member_table_ids(ensemble_table)
299
+ output = ensemble_table.replace_tables(
300
+ tables=aligned_tables,
301
+ member_table_ids=member_table_ids,
302
+ )
303
+ self._categories = BufferList(
304
+ BufferList(categories) for categories in fitted_categories
305
+ )
306
+ return output
307
+
308
+ def _transform_ensemble(
309
+ self,
310
+ ensemble_table: EnsembleTable,
311
+ ) -> EnsembleTable:
312
+ table_ids = self._member_table_ids(ensemble_table)
313
+ aligned_tables = []
314
+ offset = 0
315
+ for group in ensemble_table._iter_groups():
316
+ end = offset + group.size(0)
317
+ aligned_tables.extend(
318
+ self._align_to_categories(
319
+ group,
320
+ tuple(
321
+ cast(Sequence[Tensor], self._categories[index])
322
+ for index in range(offset, end)
323
+ ),
324
+ )
325
+ )
326
+ offset = end
327
+
328
+ return ensemble_table.replace_tables(
329
+ tables=aligned_tables,
330
+ member_table_ids=table_ids,
331
+ )
332
+
333
+ @staticmethod
334
+ def _string_category_lookups(
335
+ input_categories: tuple[StringTensor, ...],
336
+ fitted_categories: tuple[Tensor, ...],
337
+ codes: Tensor,
338
+ ) -> tuple[Tensor, ...]:
339
+ if not input_categories:
340
+ return ()
341
+
342
+ # Pair IDs for each input category: (total_input_categories,).
343
+ left_pair = torch.cat(
344
+ [
345
+ codes.new_full((categories.numel(),), pair_index)
346
+ for pair_index, categories in enumerate(input_categories)
347
+ ]
348
+ )
349
+ # Pair IDs for each fitted category: (total_fitted_categories,).
350
+ right_pair = torch.cat(
351
+ [
352
+ codes.new_full((categories.numel(),), pair_index)
353
+ for pair_index, categories in enumerate(fitted_categories)
354
+ ]
355
+ )
356
+ left_index, right_index = join_index(
357
+ left_table=TableTensor(
358
+ columns={"id": ("pair", "value")},
359
+ id=ColumnarTensor(
360
+ (left_pair, torch.cat(input_categories)),
361
+ ),
362
+ ),
363
+ right_table=TableTensor(
364
+ columns={"id": ("pair", "value")},
365
+ id=ColumnarTensor(
366
+ (right_pair, torch.cat(fitted_categories)),
367
+ ),
368
+ ),
369
+ left_keys=["pair", "value"],
370
+ right_keys=["pair", "value"],
371
+ dtype=codes.dtype,
372
+ )
373
+ fitted_codes = torch.cat(
374
+ [
375
+ torch.arange(
376
+ categories.numel(),
377
+ dtype=codes.dtype,
378
+ device=codes.device,
379
+ )
380
+ for categories in fitted_categories
381
+ ]
382
+ )
383
+ # Fitted code for each input category: (total_input_categories,).
384
+ lookup = codes.new_full(
385
+ (sum(categories.numel() for categories in input_categories),),
386
+ -1,
387
+ )
388
+ lookup[left_index] = fitted_codes[right_index]
389
+ return lookup.split(
390
+ [categories.numel() for categories in input_categories]
391
+ )
392
+
393
+ @staticmethod
394
+ def _category_lookup(
395
+ input_categories: Tensor,
396
+ fitted_categories: Tensor,
397
+ codes: Tensor,
398
+ ) -> Tensor:
399
+ lookup = codes.new_full((input_categories.numel(),), -1)
400
+ if fitted_categories.numel() == 0:
401
+ return lookup
402
+
403
+ comparable_categories = input_categories
404
+ sorted_categories = fitted_categories
405
+ if (
406
+ comparable_categories.dtype == torch.bool
407
+ or comparable_categories.dtype in _UNSIGNED_DTYPES
408
+ ):
409
+ comparable_categories = comparable_categories.to(torch.int64)
410
+ sorted_categories = sorted_categories.to(torch.int64)
411
+
412
+ sorted_categories, perm = sorted_categories.sort()
413
+ position = torch.searchsorted(
414
+ sorted_categories,
415
+ comparable_categories,
416
+ )
417
+ position = position.clamp(max=sorted_categories.numel() - 1)
418
+ match = sorted_categories[position] == comparable_categories
419
+ left_index = match.nonzero().view(-1)
420
+ right_index = perm[position[left_index]]
421
+ lookup[left_index] = right_index.to(codes.dtype)
422
+ return lookup
423
+
424
+ def _align_to_categories(
425
+ self,
426
+ table: TableTensor,
427
+ fitted_categories: Sequence[Sequence[Tensor]],
428
+ ) -> tuple[TableTensor, ...]:
429
+ codes = table.categorical.code
430
+ single_table = codes.dim() == 2
431
+ if single_table:
432
+ codes = codes.unsqueeze(0)
433
+ aligned_codes = torch.full_like(codes, -1)
434
+ lookups_by_column: list[dict[int, Tensor]] = [
435
+ {} for _ in table.categorical.categories
436
+ ]
437
+ string_requests: list[tuple[int, int, StringTensor, Tensor]] = []
438
+ for column_index, input_categories in enumerate(
439
+ table.categorical.categories
440
+ ):
441
+ if input_categories.numel() == 0:
442
+ continue
443
+
444
+ lookup_by_categories = lookups_by_column[column_index]
445
+ pending_categories: set[int] = set()
446
+ for batch_categories in fitted_categories:
447
+ column_categories = batch_categories[column_index]
448
+ identity = id(column_categories)
449
+ lookup = lookup_by_categories.get(identity)
450
+ if lookup is not None or identity in pending_categories:
451
+ continue
452
+ if column_categories.numel() == 0:
453
+ lookup_by_categories[identity] = codes.new_full(
454
+ (input_categories.numel(),),
455
+ -1,
456
+ )
457
+ continue
458
+ if isinstance(input_categories, StringTensor) != isinstance(
459
+ column_categories, StringTensor
460
+ ):
461
+ raise NotImplementedError(
462
+ "Cannot align string categories with non-string "
463
+ "categories"
464
+ )
465
+ if isinstance(input_categories, StringTensor):
466
+ pending_categories.add(identity)
467
+ string_requests.append(
468
+ (
469
+ column_index,
470
+ identity,
471
+ input_categories,
472
+ column_categories,
473
+ )
474
+ )
475
+ continue
476
+ lookup_by_categories[identity] = self._category_lookup(
477
+ input_categories,
478
+ column_categories,
479
+ codes,
480
+ )
481
+
482
+ string_lookups = self._string_category_lookups(
483
+ input_categories=tuple(
484
+ input_categories
485
+ for _, _, input_categories, _ in string_requests
486
+ ),
487
+ fitted_categories=tuple(
488
+ fitted_categories
489
+ for _, _, _, fitted_categories in string_requests
490
+ ),
491
+ codes=codes,
492
+ )
493
+ for (column_index, identity, _, _), lookup in zip(
494
+ string_requests,
495
+ string_lookups,
496
+ strict=True,
497
+ ):
498
+ lookups_by_column[column_index][identity] = lookup
499
+
500
+ for column_index, input_categories in enumerate(
501
+ table.categorical.categories
502
+ ):
503
+ if input_categories.numel() == 0:
504
+ continue
505
+ lookup_by_categories = lookups_by_column[column_index]
506
+ lookups = [
507
+ lookup_by_categories[id(batch_categories[column_index])]
508
+ for batch_categories in fitted_categories
509
+ ]
510
+ column_codes = codes[..., column_index]
511
+ mask = column_codes >= 0
512
+ lookup = torch.stack(lookups)
513
+ indices = column_codes.clamp_min(0).long()
514
+ aligned_codes[..., column_index] = torch.where(
515
+ mask,
516
+ lookup.gather(1, indices),
517
+ -1,
518
+ )
519
+
520
+ return tuple(
521
+ (table if single_table else table[batch_index]).replace_blocks(
522
+ categorical=CategoricalTensor(
523
+ aligned_codes[batch_index],
524
+ categories=batch_categories,
525
+ ),
526
+ )
527
+ for batch_index, batch_categories in enumerate(fitted_categories)
528
+ )
529
+
530
+ def __repr__(self, *, indent: int = 0) -> str:
531
+ arguments = []
532
+ if self.sort_by != "code":
533
+ arguments.append(f"sort_by={self.sort_by!r}")
534
+ if self.min_frequency != 1:
535
+ arguments.append(f"min_frequency={self.min_frequency!r}")
536
+ if not arguments:
537
+ return super().__repr__(indent=indent)
538
+ return (
539
+ f"{' ' * indent}{self.__class__.__name__}({', '.join(arguments)})"
540
+ )
@@ -0,0 +1,121 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from typing import cast
5
+
6
+ import torch
7
+ from torch import Tensor
8
+
9
+ from sdm import Stype, TableTensor
10
+ from sdm.nn._buffer import BufferList
11
+ from sdm.processing import Processor
12
+ from sdm.processing.categorical._categorical import (
13
+ _check_categorical_codes,
14
+ _check_categories,
15
+ )
16
+
17
+
18
+ class AddCategoryCounts(Processor):
19
+ """Add the log row count of each categorical value.
20
+
21
+ Each eligible column receives one numerical column ``<column>__count``
22
+ holding ``log1p`` of the number of fitted rows sharing the row's code.
23
+ Negative codes (missing or unknown values) share one count.
24
+
25
+ Transform inputs must use the fitted per-column category vocabularies.
26
+ Use :class:`~sdm.processing.AlignCategories` before this processor when
27
+ training and transform inputs were tensorized independently.
28
+
29
+ Args:
30
+ min_cardinality: A count column is added when its fitted vocabulary
31
+ size exceeds this value. Missing values do not contribute to
32
+ vocabulary size. Defaults to 0.
33
+ """
34
+
35
+ handles_stypes = frozenset({Stype.categorical})
36
+ requires_fit = True
37
+
38
+ def __init__(self, min_cardinality: int = 0) -> None:
39
+ super().__init__()
40
+ self.min_cardinality = min_cardinality
41
+ self._columns: tuple[str, ...] = ()
42
+ self._categories: BufferList[Tensor] = BufferList()
43
+ self.register_buffer("_log_counts", torch.empty(0))
44
+ self.register_buffer(
45
+ "_num_categories",
46
+ torch.empty(0, dtype=torch.long),
47
+ )
48
+
49
+ def get_extra_state(self) -> tuple[str, ...]:
50
+ r""":meta private:""" # noqa: D415
51
+ return self._columns
52
+
53
+ def set_extra_state(self, state: object) -> None:
54
+ r""":meta private:""" # noqa: D415
55
+ self._columns = cast(tuple[str, ...], state)
56
+
57
+ def _fit(
58
+ self,
59
+ table: TableTensor,
60
+ *,
61
+ generator: torch.Generator | None = None,
62
+ ) -> None:
63
+ _check_categorical_codes(table)
64
+ self._columns = tuple(
65
+ column
66
+ for column, categories in zip(
67
+ table.columns[Stype.categorical], table.categorical.categories
68
+ )
69
+ if categories.numel() > self.min_cardinality
70
+ )
71
+ categorical = table.select_columns(self._columns).categorical
72
+ codes = categorical.code # [*batch, num_rows, num_columns]
73
+ sizes = [categories.numel() for categories in categorical.categories]
74
+ num_categories = codes.new_tensor(sizes, dtype=torch.long)
75
+ # Negative codes accumulate in the slot behind their column's
76
+ # vocabulary: [*batch, num_columns, max(sizes) + 1].
77
+ slots = torch.where(codes >= 0, codes.long(), num_categories)
78
+ slots = slots.transpose(-2, -1)
79
+ counts = torch.zeros(
80
+ (*codes.shape[:-2], len(sizes), max(sizes, default=0) + 1),
81
+ dtype=torch.long,
82
+ device=codes.device,
83
+ )
84
+ counts.scatter_add_(-1, slots, torch.ones_like(slots))
85
+
86
+ dtype = table.numerical.dtype
87
+ count_dtype = (
88
+ torch.float32
89
+ if dtype in {torch.float16, torch.bfloat16}
90
+ else dtype
91
+ )
92
+ self._log_counts = counts.to(count_dtype).log1p().to(dtype)
93
+ self._num_categories = num_categories
94
+ self._categories = BufferList(table.categorical.categories)
95
+
96
+ def _transform(self, table: TableTensor) -> TableTensor:
97
+ _check_categories(table, self._categories)
98
+ _check_categorical_codes(table)
99
+ if not self._columns:
100
+ return table
101
+ codes = table.select_columns(self._columns).categorical.code
102
+ slots = torch.where(codes >= 0, codes.long(), self._num_categories)
103
+ counts = self._log_counts.gather(-1, slots.transpose(-2, -1))
104
+
105
+ out_table = TableTensor(
106
+ columns={
107
+ Stype.numerical: tuple(
108
+ f"{column}__count" for column in self._columns
109
+ ),
110
+ },
111
+ numerical=counts.transpose(-2, -1).to(table.numerical.dtype),
112
+ )
113
+ return cast(TableTensor, torch.cat([table, out_table], dim=-1))
114
+
115
+ def __repr__(self, *, indent: int = 0) -> str:
116
+ if self.min_cardinality == 0:
117
+ return super().__repr__(indent=indent)
118
+ return (
119
+ f"{' ' * indent}{self.__class__.__name__}"
120
+ f"(min_cardinality={self.min_cardinality!r})"
121
+ )