structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,478 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import math
5
+ import re
6
+ from itertools import accumulate
7
+ from typing import cast
8
+
9
+ import pyarrow as pa
10
+ import pyarrow.compute as pc
11
+ import torch
12
+ from torch import Tensor
13
+ from torch.utils.dlpack import from_dlpack
14
+
15
+ from sdm import EnsembleTable, StringTensor, Stype, TableTensor
16
+ from sdm.processing import EnsembleProcessor
17
+ from sdm.tensor.io import arrow_as_tensor
18
+
19
+
20
+ class _TFIDFState(torch.nn.Module):
21
+ def __init__(
22
+ self,
23
+ vocabularies: list[pa.Array],
24
+ idfs: list[Tensor],
25
+ ) -> None:
26
+ super().__init__()
27
+ self.vocabularies = vocabularies
28
+ for column, idf in enumerate(idfs):
29
+ self.register_buffer(f"idf_{column}", idf)
30
+
31
+
32
+ class TFIDF(EnsembleProcessor):
33
+ """Encode text columns as character n-gram TF-IDF vectors.
34
+
35
+ Tokenization follows scikit-learn's ``char_wb`` analyzer: whitespace-
36
+ delimited words are space-padded before windowing, so a word shorter than
37
+ ``n`` still yields one n-gram. Fit learns a vocabulary and smoothed idf
38
+ weights per text column. Transform replaces text with concatenated
39
+ numerical features (one per retained n-gram), applies those idf weights,
40
+ L2-normalizes each row, and ignores n-grams unseen at fit time.
41
+
42
+ Args:
43
+ ngram_range: Inclusive ``(min_n, max_n)`` character-window sizes.
44
+ max_features: If set, keep only this many most frequent n-grams per
45
+ column. ``None`` keeps the full vocabulary.
46
+ lowercase: If ``True``, lowercase text before tokenizing.
47
+ """
48
+
49
+ handles_stypes = frozenset({Stype.text})
50
+ requires_fit = True
51
+
52
+ def __init__(
53
+ self,
54
+ *,
55
+ ngram_range: tuple[int, int],
56
+ max_features: int | None = None,
57
+ lowercase: bool = True,
58
+ ) -> None:
59
+ super().__init__()
60
+ min_n, max_n = ngram_range
61
+ if min_n < 1 or max_n < min_n:
62
+ raise ValueError("ngram_range must satisfy 1 <= min_n <= max_n.")
63
+ if max_features is not None and max_features <= 0:
64
+ raise ValueError("max_features must be positive or None.")
65
+ self.ngram_range = ngram_range
66
+ self.max_features = max_features
67
+ self.lowercase = lowercase
68
+ self.register_buffer(
69
+ "_idf_template",
70
+ torch.empty(0),
71
+ persistent=False,
72
+ )
73
+ self._states = torch.nn.ModuleList()
74
+ self._member_state_ids: tuple[int, ...] = ()
75
+
76
+ def get_extra_state(
77
+ self,
78
+ ) -> tuple[tuple[tuple[tuple[str, ...], ...], ...], tuple[int, ...]]:
79
+ r""":meta private:""" # noqa: D415
80
+ vocabularies = tuple(
81
+ tuple(
82
+ tuple(cast(list[str], vocabulary.to_pylist()))
83
+ for vocabulary in cast(_TFIDFState, state).vocabularies
84
+ )
85
+ for state in self._states
86
+ )
87
+ return (
88
+ vocabularies,
89
+ self._member_state_ids,
90
+ )
91
+
92
+ def set_extra_state(self, state: object) -> None:
93
+ r""":meta private:""" # noqa: D415
94
+ vocabularies, member_state_ids = cast(
95
+ tuple[
96
+ tuple[tuple[tuple[str, ...], ...], ...],
97
+ tuple[int, ...],
98
+ ],
99
+ state,
100
+ )
101
+ self._states = torch.nn.ModuleList(
102
+ [
103
+ _TFIDFState(
104
+ [
105
+ pa.array(vocabulary, type=pa.large_string())
106
+ for vocabulary in state_vocabularies
107
+ ],
108
+ [
109
+ self._idf_template.new_empty(len(vocabulary))
110
+ for vocabulary in state_vocabularies
111
+ ],
112
+ )
113
+ for state_vocabularies in vocabularies
114
+ ]
115
+ )
116
+ self._member_state_ids = member_state_ids
117
+
118
+ def _character_ngrams(
119
+ self,
120
+ tensor: StringTensor,
121
+ ngram_range: tuple[int, int],
122
+ *,
123
+ lowercase: bool = True,
124
+ ) -> tuple[StringTensor, Tensor]:
125
+ if tensor.dim() != 1:
126
+ raise NotImplementedError(
127
+ "Expected tensor to be one-dimensional "
128
+ f"(got {tensor.dim()}D tensor)"
129
+ )
130
+
131
+ if tensor.is_cuda:
132
+ return self._character_ngrams_cuda(tensor, ngram_range, lowercase)
133
+
134
+ min_n, max_n = ngram_range
135
+ whitespace = re.compile(r"\s\s+")
136
+ ngrams: list[str] = []
137
+ offsets: list[int] = [0]
138
+ for document in tensor.to_arrow().to_pylist():
139
+ if document is None:
140
+ offsets.append(len(ngrams))
141
+ continue
142
+ if lowercase:
143
+ document = document.lower()
144
+ document = whitespace.sub(" ", document)
145
+ for word in document.split():
146
+ word = " " + word + " "
147
+ word_len = len(word)
148
+ for n in range(min_n, max_n + 1):
149
+ offset = 0
150
+ ngrams.append(word[offset : offset + n])
151
+ while offset + n < word_len:
152
+ offset += 1
153
+ ngrams.append(word[offset : offset + n])
154
+ if offset == 0: # word shorter than n: count it once
155
+ break
156
+ offsets.append(len(ngrams))
157
+
158
+ flat = tensor.from_arrow(
159
+ pa.array(ngrams, type=pa.large_string()),
160
+ device=tensor.device,
161
+ )
162
+ offset = torch.tensor(offsets, dtype=torch.int64, device=tensor.device)
163
+ return flat, offset
164
+
165
+ def _character_ngrams_cuda(
166
+ self,
167
+ tensor: StringTensor,
168
+ ngram_range: tuple[int, int],
169
+ lowercase: bool = True,
170
+ ) -> tuple[StringTensor, Tensor]:
171
+ import cudf
172
+ import cupy as cp
173
+
174
+ min_n, max_n = ngram_range
175
+ n_docs = tensor.numel()
176
+ s = tensor.to_cudf() # n_docs documents
177
+ if tensor.is_nullable:
178
+ s = s.fillna("")
179
+ if lowercase:
180
+ s = s.str.lower()
181
+ # Collapse every whitespace run to a single space and trim, so each
182
+ # document splits into words on single spaces (matches str.split()).
183
+ s = s.str.replace(r"\s+", " ", regex=True).str.strip()
184
+
185
+ words = s.str.split(" ")
186
+ doc_index = cudf.Series(
187
+ cp.repeat(cp.arange(n_docs), words.list.len().to_cupy())
188
+ )
189
+ flat_words = words.explode().reset_index(drop=True)
190
+ keep = flat_words.str.len() > 0
191
+ flat_words = flat_words[keep].reset_index(drop=True)
192
+ doc_index = doc_index[keep].reset_index(drop=True)
193
+ padded = " " + flat_words + " "
194
+ pad_len = padded.str.len()
195
+
196
+ parts: list[cudf.DataFrame] = []
197
+ # 'character_ngrams' raises when no word is long enough for 'n'.
198
+ for n in range(min_n, max_n + 1):
199
+ eligible = pad_len >= n
200
+ padded_eligable = padded[eligible]
201
+ if len(padded_eligable) == 0:
202
+ continue
203
+ grams = padded_eligable.str.character_ngrams(n, as_list=True)
204
+ long = cudf.DataFrame(
205
+ {
206
+ "doc": doc_index[eligible],
207
+ "gram": grams,
208
+ }
209
+ ).explode("gram")
210
+ parts.append(long.dropna(subset=["gram"]))
211
+
212
+ # word shorter than min_n: count it once
213
+ short = pad_len < min_n
214
+ parts.append(
215
+ cudf.DataFrame({"doc": doc_index[short], "gram": padded[short]})
216
+ )
217
+
218
+ flat = cudf.concat(parts, ignore_index=True).sort_values("doc")
219
+
220
+ counts = (
221
+ flat.groupby("doc").size().reindex(range(n_docs), fill_value=0)
222
+ )
223
+ offset = torch.zeros(
224
+ n_docs + 1,
225
+ dtype=torch.int64,
226
+ device=tensor.device,
227
+ )
228
+ offset[1:] = from_dlpack(counts.cumsum().astype("int64").to_dlpack())
229
+
230
+ ngrams = tensor.from_cudf(
231
+ flat["gram"].reset_index(drop=True), device=tensor.device
232
+ )
233
+ return ngrams, offset
234
+
235
+ def _learn_state(
236
+ self,
237
+ table: TableTensor,
238
+ ) -> _TFIDFState:
239
+ device = table.text.device
240
+ vocabularies: list[pa.Array] = []
241
+ idfs: list[Tensor] = []
242
+
243
+ for column in range(table.text.size(-1)):
244
+ column_text = cast(
245
+ StringTensor,
246
+ table.text[..., column].reshape(-1),
247
+ )
248
+ flat, offsets = self._character_ngrams(
249
+ column_text,
250
+ self.ngram_range,
251
+ lowercase=self.lowercase,
252
+ )
253
+ n_docs = offsets.numel() - 1
254
+
255
+ # Factorize n-grams into codes + the unique vocabulary
256
+ if flat.is_cuda:
257
+ import cudf
258
+
259
+ values = flat.to_cudf()
260
+ vocabulary = values.drop_duplicates(
261
+ ignore_index=True
262
+ ).to_arrow()
263
+ encoded = values.astype(
264
+ cudf.CategoricalDtype(categories=vocabulary)
265
+ )
266
+ codes = torch.from_dlpack(
267
+ encoded.cat.codes.astype("int64").to_dlpack()
268
+ ).to(device)
269
+ else:
270
+ encoded = flat.to_arrow().dictionary_encode()
271
+ vocabulary = encoded.dictionary
272
+ codes = arrow_as_tensor(
273
+ encoded.indices, dtype=torch.int64, device=device
274
+ ) # [n_ngrams]
275
+ vocab_size = len(vocabulary)
276
+ # Map each n-gram back to its document via the offsets.
277
+ doc_ids = torch.repeat_interleave(
278
+ torch.arange(n_docs, device=device),
279
+ offsets.diff(),
280
+ ) # [n_ngrams]
281
+
282
+ # Smoothed idf per n-gram from its document frequency: count each
283
+ # n-gram once per document, then apply sklearn's smoothing.
284
+ if vocab_size == 0:
285
+ idf = torch.empty(0, device=device)
286
+ else:
287
+ unique_codes = (
288
+ doc_ids * vocab_size + codes
289
+ ).unique() % vocab_size
290
+ document_freq = torch.bincount(
291
+ unique_codes, minlength=vocab_size
292
+ )
293
+ idf = ((1 + n_docs) / (1 + document_freq)).log() + 1.0
294
+ term_counts = torch.bincount(
295
+ codes, minlength=vocab_size
296
+ ) # [vocab_size]
297
+
298
+ # Keep the max_features n-grams with the highest term frequency,
299
+ # matching scikit-learn: rank by corpus occurrence, break ties by
300
+ # vocabulary order via a stable sort.
301
+ if (
302
+ self.max_features is not None
303
+ and len(vocabulary) > self.max_features
304
+ ):
305
+ ranked = term_counts.argsort(descending=True, stable=True)
306
+ keep = ranked[: self.max_features].sort().values
307
+ vocabulary = vocabulary.take(pa.array(keep.tolist()))
308
+ idf = idf[keep]
309
+ vocabularies.append(vocabulary)
310
+ idfs.append(idf)
311
+
312
+ if idfs:
313
+ self._idf_template = idfs[0].new_empty(0)
314
+ return _TFIDFState(vocabularies, idfs)
315
+
316
+ def _fit_ensemble(
317
+ self,
318
+ ensemble_table: EnsembleTable,
319
+ *,
320
+ generator: torch.Generator | None = None,
321
+ ) -> None:
322
+ states = torch.nn.ModuleList()
323
+ member_state_ids: list[int] = []
324
+ state_ids: dict[tuple[int, int], int] = {}
325
+
326
+ # TODO: Replace direct `_locations` access with a public
327
+ # `EnsembleTable` iterator over stored tables and their logical member
328
+ # IDs, then use the same abstraction when transforming.
329
+ for member_id in range(len(ensemble_table)):
330
+ location = ensemble_table._locations[member_id]
331
+ state_id = state_ids.get(location)
332
+ if state_id is None:
333
+ state_id = len(states)
334
+ state_ids[location] = state_id
335
+ states.append(self._learn_state(ensemble_table[member_id]))
336
+ member_state_ids.append(state_id)
337
+
338
+ self._states = states
339
+ self._member_state_ids = tuple(member_state_ids)
340
+
341
+ def _transform_ensemble(
342
+ self,
343
+ ensemble_table: EnsembleTable,
344
+ ) -> EnsembleTable:
345
+ if len(self._member_state_ids) != len(ensemble_table):
346
+ raise RuntimeError(
347
+ "TFIDF must be fitted with the same number of "
348
+ "ensemble members before transform."
349
+ )
350
+
351
+ output_tables: list[TableTensor] = []
352
+ member_table_ids: list[int] = []
353
+ transformed: dict[tuple[tuple[int, int], int], int] = {}
354
+ # TODO: Benchmark batching compatible query tables that share fitted
355
+ # state on CUDA with cuDF. It was about 10% slower for four 50,000-row
356
+ # tables on CPU.
357
+ for member_id, state_id in enumerate(self._member_state_ids):
358
+ key = (ensemble_table._locations[member_id], state_id)
359
+ table_id = transformed.get(key)
360
+ if table_id is None:
361
+ state = cast(_TFIDFState, self._states[state_id])
362
+ table_id = len(output_tables)
363
+ transformed[key] = table_id
364
+ output_tables.append(
365
+ self._encode(
366
+ ensemble_table[member_id],
367
+ state,
368
+ )
369
+ )
370
+ member_table_ids.append(table_id)
371
+
372
+ return ensemble_table.replace_tables(
373
+ output_tables,
374
+ member_table_ids,
375
+ )
376
+
377
+ def _encode(
378
+ self,
379
+ table: TableTensor,
380
+ state: _TFIDFState,
381
+ ) -> TableTensor:
382
+ device = table.text.device
383
+ dtype = (
384
+ state.get_buffer("idf_0").dtype
385
+ if state.vocabularies
386
+ else torch.get_default_dtype()
387
+ )
388
+ text_names = table.columns[Stype.text]
389
+ leading_shape = table.text.shape[:-1]
390
+ n_rows = math.prod(leading_shape)
391
+
392
+ vocab_sizes = [len(vocabulary) for vocabulary in state.vocabularies]
393
+ column_offsets = [0, *accumulate(vocab_sizes)]
394
+ total_width = column_offsets[-1]
395
+ numerical = torch.zeros(
396
+ (*leading_shape, total_width),
397
+ dtype=dtype,
398
+ device=device,
399
+ )
400
+ flat_numerical = numerical.view(n_rows, total_width)
401
+ names: list[str] = []
402
+ for column in range(table.text.size(-1)):
403
+ vocabulary = state.vocabularies[column]
404
+ idf = getattr(state, f"idf_{column}")
405
+ vocab_size = vocab_sizes[column]
406
+ column_start = column_offsets[column]
407
+ column_slice = flat_numerical[
408
+ :, column_start : column_offsets[column + 1]
409
+ ]
410
+
411
+ if vocab_size > 0:
412
+ column_text = cast(
413
+ StringTensor,
414
+ table.text[..., column].reshape(-1),
415
+ )
416
+ flat, offsets = self._character_ngrams(
417
+ column_text,
418
+ self.ngram_range,
419
+ lowercase=self.lowercase,
420
+ )
421
+ # Map n-grams to fitted vocab indices; unseen -> -1 (dropped).
422
+ if flat.is_cuda:
423
+ import cudf
424
+
425
+ encoded = (
426
+ flat.to_cudf()
427
+ .astype(cudf.CategoricalDtype(categories=vocabulary))
428
+ .cat.codes
429
+ )
430
+ codes = torch.from_dlpack(
431
+ encoded.astype("int64").to_cupy(na_value=-1)
432
+ ).to(device)
433
+ else:
434
+ codes = arrow_as_tensor(
435
+ pc.call_function(
436
+ "index_in",
437
+ [flat.to_arrow()],
438
+ options=pc.SetLookupOptions(value_set=vocabulary),
439
+ ).fill_null(-1),
440
+ dtype=torch.int64,
441
+ device=device,
442
+ ) # [n_ngrams]
443
+ doc_ids = torch.repeat_interleave(
444
+ torch.arange(n_rows, device=device),
445
+ offsets.diff(),
446
+ ) # [n_ngrams]
447
+ mask = (codes >= 0) & (codes < vocab_size)
448
+ flat_index = doc_ids[mask] * vocab_size + codes[mask]
449
+ counts = column_slice.new_zeros(n_rows, vocab_size)
450
+ counts.view(-1).scatter_add_(
451
+ 0,
452
+ flat_index,
453
+ torch.ones_like(flat_index, dtype=dtype),
454
+ )
455
+ counts.mul_(idf)
456
+ norm = counts.norm(dim=1, keepdim=True).clamp_min_(1e-12)
457
+ column_slice.copy_(counts.div_(norm))
458
+ names.extend(
459
+ f"{text_names[column]}_{i}" for i in range(vocab_size)
460
+ )
461
+
462
+ out = table.__class__(
463
+ columns={Stype.numerical: tuple(names)},
464
+ numerical=numerical,
465
+ )
466
+ remainder = table.drop_stypes(Stype.text)
467
+ if remainder.size(-1) == 0:
468
+ return out
469
+ return cast(TableTensor, torch.cat((remainder, out), dim=-1))
470
+
471
+ def __repr__(self, *, indent: int = 0) -> str:
472
+ return (
473
+ f"{' ' * indent}{self.__class__.__name__}("
474
+ f"ngram_range={self.ngram_range}, "
475
+ f"max_features={self.max_features}, "
476
+ f"lowercase={self.lowercase}"
477
+ ")"
478
+ )
@@ -0,0 +1,16 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Relational data processing."""
5
+
6
+ from sdm.relational.data import Relationship, RelationalData
7
+ from sdm.relational.task import TaskLink, RelatedTables
8
+ from sdm.relational.sampler import RelationalSampler
9
+
10
+ __all__ = [
11
+ "Relationship",
12
+ "RelationalData",
13
+ "TaskLink",
14
+ "RelatedTables",
15
+ "RelationalSampler",
16
+ ]
@@ -0,0 +1,12 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Backend implementations for relational data processing."""
5
+
6
+ from sdm.relational.backend._pyg_lib import PyGLibRelationalSampler
7
+ from sdm.relational.backend._cugraph import CuGraphRelationalSampler
8
+
9
+ __all__ = [
10
+ "PyGLibRelationalSampler",
11
+ "CuGraphRelationalSampler",
12
+ ]