structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,478 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import math
|
|
5
|
+
import re
|
|
6
|
+
from itertools import accumulate
|
|
7
|
+
from typing import cast
|
|
8
|
+
|
|
9
|
+
import pyarrow as pa
|
|
10
|
+
import pyarrow.compute as pc
|
|
11
|
+
import torch
|
|
12
|
+
from torch import Tensor
|
|
13
|
+
from torch.utils.dlpack import from_dlpack
|
|
14
|
+
|
|
15
|
+
from sdm import EnsembleTable, StringTensor, Stype, TableTensor
|
|
16
|
+
from sdm.processing import EnsembleProcessor
|
|
17
|
+
from sdm.tensor.io import arrow_as_tensor
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class _TFIDFState(torch.nn.Module):
|
|
21
|
+
def __init__(
|
|
22
|
+
self,
|
|
23
|
+
vocabularies: list[pa.Array],
|
|
24
|
+
idfs: list[Tensor],
|
|
25
|
+
) -> None:
|
|
26
|
+
super().__init__()
|
|
27
|
+
self.vocabularies = vocabularies
|
|
28
|
+
for column, idf in enumerate(idfs):
|
|
29
|
+
self.register_buffer(f"idf_{column}", idf)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class TFIDF(EnsembleProcessor):
|
|
33
|
+
"""Encode text columns as character n-gram TF-IDF vectors.
|
|
34
|
+
|
|
35
|
+
Tokenization follows scikit-learn's ``char_wb`` analyzer: whitespace-
|
|
36
|
+
delimited words are space-padded before windowing, so a word shorter than
|
|
37
|
+
``n`` still yields one n-gram. Fit learns a vocabulary and smoothed idf
|
|
38
|
+
weights per text column. Transform replaces text with concatenated
|
|
39
|
+
numerical features (one per retained n-gram), applies those idf weights,
|
|
40
|
+
L2-normalizes each row, and ignores n-grams unseen at fit time.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
ngram_range: Inclusive ``(min_n, max_n)`` character-window sizes.
|
|
44
|
+
max_features: If set, keep only this many most frequent n-grams per
|
|
45
|
+
column. ``None`` keeps the full vocabulary.
|
|
46
|
+
lowercase: If ``True``, lowercase text before tokenizing.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
handles_stypes = frozenset({Stype.text})
|
|
50
|
+
requires_fit = True
|
|
51
|
+
|
|
52
|
+
def __init__(
|
|
53
|
+
self,
|
|
54
|
+
*,
|
|
55
|
+
ngram_range: tuple[int, int],
|
|
56
|
+
max_features: int | None = None,
|
|
57
|
+
lowercase: bool = True,
|
|
58
|
+
) -> None:
|
|
59
|
+
super().__init__()
|
|
60
|
+
min_n, max_n = ngram_range
|
|
61
|
+
if min_n < 1 or max_n < min_n:
|
|
62
|
+
raise ValueError("ngram_range must satisfy 1 <= min_n <= max_n.")
|
|
63
|
+
if max_features is not None and max_features <= 0:
|
|
64
|
+
raise ValueError("max_features must be positive or None.")
|
|
65
|
+
self.ngram_range = ngram_range
|
|
66
|
+
self.max_features = max_features
|
|
67
|
+
self.lowercase = lowercase
|
|
68
|
+
self.register_buffer(
|
|
69
|
+
"_idf_template",
|
|
70
|
+
torch.empty(0),
|
|
71
|
+
persistent=False,
|
|
72
|
+
)
|
|
73
|
+
self._states = torch.nn.ModuleList()
|
|
74
|
+
self._member_state_ids: tuple[int, ...] = ()
|
|
75
|
+
|
|
76
|
+
def get_extra_state(
|
|
77
|
+
self,
|
|
78
|
+
) -> tuple[tuple[tuple[tuple[str, ...], ...], ...], tuple[int, ...]]:
|
|
79
|
+
r""":meta private:""" # noqa: D415
|
|
80
|
+
vocabularies = tuple(
|
|
81
|
+
tuple(
|
|
82
|
+
tuple(cast(list[str], vocabulary.to_pylist()))
|
|
83
|
+
for vocabulary in cast(_TFIDFState, state).vocabularies
|
|
84
|
+
)
|
|
85
|
+
for state in self._states
|
|
86
|
+
)
|
|
87
|
+
return (
|
|
88
|
+
vocabularies,
|
|
89
|
+
self._member_state_ids,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
def set_extra_state(self, state: object) -> None:
|
|
93
|
+
r""":meta private:""" # noqa: D415
|
|
94
|
+
vocabularies, member_state_ids = cast(
|
|
95
|
+
tuple[
|
|
96
|
+
tuple[tuple[tuple[str, ...], ...], ...],
|
|
97
|
+
tuple[int, ...],
|
|
98
|
+
],
|
|
99
|
+
state,
|
|
100
|
+
)
|
|
101
|
+
self._states = torch.nn.ModuleList(
|
|
102
|
+
[
|
|
103
|
+
_TFIDFState(
|
|
104
|
+
[
|
|
105
|
+
pa.array(vocabulary, type=pa.large_string())
|
|
106
|
+
for vocabulary in state_vocabularies
|
|
107
|
+
],
|
|
108
|
+
[
|
|
109
|
+
self._idf_template.new_empty(len(vocabulary))
|
|
110
|
+
for vocabulary in state_vocabularies
|
|
111
|
+
],
|
|
112
|
+
)
|
|
113
|
+
for state_vocabularies in vocabularies
|
|
114
|
+
]
|
|
115
|
+
)
|
|
116
|
+
self._member_state_ids = member_state_ids
|
|
117
|
+
|
|
118
|
+
def _character_ngrams(
|
|
119
|
+
self,
|
|
120
|
+
tensor: StringTensor,
|
|
121
|
+
ngram_range: tuple[int, int],
|
|
122
|
+
*,
|
|
123
|
+
lowercase: bool = True,
|
|
124
|
+
) -> tuple[StringTensor, Tensor]:
|
|
125
|
+
if tensor.dim() != 1:
|
|
126
|
+
raise NotImplementedError(
|
|
127
|
+
"Expected tensor to be one-dimensional "
|
|
128
|
+
f"(got {tensor.dim()}D tensor)"
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
if tensor.is_cuda:
|
|
132
|
+
return self._character_ngrams_cuda(tensor, ngram_range, lowercase)
|
|
133
|
+
|
|
134
|
+
min_n, max_n = ngram_range
|
|
135
|
+
whitespace = re.compile(r"\s\s+")
|
|
136
|
+
ngrams: list[str] = []
|
|
137
|
+
offsets: list[int] = [0]
|
|
138
|
+
for document in tensor.to_arrow().to_pylist():
|
|
139
|
+
if document is None:
|
|
140
|
+
offsets.append(len(ngrams))
|
|
141
|
+
continue
|
|
142
|
+
if lowercase:
|
|
143
|
+
document = document.lower()
|
|
144
|
+
document = whitespace.sub(" ", document)
|
|
145
|
+
for word in document.split():
|
|
146
|
+
word = " " + word + " "
|
|
147
|
+
word_len = len(word)
|
|
148
|
+
for n in range(min_n, max_n + 1):
|
|
149
|
+
offset = 0
|
|
150
|
+
ngrams.append(word[offset : offset + n])
|
|
151
|
+
while offset + n < word_len:
|
|
152
|
+
offset += 1
|
|
153
|
+
ngrams.append(word[offset : offset + n])
|
|
154
|
+
if offset == 0: # word shorter than n: count it once
|
|
155
|
+
break
|
|
156
|
+
offsets.append(len(ngrams))
|
|
157
|
+
|
|
158
|
+
flat = tensor.from_arrow(
|
|
159
|
+
pa.array(ngrams, type=pa.large_string()),
|
|
160
|
+
device=tensor.device,
|
|
161
|
+
)
|
|
162
|
+
offset = torch.tensor(offsets, dtype=torch.int64, device=tensor.device)
|
|
163
|
+
return flat, offset
|
|
164
|
+
|
|
165
|
+
def _character_ngrams_cuda(
|
|
166
|
+
self,
|
|
167
|
+
tensor: StringTensor,
|
|
168
|
+
ngram_range: tuple[int, int],
|
|
169
|
+
lowercase: bool = True,
|
|
170
|
+
) -> tuple[StringTensor, Tensor]:
|
|
171
|
+
import cudf
|
|
172
|
+
import cupy as cp
|
|
173
|
+
|
|
174
|
+
min_n, max_n = ngram_range
|
|
175
|
+
n_docs = tensor.numel()
|
|
176
|
+
s = tensor.to_cudf() # n_docs documents
|
|
177
|
+
if tensor.is_nullable:
|
|
178
|
+
s = s.fillna("")
|
|
179
|
+
if lowercase:
|
|
180
|
+
s = s.str.lower()
|
|
181
|
+
# Collapse every whitespace run to a single space and trim, so each
|
|
182
|
+
# document splits into words on single spaces (matches str.split()).
|
|
183
|
+
s = s.str.replace(r"\s+", " ", regex=True).str.strip()
|
|
184
|
+
|
|
185
|
+
words = s.str.split(" ")
|
|
186
|
+
doc_index = cudf.Series(
|
|
187
|
+
cp.repeat(cp.arange(n_docs), words.list.len().to_cupy())
|
|
188
|
+
)
|
|
189
|
+
flat_words = words.explode().reset_index(drop=True)
|
|
190
|
+
keep = flat_words.str.len() > 0
|
|
191
|
+
flat_words = flat_words[keep].reset_index(drop=True)
|
|
192
|
+
doc_index = doc_index[keep].reset_index(drop=True)
|
|
193
|
+
padded = " " + flat_words + " "
|
|
194
|
+
pad_len = padded.str.len()
|
|
195
|
+
|
|
196
|
+
parts: list[cudf.DataFrame] = []
|
|
197
|
+
# 'character_ngrams' raises when no word is long enough for 'n'.
|
|
198
|
+
for n in range(min_n, max_n + 1):
|
|
199
|
+
eligible = pad_len >= n
|
|
200
|
+
padded_eligable = padded[eligible]
|
|
201
|
+
if len(padded_eligable) == 0:
|
|
202
|
+
continue
|
|
203
|
+
grams = padded_eligable.str.character_ngrams(n, as_list=True)
|
|
204
|
+
long = cudf.DataFrame(
|
|
205
|
+
{
|
|
206
|
+
"doc": doc_index[eligible],
|
|
207
|
+
"gram": grams,
|
|
208
|
+
}
|
|
209
|
+
).explode("gram")
|
|
210
|
+
parts.append(long.dropna(subset=["gram"]))
|
|
211
|
+
|
|
212
|
+
# word shorter than min_n: count it once
|
|
213
|
+
short = pad_len < min_n
|
|
214
|
+
parts.append(
|
|
215
|
+
cudf.DataFrame({"doc": doc_index[short], "gram": padded[short]})
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
flat = cudf.concat(parts, ignore_index=True).sort_values("doc")
|
|
219
|
+
|
|
220
|
+
counts = (
|
|
221
|
+
flat.groupby("doc").size().reindex(range(n_docs), fill_value=0)
|
|
222
|
+
)
|
|
223
|
+
offset = torch.zeros(
|
|
224
|
+
n_docs + 1,
|
|
225
|
+
dtype=torch.int64,
|
|
226
|
+
device=tensor.device,
|
|
227
|
+
)
|
|
228
|
+
offset[1:] = from_dlpack(counts.cumsum().astype("int64").to_dlpack())
|
|
229
|
+
|
|
230
|
+
ngrams = tensor.from_cudf(
|
|
231
|
+
flat["gram"].reset_index(drop=True), device=tensor.device
|
|
232
|
+
)
|
|
233
|
+
return ngrams, offset
|
|
234
|
+
|
|
235
|
+
def _learn_state(
|
|
236
|
+
self,
|
|
237
|
+
table: TableTensor,
|
|
238
|
+
) -> _TFIDFState:
|
|
239
|
+
device = table.text.device
|
|
240
|
+
vocabularies: list[pa.Array] = []
|
|
241
|
+
idfs: list[Tensor] = []
|
|
242
|
+
|
|
243
|
+
for column in range(table.text.size(-1)):
|
|
244
|
+
column_text = cast(
|
|
245
|
+
StringTensor,
|
|
246
|
+
table.text[..., column].reshape(-1),
|
|
247
|
+
)
|
|
248
|
+
flat, offsets = self._character_ngrams(
|
|
249
|
+
column_text,
|
|
250
|
+
self.ngram_range,
|
|
251
|
+
lowercase=self.lowercase,
|
|
252
|
+
)
|
|
253
|
+
n_docs = offsets.numel() - 1
|
|
254
|
+
|
|
255
|
+
# Factorize n-grams into codes + the unique vocabulary
|
|
256
|
+
if flat.is_cuda:
|
|
257
|
+
import cudf
|
|
258
|
+
|
|
259
|
+
values = flat.to_cudf()
|
|
260
|
+
vocabulary = values.drop_duplicates(
|
|
261
|
+
ignore_index=True
|
|
262
|
+
).to_arrow()
|
|
263
|
+
encoded = values.astype(
|
|
264
|
+
cudf.CategoricalDtype(categories=vocabulary)
|
|
265
|
+
)
|
|
266
|
+
codes = torch.from_dlpack(
|
|
267
|
+
encoded.cat.codes.astype("int64").to_dlpack()
|
|
268
|
+
).to(device)
|
|
269
|
+
else:
|
|
270
|
+
encoded = flat.to_arrow().dictionary_encode()
|
|
271
|
+
vocabulary = encoded.dictionary
|
|
272
|
+
codes = arrow_as_tensor(
|
|
273
|
+
encoded.indices, dtype=torch.int64, device=device
|
|
274
|
+
) # [n_ngrams]
|
|
275
|
+
vocab_size = len(vocabulary)
|
|
276
|
+
# Map each n-gram back to its document via the offsets.
|
|
277
|
+
doc_ids = torch.repeat_interleave(
|
|
278
|
+
torch.arange(n_docs, device=device),
|
|
279
|
+
offsets.diff(),
|
|
280
|
+
) # [n_ngrams]
|
|
281
|
+
|
|
282
|
+
# Smoothed idf per n-gram from its document frequency: count each
|
|
283
|
+
# n-gram once per document, then apply sklearn's smoothing.
|
|
284
|
+
if vocab_size == 0:
|
|
285
|
+
idf = torch.empty(0, device=device)
|
|
286
|
+
else:
|
|
287
|
+
unique_codes = (
|
|
288
|
+
doc_ids * vocab_size + codes
|
|
289
|
+
).unique() % vocab_size
|
|
290
|
+
document_freq = torch.bincount(
|
|
291
|
+
unique_codes, minlength=vocab_size
|
|
292
|
+
)
|
|
293
|
+
idf = ((1 + n_docs) / (1 + document_freq)).log() + 1.0
|
|
294
|
+
term_counts = torch.bincount(
|
|
295
|
+
codes, minlength=vocab_size
|
|
296
|
+
) # [vocab_size]
|
|
297
|
+
|
|
298
|
+
# Keep the max_features n-grams with the highest term frequency,
|
|
299
|
+
# matching scikit-learn: rank by corpus occurrence, break ties by
|
|
300
|
+
# vocabulary order via a stable sort.
|
|
301
|
+
if (
|
|
302
|
+
self.max_features is not None
|
|
303
|
+
and len(vocabulary) > self.max_features
|
|
304
|
+
):
|
|
305
|
+
ranked = term_counts.argsort(descending=True, stable=True)
|
|
306
|
+
keep = ranked[: self.max_features].sort().values
|
|
307
|
+
vocabulary = vocabulary.take(pa.array(keep.tolist()))
|
|
308
|
+
idf = idf[keep]
|
|
309
|
+
vocabularies.append(vocabulary)
|
|
310
|
+
idfs.append(idf)
|
|
311
|
+
|
|
312
|
+
if idfs:
|
|
313
|
+
self._idf_template = idfs[0].new_empty(0)
|
|
314
|
+
return _TFIDFState(vocabularies, idfs)
|
|
315
|
+
|
|
316
|
+
def _fit_ensemble(
|
|
317
|
+
self,
|
|
318
|
+
ensemble_table: EnsembleTable,
|
|
319
|
+
*,
|
|
320
|
+
generator: torch.Generator | None = None,
|
|
321
|
+
) -> None:
|
|
322
|
+
states = torch.nn.ModuleList()
|
|
323
|
+
member_state_ids: list[int] = []
|
|
324
|
+
state_ids: dict[tuple[int, int], int] = {}
|
|
325
|
+
|
|
326
|
+
# TODO: Replace direct `_locations` access with a public
|
|
327
|
+
# `EnsembleTable` iterator over stored tables and their logical member
|
|
328
|
+
# IDs, then use the same abstraction when transforming.
|
|
329
|
+
for member_id in range(len(ensemble_table)):
|
|
330
|
+
location = ensemble_table._locations[member_id]
|
|
331
|
+
state_id = state_ids.get(location)
|
|
332
|
+
if state_id is None:
|
|
333
|
+
state_id = len(states)
|
|
334
|
+
state_ids[location] = state_id
|
|
335
|
+
states.append(self._learn_state(ensemble_table[member_id]))
|
|
336
|
+
member_state_ids.append(state_id)
|
|
337
|
+
|
|
338
|
+
self._states = states
|
|
339
|
+
self._member_state_ids = tuple(member_state_ids)
|
|
340
|
+
|
|
341
|
+
def _transform_ensemble(
|
|
342
|
+
self,
|
|
343
|
+
ensemble_table: EnsembleTable,
|
|
344
|
+
) -> EnsembleTable:
|
|
345
|
+
if len(self._member_state_ids) != len(ensemble_table):
|
|
346
|
+
raise RuntimeError(
|
|
347
|
+
"TFIDF must be fitted with the same number of "
|
|
348
|
+
"ensemble members before transform."
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
output_tables: list[TableTensor] = []
|
|
352
|
+
member_table_ids: list[int] = []
|
|
353
|
+
transformed: dict[tuple[tuple[int, int], int], int] = {}
|
|
354
|
+
# TODO: Benchmark batching compatible query tables that share fitted
|
|
355
|
+
# state on CUDA with cuDF. It was about 10% slower for four 50,000-row
|
|
356
|
+
# tables on CPU.
|
|
357
|
+
for member_id, state_id in enumerate(self._member_state_ids):
|
|
358
|
+
key = (ensemble_table._locations[member_id], state_id)
|
|
359
|
+
table_id = transformed.get(key)
|
|
360
|
+
if table_id is None:
|
|
361
|
+
state = cast(_TFIDFState, self._states[state_id])
|
|
362
|
+
table_id = len(output_tables)
|
|
363
|
+
transformed[key] = table_id
|
|
364
|
+
output_tables.append(
|
|
365
|
+
self._encode(
|
|
366
|
+
ensemble_table[member_id],
|
|
367
|
+
state,
|
|
368
|
+
)
|
|
369
|
+
)
|
|
370
|
+
member_table_ids.append(table_id)
|
|
371
|
+
|
|
372
|
+
return ensemble_table.replace_tables(
|
|
373
|
+
output_tables,
|
|
374
|
+
member_table_ids,
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
def _encode(
|
|
378
|
+
self,
|
|
379
|
+
table: TableTensor,
|
|
380
|
+
state: _TFIDFState,
|
|
381
|
+
) -> TableTensor:
|
|
382
|
+
device = table.text.device
|
|
383
|
+
dtype = (
|
|
384
|
+
state.get_buffer("idf_0").dtype
|
|
385
|
+
if state.vocabularies
|
|
386
|
+
else torch.get_default_dtype()
|
|
387
|
+
)
|
|
388
|
+
text_names = table.columns[Stype.text]
|
|
389
|
+
leading_shape = table.text.shape[:-1]
|
|
390
|
+
n_rows = math.prod(leading_shape)
|
|
391
|
+
|
|
392
|
+
vocab_sizes = [len(vocabulary) for vocabulary in state.vocabularies]
|
|
393
|
+
column_offsets = [0, *accumulate(vocab_sizes)]
|
|
394
|
+
total_width = column_offsets[-1]
|
|
395
|
+
numerical = torch.zeros(
|
|
396
|
+
(*leading_shape, total_width),
|
|
397
|
+
dtype=dtype,
|
|
398
|
+
device=device,
|
|
399
|
+
)
|
|
400
|
+
flat_numerical = numerical.view(n_rows, total_width)
|
|
401
|
+
names: list[str] = []
|
|
402
|
+
for column in range(table.text.size(-1)):
|
|
403
|
+
vocabulary = state.vocabularies[column]
|
|
404
|
+
idf = getattr(state, f"idf_{column}")
|
|
405
|
+
vocab_size = vocab_sizes[column]
|
|
406
|
+
column_start = column_offsets[column]
|
|
407
|
+
column_slice = flat_numerical[
|
|
408
|
+
:, column_start : column_offsets[column + 1]
|
|
409
|
+
]
|
|
410
|
+
|
|
411
|
+
if vocab_size > 0:
|
|
412
|
+
column_text = cast(
|
|
413
|
+
StringTensor,
|
|
414
|
+
table.text[..., column].reshape(-1),
|
|
415
|
+
)
|
|
416
|
+
flat, offsets = self._character_ngrams(
|
|
417
|
+
column_text,
|
|
418
|
+
self.ngram_range,
|
|
419
|
+
lowercase=self.lowercase,
|
|
420
|
+
)
|
|
421
|
+
# Map n-grams to fitted vocab indices; unseen -> -1 (dropped).
|
|
422
|
+
if flat.is_cuda:
|
|
423
|
+
import cudf
|
|
424
|
+
|
|
425
|
+
encoded = (
|
|
426
|
+
flat.to_cudf()
|
|
427
|
+
.astype(cudf.CategoricalDtype(categories=vocabulary))
|
|
428
|
+
.cat.codes
|
|
429
|
+
)
|
|
430
|
+
codes = torch.from_dlpack(
|
|
431
|
+
encoded.astype("int64").to_cupy(na_value=-1)
|
|
432
|
+
).to(device)
|
|
433
|
+
else:
|
|
434
|
+
codes = arrow_as_tensor(
|
|
435
|
+
pc.call_function(
|
|
436
|
+
"index_in",
|
|
437
|
+
[flat.to_arrow()],
|
|
438
|
+
options=pc.SetLookupOptions(value_set=vocabulary),
|
|
439
|
+
).fill_null(-1),
|
|
440
|
+
dtype=torch.int64,
|
|
441
|
+
device=device,
|
|
442
|
+
) # [n_ngrams]
|
|
443
|
+
doc_ids = torch.repeat_interleave(
|
|
444
|
+
torch.arange(n_rows, device=device),
|
|
445
|
+
offsets.diff(),
|
|
446
|
+
) # [n_ngrams]
|
|
447
|
+
mask = (codes >= 0) & (codes < vocab_size)
|
|
448
|
+
flat_index = doc_ids[mask] * vocab_size + codes[mask]
|
|
449
|
+
counts = column_slice.new_zeros(n_rows, vocab_size)
|
|
450
|
+
counts.view(-1).scatter_add_(
|
|
451
|
+
0,
|
|
452
|
+
flat_index,
|
|
453
|
+
torch.ones_like(flat_index, dtype=dtype),
|
|
454
|
+
)
|
|
455
|
+
counts.mul_(idf)
|
|
456
|
+
norm = counts.norm(dim=1, keepdim=True).clamp_min_(1e-12)
|
|
457
|
+
column_slice.copy_(counts.div_(norm))
|
|
458
|
+
names.extend(
|
|
459
|
+
f"{text_names[column]}_{i}" for i in range(vocab_size)
|
|
460
|
+
)
|
|
461
|
+
|
|
462
|
+
out = table.__class__(
|
|
463
|
+
columns={Stype.numerical: tuple(names)},
|
|
464
|
+
numerical=numerical,
|
|
465
|
+
)
|
|
466
|
+
remainder = table.drop_stypes(Stype.text)
|
|
467
|
+
if remainder.size(-1) == 0:
|
|
468
|
+
return out
|
|
469
|
+
return cast(TableTensor, torch.cat((remainder, out), dim=-1))
|
|
470
|
+
|
|
471
|
+
def __repr__(self, *, indent: int = 0) -> str:
|
|
472
|
+
return (
|
|
473
|
+
f"{' ' * indent}{self.__class__.__name__}("
|
|
474
|
+
f"ngram_range={self.ngram_range}, "
|
|
475
|
+
f"max_features={self.max_features}, "
|
|
476
|
+
f"lowercase={self.lowercase}"
|
|
477
|
+
")"
|
|
478
|
+
)
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Relational data processing."""
|
|
5
|
+
|
|
6
|
+
from sdm.relational.data import Relationship, RelationalData
|
|
7
|
+
from sdm.relational.task import TaskLink, RelatedTables
|
|
8
|
+
from sdm.relational.sampler import RelationalSampler
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"Relationship",
|
|
12
|
+
"RelationalData",
|
|
13
|
+
"TaskLink",
|
|
14
|
+
"RelatedTables",
|
|
15
|
+
"RelationalSampler",
|
|
16
|
+
]
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Backend implementations for relational data processing."""
|
|
5
|
+
|
|
6
|
+
from sdm.relational.backend._pyg_lib import PyGLibRelationalSampler
|
|
7
|
+
from sdm.relational.backend._cugraph import CuGraphRelationalSampler
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"PyGLibRelationalSampler",
|
|
11
|
+
"CuGraphRelationalSampler",
|
|
12
|
+
]
|