structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,16 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from sdm import Stype, TableTensor
5
+ from sdm.processing import Processor
6
+
7
+
8
+ class SortQuantiles(Processor):
9
+ r"""Sort quantile predictions in ascending order."""
10
+
11
+ handles_stypes = frozenset({Stype.numerical})
12
+ requires_fit = False
13
+
14
+ def _transform(self, table: TableTensor) -> TableTensor:
15
+ numerical = table.numerical.sort(dim=-1).values
16
+ return table.replace_blocks(numerical=numerical)
@@ -0,0 +1,154 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from __future__ import annotations
5
+
6
+ from collections.abc import Iterable
7
+ from dataclasses import dataclass
8
+ from typing import Self
9
+
10
+ import sdm.processing as sp
11
+ from sdm.processing import EnsembleProcessor, Processor
12
+
13
+
14
+ @dataclass(init=False, repr=False)
15
+ class Recipe:
16
+ """Processing contract around an external model boundary.
17
+
18
+ A recipe bundles three processing pipelines, one per role the data plays
19
+ relative to the model:
20
+
21
+ - ``features``: model inputs, transformed before the model.
22
+ - ``target``: labels transformed forward before the model. Regression
23
+ predictions are inverted through this pipeline; classification outputs
24
+ are reconstructed from the fitted target categories instead.
25
+ - ``output``: transforms member outputs after they have been mapped to a
26
+ common class or target space and stacked as ``[E, ..., R, O]``. An
27
+ explicit dimension-changing step such as
28
+ :class:`~sdm.processing.AverageEstimators` removes ``E``; without one,
29
+ the output remains stacked. Steps before the reducer must support
30
+ stacked outputs, while steps after it receive already-reduced outputs.
31
+
32
+ Each pipeline exposes ``fit``/``transform``/``fit_transform`` and, when its
33
+ steps are invertible, ``inverse_transform``. Call them directly, e.g.
34
+ ``recipe.features.transform(table)`` or
35
+ ``recipe.target.inverse_transform(prediction)``. Recipes do not infer each
36
+ step's non-finite input contract; order steps so values are imputed before
37
+ processors that do not explicitly document non-finite support.
38
+
39
+ Args:
40
+ features: Steps applied to model inputs before model execution.
41
+ target: Steps applied to labels before model execution.
42
+ output: Steps applied to stacked model outputs.
43
+ """
44
+
45
+ _features: EnsembleProcessor
46
+ _target: EnsembleProcessor
47
+ _output: EnsembleProcessor
48
+
49
+ def __init__(
50
+ self,
51
+ features: Processor | Iterable[Processor] | None = None,
52
+ target: Processor | Iterable[Processor] | None = None,
53
+ output: Processor | Iterable[Processor] | None = None,
54
+ ) -> None:
55
+ self.features = features
56
+ self.target = target
57
+ self.output = output
58
+
59
+ @property
60
+ def features(self) -> EnsembleProcessor:
61
+ r"""The steps applied to model inputs."""
62
+ return self._features
63
+
64
+ @features.setter
65
+ def features(
66
+ self,
67
+ processor: Processor | Iterable[Processor] | None,
68
+ ) -> None:
69
+ if processor is None:
70
+ processor = sp.Identity()
71
+ self._features = EnsembleProcessor.as_processor(processor)
72
+
73
+ @property
74
+ def target(self) -> EnsembleProcessor:
75
+ r"""The steps applied to labels."""
76
+ return self._target
77
+
78
+ @target.setter
79
+ def target(
80
+ self,
81
+ processor: Processor | Iterable[Processor] | None,
82
+ ) -> None:
83
+ if processor is None:
84
+ processor = sp.Identity()
85
+ self._target = EnsembleProcessor.as_processor(processor)
86
+
87
+ if any(isinstance(m, sp.TaskDispatch) for m in self.target.modules()):
88
+ raise ValueError(
89
+ "'TaskDispatch' is not supported in 'Recipe.target'"
90
+ )
91
+ if any(isinstance(m, sp.TableDispatch) for m in self.target.modules()):
92
+ raise ValueError(
93
+ "'TableDispatch' is not supported in 'Recipe.target'"
94
+ )
95
+
96
+ @property
97
+ def output(self) -> EnsembleProcessor:
98
+ r"""The steps applied to model outputs."""
99
+ return self._output
100
+
101
+ @output.setter
102
+ def output(
103
+ self,
104
+ processor: Processor | Iterable[Processor] | None,
105
+ ) -> None:
106
+ if processor is None:
107
+ processor = sp.Identity()
108
+ self._output = EnsembleProcessor.as_processor(processor)
109
+
110
+ if any(isinstance(m, sp.TableDispatch) for m in self.output.modules()):
111
+ raise ValueError(
112
+ "'TableDispatch' is not supported in 'Recipe.output'"
113
+ )
114
+ if self.output.requires_fit:
115
+ raise ValueError("'Recipe.output' should not require fitting")
116
+
117
+ def prepend_features(self, processor: object) -> Self:
118
+ """Prepend a processor to the feature pipeline."""
119
+ self.features = processor + self.features
120
+ return self
121
+
122
+ def append_features(self, processor: object) -> Self:
123
+ """Append a processor to the feature pipeline."""
124
+ self.features = self.features + processor
125
+ return self
126
+
127
+ def prepend_target(self, processor: object) -> Self:
128
+ """Prepend a processor to the target pipeline."""
129
+ self.target = processor + self.target
130
+ return self
131
+
132
+ def append_target(self, processor: object) -> Self:
133
+ """Append a processor to the target pipeline."""
134
+ self.target = self.target + processor
135
+ return self
136
+
137
+ def prepend_output(self, processor: object) -> Self:
138
+ """Prepend a processor to the output pipeline."""
139
+ self.output = processor + self.output
140
+ return self
141
+
142
+ def append_output(self, processor: object) -> Self:
143
+ """Append a processor to the output pipeline."""
144
+ self.output = self.output + processor
145
+ return self
146
+
147
+ def __repr__(self) -> str:
148
+ return (
149
+ f"{self.__class__.__name__}(\n"
150
+ f" features={self.features.__repr__(indent=2)[2:]},\n"
151
+ f" target={self.target.__repr__(indent=2)[2:]},\n"
152
+ f" output={self.output.__repr__(indent=2)[2:]},\n"
153
+ ")"
154
+ )
@@ -0,0 +1,12 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Text processors."""
5
+
6
+ from sdm.processing.text.tfidf import TFIDF
7
+ from sdm.processing.text.sentence_transformer import SentenceTransformer
8
+
9
+ __all__ = [
10
+ "TFIDF",
11
+ "SentenceTransformer",
12
+ ]
@@ -0,0 +1,451 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ # ruff: noqa: D205
5
+
6
+ from __future__ import annotations
7
+
8
+ import importlib.util
9
+ from functools import cached_property
10
+ from typing import TYPE_CHECKING, Any, cast
11
+
12
+ import pyarrow.compute as pc
13
+ import torch
14
+ from torch import Tensor
15
+
16
+ from sdm import StringTensor, Stype, TableTensor
17
+ from sdm._warnings import warn_once
18
+ from sdm.processing import Processor
19
+
20
+ if TYPE_CHECKING:
21
+ import sentence_transformers
22
+ from cudf.core.character_normalizer import CharacterNormalizer
23
+ from cudf.core.wordpiece_tokenize import WordPieceVocabulary
24
+
25
+
26
+ # Hugging Face treats words with more than 100 Unicode characters as [UNK],
27
+ # while cuDF does so for words with 200 or more UTF-8 bytes.
28
+ _CUDF_WORDPIECE_MAX_BYTES = 200
29
+
30
+
31
+ class _CuDFTokenizer:
32
+ def __init__(
33
+ self,
34
+ vocabulary: WordPieceVocabulary,
35
+ normalizer: CharacterNormalizer,
36
+ cls_token_id: int,
37
+ sep_token_id: int,
38
+ pad_token_id: int,
39
+ max_length: int,
40
+ max_input_chars_per_word: int,
41
+ ) -> None:
42
+ self._vocabulary = vocabulary
43
+ self._normalizer = normalizer
44
+ self._cls_token_id = cls_token_id
45
+ self._sep_token_id = sep_token_id
46
+ self._pad_token_id = pad_token_id
47
+ self._max_length = max_length
48
+ self._max_input_chars_per_word = max_input_chars_per_word
49
+
50
+ @staticmethod
51
+ def _is_supported(
52
+ model: sentence_transformers.SentenceTransformer,
53
+ ) -> bool:
54
+ import tokenizers
55
+ from sentence_transformers.sentence_transformer.modules import (
56
+ Transformer,
57
+ )
58
+ from transformers import PreTrainedTokenizerFast
59
+
60
+ module = model[0]
61
+ tokenizer = model.tokenizer
62
+
63
+ # Fall back to CPU tokenization unless this is a standard BERT
64
+ # Transformer module with a fast tokenizer.
65
+ if (
66
+ type(module) is not Transformer
67
+ or module.config.model_type != "bert"
68
+ or not isinstance(tokenizer, PreTrainedTokenizerFast)
69
+ ):
70
+ return False
71
+
72
+ if (
73
+ model.modalities != ["text"]
74
+ or model.default_prompt_name is not None
75
+ or model.truncate_dim is not None
76
+ or module.transformer_task != "feature-extraction"
77
+ or module.backend != "torch"
78
+ or module.processing_kwargs
79
+ or module.can_flatten_inputs
80
+ or module.unpad_inputs
81
+ ):
82
+ return False
83
+
84
+ backend = tokenizer.backend_tokenizer
85
+ wordpiece = backend.model
86
+ normalizer = backend.normalizer
87
+
88
+ # cuDF cannot currently construct an equivalent tokenizer directly
89
+ # from the Hugging Face tokenizer configuration. Keep these checks
90
+ # until it can.
91
+ # Require the standard BERT WordPiece pipeline.
92
+ if (
93
+ type(wordpiece) is not tokenizers.models.WordPiece
94
+ or type(normalizer) is not tokenizers.normalizers.BertNormalizer
95
+ or type(backend.pre_tokenizer)
96
+ is not tokenizers.pre_tokenizers.BertPreTokenizer
97
+ or type(backend.post_processor)
98
+ is not tokenizers.processors.TemplateProcessing
99
+ or (
100
+ normalizer.clean_text,
101
+ normalizer.handle_chinese_chars,
102
+ normalizer.strip_accents,
103
+ )
104
+ != (True, True, None)
105
+ or (wordpiece.unk_token, wordpiece.continuing_subword_prefix)
106
+ != ("[UNK]", "##")
107
+ or (tokenizer.padding_side, tokenizer.truncation_side)
108
+ != ("right", "right")
109
+ or tokenizer.model_max_length != module.max_seq_length
110
+ ):
111
+ return False
112
+
113
+ vocab = backend.get_vocab(with_added_tokens=False)
114
+ added_tokens = tuple(backend.get_added_tokens_decoder().values())
115
+ special_tokens = (
116
+ (tokenizer.cls_token, tokenizer.cls_token_id),
117
+ (tokenizer.sep_token, tokenizer.sep_token_id),
118
+ (tokenizer.pad_token, tokenizer.pad_token_id),
119
+ (tokenizer.unk_token, tokenizer.unk_token_id),
120
+ )
121
+
122
+ # Vocabulary IDs and added-token behavior must match cuDF's lookup.
123
+ if (
124
+ backend.get_vocab_size(with_added_tokens=True) != len(vocab)
125
+ or set(vocab.values()) != set(range(len(vocab)))
126
+ or not all(
127
+ vocab.get(token) == token_id
128
+ for token, token_id in special_tokens
129
+ )
130
+ or {token.content for token in added_tokens}
131
+ != set(tokenizer.all_special_tokens)
132
+ or not all(
133
+ token.special
134
+ and not token.normalized
135
+ and not token.lstrip
136
+ and not token.rstrip
137
+ and not token.single_word
138
+ for token in added_tokens
139
+ )
140
+ ):
141
+ return False
142
+
143
+ encoded = backend.encode(tokenizer.unk_token, add_special_tokens=True)
144
+ return (
145
+ backend.encode(
146
+ tokenizer.unk_token,
147
+ add_special_tokens=False,
148
+ ).ids
149
+ == [tokenizer.unk_token_id]
150
+ and encoded.ids
151
+ == [
152
+ tokenizer.cls_token_id,
153
+ tokenizer.unk_token_id,
154
+ tokenizer.sep_token_id,
155
+ ]
156
+ and not any(encoded.type_ids)
157
+ )
158
+
159
+ @classmethod
160
+ def build(
161
+ cls,
162
+ model: sentence_transformers.SentenceTransformer,
163
+ ) -> _CuDFTokenizer | None:
164
+ if not cls._is_supported(model):
165
+ return None
166
+
167
+ tokenizer = model.tokenizer
168
+ backend = tokenizer.backend_tokenizer
169
+ wordpiece = backend.model
170
+ normalizer = backend.normalizer
171
+ vocab = backend.get_vocab(with_added_tokens=False)
172
+
173
+ if importlib.util.find_spec("cudf") is None:
174
+ warn_once(
175
+ key="on-device-tokenization-available-but-cudf-unavailable",
176
+ message=(
177
+ "Falling back to CPU-based tokenization because cuDF is "
178
+ "not installed. Install cuDF to enable faster CUDA-based "
179
+ "tokenization without device synchronization."
180
+ ),
181
+ )
182
+ return None
183
+
184
+ import cudf
185
+ from cudf.core.character_normalizer import CharacterNormalizer
186
+ from cudf.core.wordpiece_tokenize import WordPieceVocabulary
187
+
188
+ vocab_tokens = [
189
+ token for token, _ in sorted(vocab.items(), key=lambda x: x[1])
190
+ ]
191
+ return cls(
192
+ vocabulary=WordPieceVocabulary(cudf.Series(vocab_tokens)),
193
+ normalizer=CharacterNormalizer(
194
+ do_lower=normalizer.lowercase,
195
+ special_tokens=cudf.Series(tokenizer.all_special_tokens),
196
+ ),
197
+ cls_token_id=tokenizer.cls_token_id,
198
+ sep_token_id=tokenizer.sep_token_id,
199
+ pad_token_id=tokenizer.pad_token_id,
200
+ max_length=tokenizer.model_max_length,
201
+ max_input_chars_per_word=wordpiece.max_input_chars_per_word,
202
+ )
203
+
204
+ def tokenize(self, text: StringTensor) -> tuple[Tensor, Tensor]:
205
+ device = text.device
206
+ ser = text.to_cudf()
207
+ ser = ser.fillna("") if text.is_nullable else ser
208
+ ser = self._normalizer.normalize(ser)
209
+ ser = self._vocabulary.tokenize(ser)
210
+
211
+ flat_values = torch.from_dlpack(ser.list.leaves.to_cupy())
212
+ lengths = torch.from_dlpack(ser.list.len().to_cupy())
213
+
214
+ num_strings = text.numel()
215
+ offsets = torch.zeros(
216
+ num_strings + 1,
217
+ device=device,
218
+ dtype=torch.int32,
219
+ )
220
+ torch.cumsum(lengths, dim=0, out=offsets[1:])
221
+ lengths.clamp_(max=self._max_length - 2)
222
+ input_ids = torch.full(
223
+ (num_strings, self._max_length),
224
+ self._pad_token_id,
225
+ device=device,
226
+ dtype=torch.int32,
227
+ )
228
+ input_ids[:, 0] = self._cls_token_id
229
+
230
+ if flat_values.numel() > 0:
231
+ max_content = self._max_length - 2
232
+ col_idx = torch.arange(
233
+ max_content,
234
+ device=device,
235
+ dtype=torch.int32,
236
+ ).unsqueeze(0)
237
+ mask = col_idx < lengths.unsqueeze(1)
238
+ src = col_idx + offsets[:-1].unsqueeze(1)
239
+ safe_src = torch.where(mask, src, torch.zeros_like(src))
240
+ input_ids[:, 1 : max_content + 1] = torch.where(
241
+ mask,
242
+ flat_values[safe_src],
243
+ input_ids[:, 1 : max_content + 1],
244
+ )
245
+
246
+ input_ids[torch.arange(num_strings, device=device), lengths + 1] = (
247
+ self._sep_token_id
248
+ )
249
+ attention_mask = (
250
+ torch.arange(self._max_length, device=device).unsqueeze(0)
251
+ < (lengths + 2).unsqueeze(1)
252
+ ).to(torch.int32)
253
+
254
+ return input_ids, attention_mask
255
+
256
+
257
+ class _Encoder(torch.nn.Module):
258
+ def __init__(
259
+ self,
260
+ model: sentence_transformers.SentenceTransformer,
261
+ batch_size: int,
262
+ embedding_dim: int,
263
+ ) -> None:
264
+ super().__init__()
265
+ self._model = model
266
+ self._batch_size = batch_size
267
+ self._embedding_dim = embedding_dim
268
+
269
+ @cached_property
270
+ def _cudf_tokenizer(self) -> _CuDFTokenizer | None:
271
+ return _CuDFTokenizer.build(self._model)
272
+
273
+ def __deepcopy__(self, memo: dict[int, Any]) -> _Encoder:
274
+ return self
275
+
276
+ def _forward_cpu(self, text: StringTensor) -> Tensor:
277
+ array = text.to_arrow()
278
+ if text.is_nullable:
279
+ array = pc.fill_null(array, "")
280
+ emb = self._model.encode(
281
+ array.to_pylist(),
282
+ show_progress_bar=False,
283
+ convert_to_tensor=True,
284
+ device=str(text.device),
285
+ batch_size=self._batch_size,
286
+ )
287
+ assert isinstance(emb, Tensor)
288
+ return emb
289
+
290
+ def _forward_cudf(self, text: StringTensor) -> Tensor:
291
+ assert self._cudf_tokenizer is not None
292
+ self._model.eval()
293
+ input_ids, attention_mask = self._cudf_tokenizer.tokenize(text)
294
+
295
+ seq_lengths = attention_mask.sum(dim=1)
296
+ sort_idx = seq_lengths.argsort()
297
+ sorted_input_ids = input_ids[sort_idx]
298
+ sorted_attention_mask = attention_mask[sort_idx]
299
+ sorted_seq_lengths = seq_lengths[sort_idx]
300
+
301
+ device = text.device
302
+ num_strings = text.numel()
303
+ batch_ends = torch.arange(
304
+ self._batch_size,
305
+ num_strings + self._batch_size,
306
+ self._batch_size,
307
+ device=device,
308
+ )
309
+ batch_ends.clamp_(max=num_strings)
310
+ batch_ends -= 1
311
+ # Triggers a host device sync to get chunk sizes to
312
+ # minimize the padding
313
+ batch_max_lengths = sorted_seq_lengths[batch_ends].tolist()
314
+
315
+ embeddings = torch.empty(
316
+ num_strings,
317
+ self._embedding_dim,
318
+ device=device,
319
+ dtype=torch.float32,
320
+ )
321
+ for i, start in enumerate(range(0, num_strings, self._batch_size)):
322
+ end = min(start + self._batch_size, num_strings)
323
+ max_len = batch_max_lengths[i]
324
+ features: dict[str, Tensor] = {
325
+ "input_ids": sorted_input_ids[start:end, :max_len],
326
+ "attention_mask": sorted_attention_mask[start:end, :max_len],
327
+ }
328
+ features = self._model(features)
329
+ embeddings[sort_idx[start:end]] = features["sentence_embedding"]
330
+
331
+ return embeddings
332
+
333
+ def forward(self, text: StringTensor) -> Tensor:
334
+ if text.device.type == "cpu":
335
+ return self._forward_cpu(text)
336
+
337
+ tokenizer = self._cudf_tokenizer
338
+ if tokenizer is None:
339
+ return self._forward_cpu(text)
340
+
341
+ series = text.to_cudf()
342
+ series = series.fillna("") if text.is_nullable else series
343
+ words = tokenizer._normalizer.normalize(series).str.tokenize()
344
+
345
+ # CPU rejects long words by Unicode characters, while cuDF uses UTF-8
346
+ # bytes. The following lines find rows where those decisions differ.
347
+ # Remove this fallback if cuDF exposes the tokenizer's
348
+ # max_input_chars_per_word setting.
349
+ use_cpu = (words.str.len() > tokenizer._max_input_chars_per_word) != (
350
+ words.str.byte_count() >= _CUDF_WORDPIECE_MAX_BYTES
351
+ )
352
+ cpu_indices = torch.from_dlpack(words[use_cpu].index.unique().values)
353
+
354
+ num_strings = text.numel()
355
+ if cpu_indices.numel() == 0:
356
+ return self._forward_cudf(text)
357
+ if cpu_indices.numel() == num_strings:
358
+ return self._forward_cpu(text)
359
+
360
+ use_cudf = torch.ones(
361
+ num_strings,
362
+ device=text.device,
363
+ dtype=torch.bool,
364
+ )
365
+ use_cudf[cpu_indices] = False
366
+ cudf_indices = use_cudf.nonzero().flatten()
367
+
368
+ # Tokenize each subset with the matching path, then restore the
369
+ # original input order through indexed assignment.
370
+ embeddings = torch.empty(
371
+ num_strings,
372
+ self._embedding_dim,
373
+ device=text.device,
374
+ dtype=torch.float32,
375
+ )
376
+ cudf_text = cast(StringTensor, text[cudf_indices])
377
+ cpu_text = cast(StringTensor, text[cpu_indices])
378
+ embeddings[cudf_indices] = self._forward_cudf(cudf_text)
379
+ embeddings[cpu_indices] = self._forward_cpu(cpu_text)
380
+ return embeddings
381
+
382
+
383
+ class SentenceTransformer(Processor):
384
+ r"""Transform text columns with a
385
+ :class:`sentence_transformers.SentenceTransformer` model.
386
+
387
+ Args:
388
+ model_name: Model name or local path passed to
389
+ :class:`sentence_transformers.SentenceTransformer`
390
+ batch_size: Batch size passed to
391
+ :meth:`sentence_transformers.SentenceTransformer.encode`.
392
+ Adjusting the batch size can significantly improve processing
393
+ speed. The optimal value depends on your hardware, model size,
394
+ precision, and input length.
395
+ """
396
+
397
+ requires_fit = False
398
+ handles_stypes = frozenset({Stype.text})
399
+
400
+ def __init__(
401
+ self,
402
+ model_name: str,
403
+ *,
404
+ batch_size: int = 32,
405
+ ) -> None:
406
+ super().__init__()
407
+ import sentence_transformers
408
+
409
+ self.batch_size = batch_size
410
+ model = sentence_transformers.SentenceTransformer(model_name)
411
+ embedding_dim = model.get_embedding_dimension()
412
+ assert isinstance(embedding_dim, int)
413
+ self._embedding_dim = embedding_dim
414
+ self._model = _Encoder(model, batch_size, embedding_dim)
415
+
416
+ def _transform(self, table: TableTensor) -> TableTensor:
417
+ columns = table.columns[Stype.text]
418
+ batch_shape = table.text.shape[:-1]
419
+ output_columns = tuple(
420
+ f"{column}__emb{i}"
421
+ for column in columns
422
+ for i in range(self._embedding_dim)
423
+ )
424
+
425
+ if table.text.numel() == 0:
426
+ numerical = torch.empty(
427
+ (*batch_shape, len(output_columns)),
428
+ dtype=table.dtype,
429
+ device=table.device,
430
+ )
431
+ else:
432
+ text = cast(StringTensor, table.text.movedim(-1, 0).reshape(-1))
433
+ emb = self._model(text)
434
+ emb = emb.to(device=table.device, dtype=table.dtype)
435
+ numerical = (
436
+ emb.reshape(len(columns), *batch_shape, self._embedding_dim)
437
+ .movedim(0, -2)
438
+ .reshape(*batch_shape, len(output_columns))
439
+ )
440
+
441
+ out = torch.cat(
442
+ [
443
+ table.drop_stypes(Stype.text),
444
+ TableTensor(
445
+ columns={Stype.numerical: output_columns},
446
+ numerical=numerical,
447
+ ),
448
+ ],
449
+ dim=-1,
450
+ )
451
+ return cast(TableTensor, out)