structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from sdm import Stype, TableTensor
|
|
5
|
+
from sdm.processing import Processor
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class SortQuantiles(Processor):
|
|
9
|
+
r"""Sort quantile predictions in ascending order."""
|
|
10
|
+
|
|
11
|
+
handles_stypes = frozenset({Stype.numerical})
|
|
12
|
+
requires_fit = False
|
|
13
|
+
|
|
14
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
15
|
+
numerical = table.numerical.sort(dim=-1).values
|
|
16
|
+
return table.replace_blocks(numerical=numerical)
|
sdm/processing/recipe.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Self
|
|
9
|
+
|
|
10
|
+
import sdm.processing as sp
|
|
11
|
+
from sdm.processing import EnsembleProcessor, Processor
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(init=False, repr=False)
|
|
15
|
+
class Recipe:
|
|
16
|
+
"""Processing contract around an external model boundary.
|
|
17
|
+
|
|
18
|
+
A recipe bundles three processing pipelines, one per role the data plays
|
|
19
|
+
relative to the model:
|
|
20
|
+
|
|
21
|
+
- ``features``: model inputs, transformed before the model.
|
|
22
|
+
- ``target``: labels transformed forward before the model. Regression
|
|
23
|
+
predictions are inverted through this pipeline; classification outputs
|
|
24
|
+
are reconstructed from the fitted target categories instead.
|
|
25
|
+
- ``output``: transforms member outputs after they have been mapped to a
|
|
26
|
+
common class or target space and stacked as ``[E, ..., R, O]``. An
|
|
27
|
+
explicit dimension-changing step such as
|
|
28
|
+
:class:`~sdm.processing.AverageEstimators` removes ``E``; without one,
|
|
29
|
+
the output remains stacked. Steps before the reducer must support
|
|
30
|
+
stacked outputs, while steps after it receive already-reduced outputs.
|
|
31
|
+
|
|
32
|
+
Each pipeline exposes ``fit``/``transform``/``fit_transform`` and, when its
|
|
33
|
+
steps are invertible, ``inverse_transform``. Call them directly, e.g.
|
|
34
|
+
``recipe.features.transform(table)`` or
|
|
35
|
+
``recipe.target.inverse_transform(prediction)``. Recipes do not infer each
|
|
36
|
+
step's non-finite input contract; order steps so values are imputed before
|
|
37
|
+
processors that do not explicitly document non-finite support.
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
features: Steps applied to model inputs before model execution.
|
|
41
|
+
target: Steps applied to labels before model execution.
|
|
42
|
+
output: Steps applied to stacked model outputs.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
_features: EnsembleProcessor
|
|
46
|
+
_target: EnsembleProcessor
|
|
47
|
+
_output: EnsembleProcessor
|
|
48
|
+
|
|
49
|
+
def __init__(
|
|
50
|
+
self,
|
|
51
|
+
features: Processor | Iterable[Processor] | None = None,
|
|
52
|
+
target: Processor | Iterable[Processor] | None = None,
|
|
53
|
+
output: Processor | Iterable[Processor] | None = None,
|
|
54
|
+
) -> None:
|
|
55
|
+
self.features = features
|
|
56
|
+
self.target = target
|
|
57
|
+
self.output = output
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def features(self) -> EnsembleProcessor:
|
|
61
|
+
r"""The steps applied to model inputs."""
|
|
62
|
+
return self._features
|
|
63
|
+
|
|
64
|
+
@features.setter
|
|
65
|
+
def features(
|
|
66
|
+
self,
|
|
67
|
+
processor: Processor | Iterable[Processor] | None,
|
|
68
|
+
) -> None:
|
|
69
|
+
if processor is None:
|
|
70
|
+
processor = sp.Identity()
|
|
71
|
+
self._features = EnsembleProcessor.as_processor(processor)
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def target(self) -> EnsembleProcessor:
|
|
75
|
+
r"""The steps applied to labels."""
|
|
76
|
+
return self._target
|
|
77
|
+
|
|
78
|
+
@target.setter
|
|
79
|
+
def target(
|
|
80
|
+
self,
|
|
81
|
+
processor: Processor | Iterable[Processor] | None,
|
|
82
|
+
) -> None:
|
|
83
|
+
if processor is None:
|
|
84
|
+
processor = sp.Identity()
|
|
85
|
+
self._target = EnsembleProcessor.as_processor(processor)
|
|
86
|
+
|
|
87
|
+
if any(isinstance(m, sp.TaskDispatch) for m in self.target.modules()):
|
|
88
|
+
raise ValueError(
|
|
89
|
+
"'TaskDispatch' is not supported in 'Recipe.target'"
|
|
90
|
+
)
|
|
91
|
+
if any(isinstance(m, sp.TableDispatch) for m in self.target.modules()):
|
|
92
|
+
raise ValueError(
|
|
93
|
+
"'TableDispatch' is not supported in 'Recipe.target'"
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
@property
|
|
97
|
+
def output(self) -> EnsembleProcessor:
|
|
98
|
+
r"""The steps applied to model outputs."""
|
|
99
|
+
return self._output
|
|
100
|
+
|
|
101
|
+
@output.setter
|
|
102
|
+
def output(
|
|
103
|
+
self,
|
|
104
|
+
processor: Processor | Iterable[Processor] | None,
|
|
105
|
+
) -> None:
|
|
106
|
+
if processor is None:
|
|
107
|
+
processor = sp.Identity()
|
|
108
|
+
self._output = EnsembleProcessor.as_processor(processor)
|
|
109
|
+
|
|
110
|
+
if any(isinstance(m, sp.TableDispatch) for m in self.output.modules()):
|
|
111
|
+
raise ValueError(
|
|
112
|
+
"'TableDispatch' is not supported in 'Recipe.output'"
|
|
113
|
+
)
|
|
114
|
+
if self.output.requires_fit:
|
|
115
|
+
raise ValueError("'Recipe.output' should not require fitting")
|
|
116
|
+
|
|
117
|
+
def prepend_features(self, processor: object) -> Self:
|
|
118
|
+
"""Prepend a processor to the feature pipeline."""
|
|
119
|
+
self.features = processor + self.features
|
|
120
|
+
return self
|
|
121
|
+
|
|
122
|
+
def append_features(self, processor: object) -> Self:
|
|
123
|
+
"""Append a processor to the feature pipeline."""
|
|
124
|
+
self.features = self.features + processor
|
|
125
|
+
return self
|
|
126
|
+
|
|
127
|
+
def prepend_target(self, processor: object) -> Self:
|
|
128
|
+
"""Prepend a processor to the target pipeline."""
|
|
129
|
+
self.target = processor + self.target
|
|
130
|
+
return self
|
|
131
|
+
|
|
132
|
+
def append_target(self, processor: object) -> Self:
|
|
133
|
+
"""Append a processor to the target pipeline."""
|
|
134
|
+
self.target = self.target + processor
|
|
135
|
+
return self
|
|
136
|
+
|
|
137
|
+
def prepend_output(self, processor: object) -> Self:
|
|
138
|
+
"""Prepend a processor to the output pipeline."""
|
|
139
|
+
self.output = processor + self.output
|
|
140
|
+
return self
|
|
141
|
+
|
|
142
|
+
def append_output(self, processor: object) -> Self:
|
|
143
|
+
"""Append a processor to the output pipeline."""
|
|
144
|
+
self.output = self.output + processor
|
|
145
|
+
return self
|
|
146
|
+
|
|
147
|
+
def __repr__(self) -> str:
|
|
148
|
+
return (
|
|
149
|
+
f"{self.__class__.__name__}(\n"
|
|
150
|
+
f" features={self.features.__repr__(indent=2)[2:]},\n"
|
|
151
|
+
f" target={self.target.__repr__(indent=2)[2:]},\n"
|
|
152
|
+
f" output={self.output.__repr__(indent=2)[2:]},\n"
|
|
153
|
+
")"
|
|
154
|
+
)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Text processors."""
|
|
5
|
+
|
|
6
|
+
from sdm.processing.text.tfidf import TFIDF
|
|
7
|
+
from sdm.processing.text.sentence_transformer import SentenceTransformer
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"TFIDF",
|
|
11
|
+
"SentenceTransformer",
|
|
12
|
+
]
|
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
# ruff: noqa: D205
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import importlib.util
|
|
9
|
+
from functools import cached_property
|
|
10
|
+
from typing import TYPE_CHECKING, Any, cast
|
|
11
|
+
|
|
12
|
+
import pyarrow.compute as pc
|
|
13
|
+
import torch
|
|
14
|
+
from torch import Tensor
|
|
15
|
+
|
|
16
|
+
from sdm import StringTensor, Stype, TableTensor
|
|
17
|
+
from sdm._warnings import warn_once
|
|
18
|
+
from sdm.processing import Processor
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
import sentence_transformers
|
|
22
|
+
from cudf.core.character_normalizer import CharacterNormalizer
|
|
23
|
+
from cudf.core.wordpiece_tokenize import WordPieceVocabulary
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# Hugging Face treats words with more than 100 Unicode characters as [UNK],
|
|
27
|
+
# while cuDF does so for words with 200 or more UTF-8 bytes.
|
|
28
|
+
_CUDF_WORDPIECE_MAX_BYTES = 200
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class _CuDFTokenizer:
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
vocabulary: WordPieceVocabulary,
|
|
35
|
+
normalizer: CharacterNormalizer,
|
|
36
|
+
cls_token_id: int,
|
|
37
|
+
sep_token_id: int,
|
|
38
|
+
pad_token_id: int,
|
|
39
|
+
max_length: int,
|
|
40
|
+
max_input_chars_per_word: int,
|
|
41
|
+
) -> None:
|
|
42
|
+
self._vocabulary = vocabulary
|
|
43
|
+
self._normalizer = normalizer
|
|
44
|
+
self._cls_token_id = cls_token_id
|
|
45
|
+
self._sep_token_id = sep_token_id
|
|
46
|
+
self._pad_token_id = pad_token_id
|
|
47
|
+
self._max_length = max_length
|
|
48
|
+
self._max_input_chars_per_word = max_input_chars_per_word
|
|
49
|
+
|
|
50
|
+
@staticmethod
|
|
51
|
+
def _is_supported(
|
|
52
|
+
model: sentence_transformers.SentenceTransformer,
|
|
53
|
+
) -> bool:
|
|
54
|
+
import tokenizers
|
|
55
|
+
from sentence_transformers.sentence_transformer.modules import (
|
|
56
|
+
Transformer,
|
|
57
|
+
)
|
|
58
|
+
from transformers import PreTrainedTokenizerFast
|
|
59
|
+
|
|
60
|
+
module = model[0]
|
|
61
|
+
tokenizer = model.tokenizer
|
|
62
|
+
|
|
63
|
+
# Fall back to CPU tokenization unless this is a standard BERT
|
|
64
|
+
# Transformer module with a fast tokenizer.
|
|
65
|
+
if (
|
|
66
|
+
type(module) is not Transformer
|
|
67
|
+
or module.config.model_type != "bert"
|
|
68
|
+
or not isinstance(tokenizer, PreTrainedTokenizerFast)
|
|
69
|
+
):
|
|
70
|
+
return False
|
|
71
|
+
|
|
72
|
+
if (
|
|
73
|
+
model.modalities != ["text"]
|
|
74
|
+
or model.default_prompt_name is not None
|
|
75
|
+
or model.truncate_dim is not None
|
|
76
|
+
or module.transformer_task != "feature-extraction"
|
|
77
|
+
or module.backend != "torch"
|
|
78
|
+
or module.processing_kwargs
|
|
79
|
+
or module.can_flatten_inputs
|
|
80
|
+
or module.unpad_inputs
|
|
81
|
+
):
|
|
82
|
+
return False
|
|
83
|
+
|
|
84
|
+
backend = tokenizer.backend_tokenizer
|
|
85
|
+
wordpiece = backend.model
|
|
86
|
+
normalizer = backend.normalizer
|
|
87
|
+
|
|
88
|
+
# cuDF cannot currently construct an equivalent tokenizer directly
|
|
89
|
+
# from the Hugging Face tokenizer configuration. Keep these checks
|
|
90
|
+
# until it can.
|
|
91
|
+
# Require the standard BERT WordPiece pipeline.
|
|
92
|
+
if (
|
|
93
|
+
type(wordpiece) is not tokenizers.models.WordPiece
|
|
94
|
+
or type(normalizer) is not tokenizers.normalizers.BertNormalizer
|
|
95
|
+
or type(backend.pre_tokenizer)
|
|
96
|
+
is not tokenizers.pre_tokenizers.BertPreTokenizer
|
|
97
|
+
or type(backend.post_processor)
|
|
98
|
+
is not tokenizers.processors.TemplateProcessing
|
|
99
|
+
or (
|
|
100
|
+
normalizer.clean_text,
|
|
101
|
+
normalizer.handle_chinese_chars,
|
|
102
|
+
normalizer.strip_accents,
|
|
103
|
+
)
|
|
104
|
+
!= (True, True, None)
|
|
105
|
+
or (wordpiece.unk_token, wordpiece.continuing_subword_prefix)
|
|
106
|
+
!= ("[UNK]", "##")
|
|
107
|
+
or (tokenizer.padding_side, tokenizer.truncation_side)
|
|
108
|
+
!= ("right", "right")
|
|
109
|
+
or tokenizer.model_max_length != module.max_seq_length
|
|
110
|
+
):
|
|
111
|
+
return False
|
|
112
|
+
|
|
113
|
+
vocab = backend.get_vocab(with_added_tokens=False)
|
|
114
|
+
added_tokens = tuple(backend.get_added_tokens_decoder().values())
|
|
115
|
+
special_tokens = (
|
|
116
|
+
(tokenizer.cls_token, tokenizer.cls_token_id),
|
|
117
|
+
(tokenizer.sep_token, tokenizer.sep_token_id),
|
|
118
|
+
(tokenizer.pad_token, tokenizer.pad_token_id),
|
|
119
|
+
(tokenizer.unk_token, tokenizer.unk_token_id),
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
# Vocabulary IDs and added-token behavior must match cuDF's lookup.
|
|
123
|
+
if (
|
|
124
|
+
backend.get_vocab_size(with_added_tokens=True) != len(vocab)
|
|
125
|
+
or set(vocab.values()) != set(range(len(vocab)))
|
|
126
|
+
or not all(
|
|
127
|
+
vocab.get(token) == token_id
|
|
128
|
+
for token, token_id in special_tokens
|
|
129
|
+
)
|
|
130
|
+
or {token.content for token in added_tokens}
|
|
131
|
+
!= set(tokenizer.all_special_tokens)
|
|
132
|
+
or not all(
|
|
133
|
+
token.special
|
|
134
|
+
and not token.normalized
|
|
135
|
+
and not token.lstrip
|
|
136
|
+
and not token.rstrip
|
|
137
|
+
and not token.single_word
|
|
138
|
+
for token in added_tokens
|
|
139
|
+
)
|
|
140
|
+
):
|
|
141
|
+
return False
|
|
142
|
+
|
|
143
|
+
encoded = backend.encode(tokenizer.unk_token, add_special_tokens=True)
|
|
144
|
+
return (
|
|
145
|
+
backend.encode(
|
|
146
|
+
tokenizer.unk_token,
|
|
147
|
+
add_special_tokens=False,
|
|
148
|
+
).ids
|
|
149
|
+
== [tokenizer.unk_token_id]
|
|
150
|
+
and encoded.ids
|
|
151
|
+
== [
|
|
152
|
+
tokenizer.cls_token_id,
|
|
153
|
+
tokenizer.unk_token_id,
|
|
154
|
+
tokenizer.sep_token_id,
|
|
155
|
+
]
|
|
156
|
+
and not any(encoded.type_ids)
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
@classmethod
|
|
160
|
+
def build(
|
|
161
|
+
cls,
|
|
162
|
+
model: sentence_transformers.SentenceTransformer,
|
|
163
|
+
) -> _CuDFTokenizer | None:
|
|
164
|
+
if not cls._is_supported(model):
|
|
165
|
+
return None
|
|
166
|
+
|
|
167
|
+
tokenizer = model.tokenizer
|
|
168
|
+
backend = tokenizer.backend_tokenizer
|
|
169
|
+
wordpiece = backend.model
|
|
170
|
+
normalizer = backend.normalizer
|
|
171
|
+
vocab = backend.get_vocab(with_added_tokens=False)
|
|
172
|
+
|
|
173
|
+
if importlib.util.find_spec("cudf") is None:
|
|
174
|
+
warn_once(
|
|
175
|
+
key="on-device-tokenization-available-but-cudf-unavailable",
|
|
176
|
+
message=(
|
|
177
|
+
"Falling back to CPU-based tokenization because cuDF is "
|
|
178
|
+
"not installed. Install cuDF to enable faster CUDA-based "
|
|
179
|
+
"tokenization without device synchronization."
|
|
180
|
+
),
|
|
181
|
+
)
|
|
182
|
+
return None
|
|
183
|
+
|
|
184
|
+
import cudf
|
|
185
|
+
from cudf.core.character_normalizer import CharacterNormalizer
|
|
186
|
+
from cudf.core.wordpiece_tokenize import WordPieceVocabulary
|
|
187
|
+
|
|
188
|
+
vocab_tokens = [
|
|
189
|
+
token for token, _ in sorted(vocab.items(), key=lambda x: x[1])
|
|
190
|
+
]
|
|
191
|
+
return cls(
|
|
192
|
+
vocabulary=WordPieceVocabulary(cudf.Series(vocab_tokens)),
|
|
193
|
+
normalizer=CharacterNormalizer(
|
|
194
|
+
do_lower=normalizer.lowercase,
|
|
195
|
+
special_tokens=cudf.Series(tokenizer.all_special_tokens),
|
|
196
|
+
),
|
|
197
|
+
cls_token_id=tokenizer.cls_token_id,
|
|
198
|
+
sep_token_id=tokenizer.sep_token_id,
|
|
199
|
+
pad_token_id=tokenizer.pad_token_id,
|
|
200
|
+
max_length=tokenizer.model_max_length,
|
|
201
|
+
max_input_chars_per_word=wordpiece.max_input_chars_per_word,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
def tokenize(self, text: StringTensor) -> tuple[Tensor, Tensor]:
|
|
205
|
+
device = text.device
|
|
206
|
+
ser = text.to_cudf()
|
|
207
|
+
ser = ser.fillna("") if text.is_nullable else ser
|
|
208
|
+
ser = self._normalizer.normalize(ser)
|
|
209
|
+
ser = self._vocabulary.tokenize(ser)
|
|
210
|
+
|
|
211
|
+
flat_values = torch.from_dlpack(ser.list.leaves.to_cupy())
|
|
212
|
+
lengths = torch.from_dlpack(ser.list.len().to_cupy())
|
|
213
|
+
|
|
214
|
+
num_strings = text.numel()
|
|
215
|
+
offsets = torch.zeros(
|
|
216
|
+
num_strings + 1,
|
|
217
|
+
device=device,
|
|
218
|
+
dtype=torch.int32,
|
|
219
|
+
)
|
|
220
|
+
torch.cumsum(lengths, dim=0, out=offsets[1:])
|
|
221
|
+
lengths.clamp_(max=self._max_length - 2)
|
|
222
|
+
input_ids = torch.full(
|
|
223
|
+
(num_strings, self._max_length),
|
|
224
|
+
self._pad_token_id,
|
|
225
|
+
device=device,
|
|
226
|
+
dtype=torch.int32,
|
|
227
|
+
)
|
|
228
|
+
input_ids[:, 0] = self._cls_token_id
|
|
229
|
+
|
|
230
|
+
if flat_values.numel() > 0:
|
|
231
|
+
max_content = self._max_length - 2
|
|
232
|
+
col_idx = torch.arange(
|
|
233
|
+
max_content,
|
|
234
|
+
device=device,
|
|
235
|
+
dtype=torch.int32,
|
|
236
|
+
).unsqueeze(0)
|
|
237
|
+
mask = col_idx < lengths.unsqueeze(1)
|
|
238
|
+
src = col_idx + offsets[:-1].unsqueeze(1)
|
|
239
|
+
safe_src = torch.where(mask, src, torch.zeros_like(src))
|
|
240
|
+
input_ids[:, 1 : max_content + 1] = torch.where(
|
|
241
|
+
mask,
|
|
242
|
+
flat_values[safe_src],
|
|
243
|
+
input_ids[:, 1 : max_content + 1],
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
input_ids[torch.arange(num_strings, device=device), lengths + 1] = (
|
|
247
|
+
self._sep_token_id
|
|
248
|
+
)
|
|
249
|
+
attention_mask = (
|
|
250
|
+
torch.arange(self._max_length, device=device).unsqueeze(0)
|
|
251
|
+
< (lengths + 2).unsqueeze(1)
|
|
252
|
+
).to(torch.int32)
|
|
253
|
+
|
|
254
|
+
return input_ids, attention_mask
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
class _Encoder(torch.nn.Module):
|
|
258
|
+
def __init__(
|
|
259
|
+
self,
|
|
260
|
+
model: sentence_transformers.SentenceTransformer,
|
|
261
|
+
batch_size: int,
|
|
262
|
+
embedding_dim: int,
|
|
263
|
+
) -> None:
|
|
264
|
+
super().__init__()
|
|
265
|
+
self._model = model
|
|
266
|
+
self._batch_size = batch_size
|
|
267
|
+
self._embedding_dim = embedding_dim
|
|
268
|
+
|
|
269
|
+
@cached_property
|
|
270
|
+
def _cudf_tokenizer(self) -> _CuDFTokenizer | None:
|
|
271
|
+
return _CuDFTokenizer.build(self._model)
|
|
272
|
+
|
|
273
|
+
def __deepcopy__(self, memo: dict[int, Any]) -> _Encoder:
|
|
274
|
+
return self
|
|
275
|
+
|
|
276
|
+
def _forward_cpu(self, text: StringTensor) -> Tensor:
|
|
277
|
+
array = text.to_arrow()
|
|
278
|
+
if text.is_nullable:
|
|
279
|
+
array = pc.fill_null(array, "")
|
|
280
|
+
emb = self._model.encode(
|
|
281
|
+
array.to_pylist(),
|
|
282
|
+
show_progress_bar=False,
|
|
283
|
+
convert_to_tensor=True,
|
|
284
|
+
device=str(text.device),
|
|
285
|
+
batch_size=self._batch_size,
|
|
286
|
+
)
|
|
287
|
+
assert isinstance(emb, Tensor)
|
|
288
|
+
return emb
|
|
289
|
+
|
|
290
|
+
def _forward_cudf(self, text: StringTensor) -> Tensor:
|
|
291
|
+
assert self._cudf_tokenizer is not None
|
|
292
|
+
self._model.eval()
|
|
293
|
+
input_ids, attention_mask = self._cudf_tokenizer.tokenize(text)
|
|
294
|
+
|
|
295
|
+
seq_lengths = attention_mask.sum(dim=1)
|
|
296
|
+
sort_idx = seq_lengths.argsort()
|
|
297
|
+
sorted_input_ids = input_ids[sort_idx]
|
|
298
|
+
sorted_attention_mask = attention_mask[sort_idx]
|
|
299
|
+
sorted_seq_lengths = seq_lengths[sort_idx]
|
|
300
|
+
|
|
301
|
+
device = text.device
|
|
302
|
+
num_strings = text.numel()
|
|
303
|
+
batch_ends = torch.arange(
|
|
304
|
+
self._batch_size,
|
|
305
|
+
num_strings + self._batch_size,
|
|
306
|
+
self._batch_size,
|
|
307
|
+
device=device,
|
|
308
|
+
)
|
|
309
|
+
batch_ends.clamp_(max=num_strings)
|
|
310
|
+
batch_ends -= 1
|
|
311
|
+
# Triggers a host device sync to get chunk sizes to
|
|
312
|
+
# minimize the padding
|
|
313
|
+
batch_max_lengths = sorted_seq_lengths[batch_ends].tolist()
|
|
314
|
+
|
|
315
|
+
embeddings = torch.empty(
|
|
316
|
+
num_strings,
|
|
317
|
+
self._embedding_dim,
|
|
318
|
+
device=device,
|
|
319
|
+
dtype=torch.float32,
|
|
320
|
+
)
|
|
321
|
+
for i, start in enumerate(range(0, num_strings, self._batch_size)):
|
|
322
|
+
end = min(start + self._batch_size, num_strings)
|
|
323
|
+
max_len = batch_max_lengths[i]
|
|
324
|
+
features: dict[str, Tensor] = {
|
|
325
|
+
"input_ids": sorted_input_ids[start:end, :max_len],
|
|
326
|
+
"attention_mask": sorted_attention_mask[start:end, :max_len],
|
|
327
|
+
}
|
|
328
|
+
features = self._model(features)
|
|
329
|
+
embeddings[sort_idx[start:end]] = features["sentence_embedding"]
|
|
330
|
+
|
|
331
|
+
return embeddings
|
|
332
|
+
|
|
333
|
+
def forward(self, text: StringTensor) -> Tensor:
|
|
334
|
+
if text.device.type == "cpu":
|
|
335
|
+
return self._forward_cpu(text)
|
|
336
|
+
|
|
337
|
+
tokenizer = self._cudf_tokenizer
|
|
338
|
+
if tokenizer is None:
|
|
339
|
+
return self._forward_cpu(text)
|
|
340
|
+
|
|
341
|
+
series = text.to_cudf()
|
|
342
|
+
series = series.fillna("") if text.is_nullable else series
|
|
343
|
+
words = tokenizer._normalizer.normalize(series).str.tokenize()
|
|
344
|
+
|
|
345
|
+
# CPU rejects long words by Unicode characters, while cuDF uses UTF-8
|
|
346
|
+
# bytes. The following lines find rows where those decisions differ.
|
|
347
|
+
# Remove this fallback if cuDF exposes the tokenizer's
|
|
348
|
+
# max_input_chars_per_word setting.
|
|
349
|
+
use_cpu = (words.str.len() > tokenizer._max_input_chars_per_word) != (
|
|
350
|
+
words.str.byte_count() >= _CUDF_WORDPIECE_MAX_BYTES
|
|
351
|
+
)
|
|
352
|
+
cpu_indices = torch.from_dlpack(words[use_cpu].index.unique().values)
|
|
353
|
+
|
|
354
|
+
num_strings = text.numel()
|
|
355
|
+
if cpu_indices.numel() == 0:
|
|
356
|
+
return self._forward_cudf(text)
|
|
357
|
+
if cpu_indices.numel() == num_strings:
|
|
358
|
+
return self._forward_cpu(text)
|
|
359
|
+
|
|
360
|
+
use_cudf = torch.ones(
|
|
361
|
+
num_strings,
|
|
362
|
+
device=text.device,
|
|
363
|
+
dtype=torch.bool,
|
|
364
|
+
)
|
|
365
|
+
use_cudf[cpu_indices] = False
|
|
366
|
+
cudf_indices = use_cudf.nonzero().flatten()
|
|
367
|
+
|
|
368
|
+
# Tokenize each subset with the matching path, then restore the
|
|
369
|
+
# original input order through indexed assignment.
|
|
370
|
+
embeddings = torch.empty(
|
|
371
|
+
num_strings,
|
|
372
|
+
self._embedding_dim,
|
|
373
|
+
device=text.device,
|
|
374
|
+
dtype=torch.float32,
|
|
375
|
+
)
|
|
376
|
+
cudf_text = cast(StringTensor, text[cudf_indices])
|
|
377
|
+
cpu_text = cast(StringTensor, text[cpu_indices])
|
|
378
|
+
embeddings[cudf_indices] = self._forward_cudf(cudf_text)
|
|
379
|
+
embeddings[cpu_indices] = self._forward_cpu(cpu_text)
|
|
380
|
+
return embeddings
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
class SentenceTransformer(Processor):
|
|
384
|
+
r"""Transform text columns with a
|
|
385
|
+
:class:`sentence_transformers.SentenceTransformer` model.
|
|
386
|
+
|
|
387
|
+
Args:
|
|
388
|
+
model_name: Model name or local path passed to
|
|
389
|
+
:class:`sentence_transformers.SentenceTransformer`
|
|
390
|
+
batch_size: Batch size passed to
|
|
391
|
+
:meth:`sentence_transformers.SentenceTransformer.encode`.
|
|
392
|
+
Adjusting the batch size can significantly improve processing
|
|
393
|
+
speed. The optimal value depends on your hardware, model size,
|
|
394
|
+
precision, and input length.
|
|
395
|
+
"""
|
|
396
|
+
|
|
397
|
+
requires_fit = False
|
|
398
|
+
handles_stypes = frozenset({Stype.text})
|
|
399
|
+
|
|
400
|
+
def __init__(
|
|
401
|
+
self,
|
|
402
|
+
model_name: str,
|
|
403
|
+
*,
|
|
404
|
+
batch_size: int = 32,
|
|
405
|
+
) -> None:
|
|
406
|
+
super().__init__()
|
|
407
|
+
import sentence_transformers
|
|
408
|
+
|
|
409
|
+
self.batch_size = batch_size
|
|
410
|
+
model = sentence_transformers.SentenceTransformer(model_name)
|
|
411
|
+
embedding_dim = model.get_embedding_dimension()
|
|
412
|
+
assert isinstance(embedding_dim, int)
|
|
413
|
+
self._embedding_dim = embedding_dim
|
|
414
|
+
self._model = _Encoder(model, batch_size, embedding_dim)
|
|
415
|
+
|
|
416
|
+
def _transform(self, table: TableTensor) -> TableTensor:
|
|
417
|
+
columns = table.columns[Stype.text]
|
|
418
|
+
batch_shape = table.text.shape[:-1]
|
|
419
|
+
output_columns = tuple(
|
|
420
|
+
f"{column}__emb{i}"
|
|
421
|
+
for column in columns
|
|
422
|
+
for i in range(self._embedding_dim)
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
if table.text.numel() == 0:
|
|
426
|
+
numerical = torch.empty(
|
|
427
|
+
(*batch_shape, len(output_columns)),
|
|
428
|
+
dtype=table.dtype,
|
|
429
|
+
device=table.device,
|
|
430
|
+
)
|
|
431
|
+
else:
|
|
432
|
+
text = cast(StringTensor, table.text.movedim(-1, 0).reshape(-1))
|
|
433
|
+
emb = self._model(text)
|
|
434
|
+
emb = emb.to(device=table.device, dtype=table.dtype)
|
|
435
|
+
numerical = (
|
|
436
|
+
emb.reshape(len(columns), *batch_shape, self._embedding_dim)
|
|
437
|
+
.movedim(0, -2)
|
|
438
|
+
.reshape(*batch_shape, len(output_columns))
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
out = torch.cat(
|
|
442
|
+
[
|
|
443
|
+
table.drop_stypes(Stype.text),
|
|
444
|
+
TableTensor(
|
|
445
|
+
columns={Stype.numerical: output_columns},
|
|
446
|
+
numerical=numerical,
|
|
447
|
+
),
|
|
448
|
+
],
|
|
449
|
+
dim=-1,
|
|
450
|
+
)
|
|
451
|
+
return cast(TableTensor, out)
|