structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/tensor/string.py
ADDED
|
@@ -0,0 +1,526 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import importlib.util
|
|
7
|
+
import math
|
|
8
|
+
from collections.abc import Sequence
|
|
9
|
+
from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self, cast
|
|
10
|
+
|
|
11
|
+
import pyarrow as pa
|
|
12
|
+
import pyarrow.compute as pc
|
|
13
|
+
import torch
|
|
14
|
+
from torch import Tensor
|
|
15
|
+
from typing_extensions import override
|
|
16
|
+
|
|
17
|
+
from sdm._warnings import warn_once
|
|
18
|
+
from sdm.tensor import VarLenTensor
|
|
19
|
+
from sdm.tensor.io import arrow_as_tensor, combine_arrow_chunks
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
import cudf
|
|
23
|
+
|
|
24
|
+
aten = torch.ops.aten
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class StringTensor(VarLenTensor):
|
|
28
|
+
r"""A :class:`torch.Tensor` for UTF-8 encoded string values.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
data: Flat contiguous ``uint8`` tensor containing all string values.
|
|
32
|
+
offset: One-dimensional offsets into ``data``.
|
|
33
|
+
valid: One-dimensional mask indicating valid, non-null string values.
|
|
34
|
+
size: The shape of the tensor.
|
|
35
|
+
stride: The stride of the tensor.
|
|
36
|
+
storage_offset: The offset into the logical ``offset`` storage.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
# NOTE Assume that `data` stores valid UTF-8 bytes and do not validate it.
|
|
40
|
+
ALLOWED_DTYPES: ClassVar[tuple[torch.dtype, ...] | None] = (torch.uint8,)
|
|
41
|
+
__hash__ = Tensor.__hash__
|
|
42
|
+
|
|
43
|
+
@override
|
|
44
|
+
@classmethod
|
|
45
|
+
def from_arrow(
|
|
46
|
+
cls,
|
|
47
|
+
array: pa.Array | pa.ChunkedArray,
|
|
48
|
+
*,
|
|
49
|
+
size: Sequence[int] | None = None,
|
|
50
|
+
device: torch.device | str | None = None,
|
|
51
|
+
) -> Self:
|
|
52
|
+
r"""Create tensor from a string :class:`pyarrow.Array`.
|
|
53
|
+
|
|
54
|
+
.. testcode::
|
|
55
|
+
|
|
56
|
+
import pyarrow as pa
|
|
57
|
+
from sdm import StringTensor
|
|
58
|
+
|
|
59
|
+
array = pa.array(["foo", "bar", "hello world", ""])
|
|
60
|
+
tensor = StringTensor.from_arrow(array, size=(2, 2))
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
array: The string :class:`pyarrow.Array` or
|
|
64
|
+
:class:`pyarrow.ChunkedArray`.
|
|
65
|
+
size: The shape of the tensor.
|
|
66
|
+
device: The device.
|
|
67
|
+
"""
|
|
68
|
+
if isinstance(array, pa.ChunkedArray):
|
|
69
|
+
array = combine_arrow_chunks(array)
|
|
70
|
+
|
|
71
|
+
if size is None:
|
|
72
|
+
size = (len(array),)
|
|
73
|
+
elif math.prod(size) != len(array):
|
|
74
|
+
raise ValueError(
|
|
75
|
+
f"Expected 'size' in '{cls.__name__}.from_arrow' to contain "
|
|
76
|
+
f"{len(array)} elements (got {math.prod(size)})"
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
is_string = pa.types.is_string(array.type)
|
|
80
|
+
is_large_string = pa.types.is_large_string(array.type)
|
|
81
|
+
if not is_string and not is_large_string:
|
|
82
|
+
raise TypeError(
|
|
83
|
+
f"Expected 'array' in '{cls.__name__}.from_arrow' to have "
|
|
84
|
+
f"'string' or 'large_string' type (got '{array.type}')"
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
buffers = array.buffers()
|
|
88
|
+
offset = torch.frombuffer(
|
|
89
|
+
buffers[1],
|
|
90
|
+
dtype=torch.int32 if is_string else torch.int64,
|
|
91
|
+
)
|
|
92
|
+
valid: Tensor | None = None
|
|
93
|
+
storage_offset = array.offset
|
|
94
|
+
if array.null_count > 0:
|
|
95
|
+
offset = offset[array.offset : array.offset + len(array) + 1]
|
|
96
|
+
valid = arrow_as_tensor(
|
|
97
|
+
array.is_valid(),
|
|
98
|
+
dtype=torch.bool,
|
|
99
|
+
device=device,
|
|
100
|
+
)
|
|
101
|
+
storage_offset = 0
|
|
102
|
+
|
|
103
|
+
return cls(
|
|
104
|
+
data=torch.frombuffer(buffers[2], dtype=torch.uint8).to(device)
|
|
105
|
+
if buffers[2] is not None and buffers[2].size > 0
|
|
106
|
+
else torch.empty(0, dtype=torch.uint8, device=device),
|
|
107
|
+
offset=offset.to(device),
|
|
108
|
+
valid=valid,
|
|
109
|
+
size=size,
|
|
110
|
+
storage_offset=storage_offset,
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
@override
|
|
114
|
+
def to_arrow(self) -> pa.Array:
|
|
115
|
+
r"""Convert this tensor to a flat :class:`pyarrow.Array`."""
|
|
116
|
+
tensor = cast(StringTensor, self.contiguous().cpu())
|
|
117
|
+
|
|
118
|
+
return pa.Array.from_buffers(
|
|
119
|
+
pa.string()
|
|
120
|
+
if tensor._offset.dtype == torch.int32
|
|
121
|
+
else pa.large_string(),
|
|
122
|
+
length=tensor.numel(),
|
|
123
|
+
buffers=[
|
|
124
|
+
pa.array(tensor._valid.numpy(), type=pa.bool_()).buffers()[1]
|
|
125
|
+
if tensor._valid is not None
|
|
126
|
+
else None,
|
|
127
|
+
pa.py_buffer(tensor._offset.numpy()),
|
|
128
|
+
pa.py_buffer(tensor._data.numpy()),
|
|
129
|
+
],
|
|
130
|
+
null_count=-1,
|
|
131
|
+
offset=int(tensor.storage_offset()),
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
@classmethod
|
|
135
|
+
def from_cudf(
|
|
136
|
+
cls,
|
|
137
|
+
ser: cudf.Series | cudf.Index,
|
|
138
|
+
*,
|
|
139
|
+
size: Sequence[int] | None = None,
|
|
140
|
+
device: torch.device | str | None = None,
|
|
141
|
+
) -> Self:
|
|
142
|
+
r"""Create tensor from a string :class:`cudf.Series`.
|
|
143
|
+
|
|
144
|
+
Args:
|
|
145
|
+
ser: The string :class:`cudf.Series` or :class:`cudf.Index`.
|
|
146
|
+
size: The shape of the tensor.
|
|
147
|
+
device: The device.
|
|
148
|
+
"""
|
|
149
|
+
import cupy as cp
|
|
150
|
+
import pylibcudf as plc
|
|
151
|
+
from cudf.api.types import is_string_dtype
|
|
152
|
+
|
|
153
|
+
if size is None:
|
|
154
|
+
size = (len(ser),)
|
|
155
|
+
elif math.prod(size) != len(ser):
|
|
156
|
+
raise ValueError(
|
|
157
|
+
f"Expected 'size' in '{cls.__name__}.from_cudf' to contain "
|
|
158
|
+
f"{len(ser)} elements (got {math.prod(size)})"
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
if not is_string_dtype(ser.dtype):
|
|
162
|
+
raise TypeError(
|
|
163
|
+
f"Expected 'values' in '{cls.__name__}.from_cudf' to have "
|
|
164
|
+
f"string type (got '{ser.dtype}')"
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
# `Series.to_pylibcudf` returns a zero-copy Arrow-style view: base
|
|
168
|
+
# character/offset buffers plus a row offset into the offsets.
|
|
169
|
+
column, _ = ser.to_pylibcudf()
|
|
170
|
+
|
|
171
|
+
# `None` or zero-length when the column holds no characters:
|
|
172
|
+
chars = column.data()
|
|
173
|
+
data = torch.from_dlpack(
|
|
174
|
+
cp.asarray(chars) if chars is not None else cp.empty(0, cp.uint8)
|
|
175
|
+
).to(device)
|
|
176
|
+
|
|
177
|
+
if len(ser) == 0:
|
|
178
|
+
return cls(
|
|
179
|
+
data=data,
|
|
180
|
+
offset=torch.zeros(1, dtype=torch.int32, device=data.device),
|
|
181
|
+
valid=None,
|
|
182
|
+
size=size,
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
offsets = column.children()[0] # (base_size + 1,) INT32/INT64 values
|
|
186
|
+
offset_dtype = (
|
|
187
|
+
cp.int32
|
|
188
|
+
if offsets.type().id() == plc.types.TypeId.INT32
|
|
189
|
+
else cp.int64
|
|
190
|
+
)
|
|
191
|
+
offset = torch.from_dlpack(
|
|
192
|
+
cp.asarray(offsets.data()).view(offset_dtype)
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
valid: Tensor | None = None
|
|
196
|
+
storage_offset = column.offset()
|
|
197
|
+
if ser.hasnans:
|
|
198
|
+
offset = offset[column.offset() : column.offset() + len(ser) + 1]
|
|
199
|
+
valid = torch.from_dlpack(ser.notnull().to_cupy()).to(device)
|
|
200
|
+
storage_offset = 0
|
|
201
|
+
|
|
202
|
+
return cls(
|
|
203
|
+
data=data,
|
|
204
|
+
offset=offset.to(device),
|
|
205
|
+
valid=valid,
|
|
206
|
+
size=size,
|
|
207
|
+
storage_offset=storage_offset,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
def to_cudf(self) -> cudf.Series:
|
|
211
|
+
r"""Convert this CUDA tensor to a flat :class:`cudf.Series`."""
|
|
212
|
+
if not self.is_cuda:
|
|
213
|
+
raise RuntimeError(
|
|
214
|
+
f"Expected tensor to be on a CUDA device (got '{self.device}')"
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
tensor = cast(StringTensor, self.contiguous())
|
|
218
|
+
|
|
219
|
+
with torch.cuda.device(self.device):
|
|
220
|
+
import cudf
|
|
221
|
+
import pylibcudf as plc
|
|
222
|
+
|
|
223
|
+
mask = None
|
|
224
|
+
null_count = 0
|
|
225
|
+
start = int(tensor.storage_offset())
|
|
226
|
+
if tensor._valid is not None:
|
|
227
|
+
mask = cudf.Series(tensor._valid, copy=False)._column.as_mask()
|
|
228
|
+
if isinstance(mask, tuple):
|
|
229
|
+
mask = mask[0]
|
|
230
|
+
null_count = plc.null_mask.null_count(
|
|
231
|
+
mask, start, start + tensor.numel()
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
plc_column = plc.Column(
|
|
235
|
+
data_type=plc.DataType(plc.TypeId.STRING),
|
|
236
|
+
size=tensor.numel(),
|
|
237
|
+
data=plc.gpumemoryview(tensor._data),
|
|
238
|
+
mask=mask,
|
|
239
|
+
null_count=null_count,
|
|
240
|
+
offset=start,
|
|
241
|
+
children=[plc.Column.from_array(obj=tensor._offset)],
|
|
242
|
+
)
|
|
243
|
+
return cudf.Series.from_pylibcudf(plc_column)
|
|
244
|
+
|
|
245
|
+
@classmethod
|
|
246
|
+
@override
|
|
247
|
+
def from_list(
|
|
248
|
+
cls,
|
|
249
|
+
values: str | Sequence[Any] | None,
|
|
250
|
+
*,
|
|
251
|
+
dtype: torch.dtype | None = None,
|
|
252
|
+
device: torch.device | str | None = None,
|
|
253
|
+
offset_dtype: torch.dtype = torch.int64,
|
|
254
|
+
) -> Self:
|
|
255
|
+
r"""Create tensor from a rectangular Python list of strings.
|
|
256
|
+
|
|
257
|
+
.. testcode::
|
|
258
|
+
|
|
259
|
+
from sdm import StringTensor
|
|
260
|
+
|
|
261
|
+
tensor = StringTensor.from_list([
|
|
262
|
+
["foo", "bar"],
|
|
263
|
+
["hello world", ""],
|
|
264
|
+
[None, "xzy"],
|
|
265
|
+
])
|
|
266
|
+
|
|
267
|
+
Args:
|
|
268
|
+
values: The rectangular Python list of strings.
|
|
269
|
+
dtype: The dtype of the ``value`` tensor.
|
|
270
|
+
device: The device.
|
|
271
|
+
offset_dtype: The dtype of the ``offset`` tensor.
|
|
272
|
+
"""
|
|
273
|
+
if dtype is not None and dtype != torch.uint8:
|
|
274
|
+
raise ValueError(
|
|
275
|
+
f"Expected 'dtype' in '{cls.__name__}.from_list' to be "
|
|
276
|
+
f"'torch.uint8' (got '{dtype}')"
|
|
277
|
+
)
|
|
278
|
+
if offset_dtype not in (torch.int32, torch.int64):
|
|
279
|
+
raise ValueError(
|
|
280
|
+
f"Expected 'offset_dtype' in '{cls.__name__}.from_list' "
|
|
281
|
+
f"to be 'torch.int32' or 'torch.int64' "
|
|
282
|
+
f"(got '{offset_dtype}')"
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
data: list[str | None] = []
|
|
286
|
+
|
|
287
|
+
def flatten(value: Any) -> tuple[int, ...]:
|
|
288
|
+
if value is None or isinstance(value, str):
|
|
289
|
+
data.append(value)
|
|
290
|
+
return ()
|
|
291
|
+
|
|
292
|
+
if not isinstance(value, Sequence):
|
|
293
|
+
raise ValueError(f"{cls.__name__!r} data must be rectangular")
|
|
294
|
+
|
|
295
|
+
if len(value) == 0:
|
|
296
|
+
return (0,)
|
|
297
|
+
|
|
298
|
+
if not isinstance(value[0], Sequence) or isinstance(value[0], str):
|
|
299
|
+
data.extend(value)
|
|
300
|
+
return (len(value),)
|
|
301
|
+
|
|
302
|
+
child_size: tuple[int, ...] | None = None
|
|
303
|
+
for item in value:
|
|
304
|
+
item_size = flatten(item)
|
|
305
|
+
if child_size is None:
|
|
306
|
+
child_size = item_size
|
|
307
|
+
elif item_size != child_size:
|
|
308
|
+
raise ValueError(
|
|
309
|
+
f"{cls.__name__!r} data must be rectangular"
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
assert child_size is not None
|
|
313
|
+
return (len(value), *child_size)
|
|
314
|
+
|
|
315
|
+
size = flatten(values)
|
|
316
|
+
|
|
317
|
+
pa_type = pa.large_string()
|
|
318
|
+
if offset_dtype == torch.int32:
|
|
319
|
+
pa_type = pa.string()
|
|
320
|
+
|
|
321
|
+
return cls.from_arrow(
|
|
322
|
+
array=pa.array(data, type=pa_type),
|
|
323
|
+
device=device,
|
|
324
|
+
size=size,
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
@override
|
|
328
|
+
def item(self) -> str | None: # type: ignore
|
|
329
|
+
return cast(str | None, super().item())
|
|
330
|
+
|
|
331
|
+
def __eq__(self, other: object) -> Tensor: # type: ignore
|
|
332
|
+
if isinstance(other, str):
|
|
333
|
+
return _eq(self, other)
|
|
334
|
+
return cast(Tensor, super().__eq__(other))
|
|
335
|
+
|
|
336
|
+
def __ne__(self, other: object) -> Tensor: # type: ignore
|
|
337
|
+
if isinstance(other, str):
|
|
338
|
+
return _ne(self, other)
|
|
339
|
+
return cast(Tensor, super().__ne__(other))
|
|
340
|
+
|
|
341
|
+
def __repr__(self, *, tensor_contents: Any = None) -> str:
|
|
342
|
+
# TODO Support tensor content printing.
|
|
343
|
+
out = f"{self.__class__.__name__}("
|
|
344
|
+
out += f"size={tuple(self.size())}"
|
|
345
|
+
if self.valid is not None:
|
|
346
|
+
out += f", null_count={int((~self.valid).sum())}"
|
|
347
|
+
if not self.is_cpu:
|
|
348
|
+
out += f", device={self.device}"
|
|
349
|
+
out += ")"
|
|
350
|
+
return out
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
@StringTensor.implements(aten.eq.Tensor)
|
|
354
|
+
@StringTensor.implements(aten.eq.str)
|
|
355
|
+
def _eq(inp: StringTensor, other: Tensor | str) -> Tensor:
|
|
356
|
+
if isinstance(other, Tensor) and inp.device != other.device:
|
|
357
|
+
raise RuntimeError(
|
|
358
|
+
f"Expected both tensors to be on the same device "
|
|
359
|
+
f"(got '{inp.device}' and '{other.device}')"
|
|
360
|
+
)
|
|
361
|
+
|
|
362
|
+
if not isinstance(other, StringTensor | str):
|
|
363
|
+
return torch.zeros(
|
|
364
|
+
torch.broadcast_shapes(inp.size(), other.size()),
|
|
365
|
+
dtype=torch.bool,
|
|
366
|
+
device=inp.device,
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
if isinstance(other, Tensor):
|
|
370
|
+
size = torch.broadcast_shapes(inp.size(), other.size())
|
|
371
|
+
inp = cast(StringTensor, inp.expand(size))
|
|
372
|
+
other = cast(StringTensor, other.expand(size))
|
|
373
|
+
else:
|
|
374
|
+
size = inp.size()
|
|
375
|
+
|
|
376
|
+
backend: Literal["arrow", "cudf"] = "arrow"
|
|
377
|
+
if inp.is_cuda:
|
|
378
|
+
if importlib.util.find_spec("cudf") is not None:
|
|
379
|
+
backend = "cudf"
|
|
380
|
+
else:
|
|
381
|
+
warn_once(
|
|
382
|
+
key="missing-cudf-eq",
|
|
383
|
+
message=(
|
|
384
|
+
"Falling back to a CPU-based string comparison because "
|
|
385
|
+
"cuDF is not installed. Install cuDF to enable faster "
|
|
386
|
+
"CUDA-based string comparisons without device "
|
|
387
|
+
"synchronization."
|
|
388
|
+
),
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
if backend == "arrow":
|
|
392
|
+
out = pc.call_function(
|
|
393
|
+
"equal",
|
|
394
|
+
[
|
|
395
|
+
inp.to_arrow(),
|
|
396
|
+
other.to_arrow() if isinstance(other, StringTensor) else other,
|
|
397
|
+
],
|
|
398
|
+
)
|
|
399
|
+
if out.null_count > 0:
|
|
400
|
+
out = out.fill_null(False)
|
|
401
|
+
mask = arrow_as_tensor(out, dtype=torch.bool, device=inp.device)
|
|
402
|
+
else:
|
|
403
|
+
assert backend == "cudf"
|
|
404
|
+
with torch.cuda.device(inp.device):
|
|
405
|
+
if isinstance(other, StringTensor):
|
|
406
|
+
out = inp.to_cudf() == other.to_cudf()
|
|
407
|
+
else:
|
|
408
|
+
out = inp.to_cudf() == other
|
|
409
|
+
if out.hasnans:
|
|
410
|
+
out = out.fillna(False)
|
|
411
|
+
mask = torch.from_dlpack(out.to_cupy()).view(size)
|
|
412
|
+
|
|
413
|
+
return mask.view(size)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
@StringTensor.implements(aten.ne.Tensor)
|
|
417
|
+
@StringTensor.implements(aten.ne.str)
|
|
418
|
+
def _ne(inp: StringTensor, other: Tensor | str) -> Tensor:
|
|
419
|
+
if isinstance(other, Tensor) and inp.device != other.device:
|
|
420
|
+
raise RuntimeError(
|
|
421
|
+
f"Expected both tensors to be on the same device "
|
|
422
|
+
f"(got '{inp.device}' and '{other.device}')"
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
if not isinstance(other, StringTensor | str):
|
|
426
|
+
return torch.ones(
|
|
427
|
+
torch.broadcast_shapes(inp.size(), other.size()),
|
|
428
|
+
dtype=torch.bool,
|
|
429
|
+
device=inp.device,
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
if isinstance(other, Tensor):
|
|
433
|
+
size = torch.broadcast_shapes(inp.size(), other.size())
|
|
434
|
+
inp = cast(StringTensor, inp.expand(size))
|
|
435
|
+
other = cast(StringTensor, other.expand(size))
|
|
436
|
+
else:
|
|
437
|
+
size = inp.size()
|
|
438
|
+
|
|
439
|
+
backend: Literal["arrow", "cudf"] = "arrow"
|
|
440
|
+
if inp.is_cuda:
|
|
441
|
+
if importlib.util.find_spec("cudf") is not None:
|
|
442
|
+
backend = "cudf"
|
|
443
|
+
else:
|
|
444
|
+
warn_once(
|
|
445
|
+
key="missing-cudf-eq",
|
|
446
|
+
message=(
|
|
447
|
+
"Falling back to a CPU-based string comparison because "
|
|
448
|
+
"cuDF is not installed. Install cuDF to enable faster "
|
|
449
|
+
"CUDA-based string comparisons without device "
|
|
450
|
+
"synchronization."
|
|
451
|
+
),
|
|
452
|
+
)
|
|
453
|
+
|
|
454
|
+
if backend == "arrow":
|
|
455
|
+
out = pc.call_function(
|
|
456
|
+
"not_equal",
|
|
457
|
+
[
|
|
458
|
+
inp.to_arrow(),
|
|
459
|
+
other.to_arrow() if isinstance(other, StringTensor) else other,
|
|
460
|
+
],
|
|
461
|
+
)
|
|
462
|
+
if out.null_count > 0:
|
|
463
|
+
out = out.fill_null(False)
|
|
464
|
+
mask = arrow_as_tensor(out, dtype=torch.bool, device=inp.device)
|
|
465
|
+
else:
|
|
466
|
+
assert backend == "cudf"
|
|
467
|
+
with torch.cuda.device(inp.device):
|
|
468
|
+
if isinstance(other, StringTensor):
|
|
469
|
+
out = inp.to_cudf() != other.to_cudf()
|
|
470
|
+
else:
|
|
471
|
+
out = inp.to_cudf() != other
|
|
472
|
+
if out.hasnans:
|
|
473
|
+
out = out.fillna(False)
|
|
474
|
+
mask = torch.from_dlpack(out.to_cupy()).view(size)
|
|
475
|
+
|
|
476
|
+
return mask.view(size)
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
@StringTensor.implements(aten.sort.default)
|
|
480
|
+
@StringTensor.implements(aten.sort.stable)
|
|
481
|
+
def _sort(
|
|
482
|
+
inp: StringTensor,
|
|
483
|
+
dim: int = -1,
|
|
484
|
+
descending: bool = False,
|
|
485
|
+
*,
|
|
486
|
+
stable: bool | None = None,
|
|
487
|
+
) -> tuple[StringTensor, Tensor]:
|
|
488
|
+
if dim < -inp.dim() or dim >= inp.dim():
|
|
489
|
+
raise IndexError(
|
|
490
|
+
f"Dimension out of range (expected to be in range of "
|
|
491
|
+
f"[{-inp.dim()}, {inp.dim() - 1}], but got {dim})"
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
if inp.dim() != 1:
|
|
495
|
+
raise NotImplementedError("'sort' only supports one-dimensional input")
|
|
496
|
+
|
|
497
|
+
backend: Literal["arrow", "cudf"] = "arrow"
|
|
498
|
+
if inp.is_cuda:
|
|
499
|
+
if importlib.util.find_spec("cudf") is not None:
|
|
500
|
+
backend = "cudf"
|
|
501
|
+
else:
|
|
502
|
+
warn_once(
|
|
503
|
+
key="missing-cudf-sort",
|
|
504
|
+
message=(
|
|
505
|
+
"Falling back to a CPU-based string sort because cuDF is "
|
|
506
|
+
"not installed. Install cuDF to enable faster CUDA-based "
|
|
507
|
+
"string sorting without device synchronization."
|
|
508
|
+
),
|
|
509
|
+
)
|
|
510
|
+
|
|
511
|
+
if backend == "arrow":
|
|
512
|
+
out = pc.call_function(
|
|
513
|
+
"array_sort_indices",
|
|
514
|
+
[inp.to_arrow()],
|
|
515
|
+
options=pc.ArraySortOptions(
|
|
516
|
+
order="descending" if descending else "ascending",
|
|
517
|
+
),
|
|
518
|
+
)
|
|
519
|
+
perm = arrow_as_tensor(out, dtype=torch.int64, device=inp.device)
|
|
520
|
+
else:
|
|
521
|
+
assert backend == "cudf"
|
|
522
|
+
with torch.cuda.device(inp.device):
|
|
523
|
+
perm_ser = inp.to_cudf().argsort(ascending=not descending)
|
|
524
|
+
perm = torch.from_dlpack(perm_ser.astype("int64").to_cupy())
|
|
525
|
+
|
|
526
|
+
return cast(StringTensor, inp[perm]), perm
|