structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/tensor/string.py ADDED
@@ -0,0 +1,526 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from __future__ import annotations
5
+
6
+ import importlib.util
7
+ import math
8
+ from collections.abc import Sequence
9
+ from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self, cast
10
+
11
+ import pyarrow as pa
12
+ import pyarrow.compute as pc
13
+ import torch
14
+ from torch import Tensor
15
+ from typing_extensions import override
16
+
17
+ from sdm._warnings import warn_once
18
+ from sdm.tensor import VarLenTensor
19
+ from sdm.tensor.io import arrow_as_tensor, combine_arrow_chunks
20
+
21
+ if TYPE_CHECKING:
22
+ import cudf
23
+
24
+ aten = torch.ops.aten
25
+
26
+
27
+ class StringTensor(VarLenTensor):
28
+ r"""A :class:`torch.Tensor` for UTF-8 encoded string values.
29
+
30
+ Args:
31
+ data: Flat contiguous ``uint8`` tensor containing all string values.
32
+ offset: One-dimensional offsets into ``data``.
33
+ valid: One-dimensional mask indicating valid, non-null string values.
34
+ size: The shape of the tensor.
35
+ stride: The stride of the tensor.
36
+ storage_offset: The offset into the logical ``offset`` storage.
37
+ """
38
+
39
+ # NOTE Assume that `data` stores valid UTF-8 bytes and do not validate it.
40
+ ALLOWED_DTYPES: ClassVar[tuple[torch.dtype, ...] | None] = (torch.uint8,)
41
+ __hash__ = Tensor.__hash__
42
+
43
+ @override
44
+ @classmethod
45
+ def from_arrow(
46
+ cls,
47
+ array: pa.Array | pa.ChunkedArray,
48
+ *,
49
+ size: Sequence[int] | None = None,
50
+ device: torch.device | str | None = None,
51
+ ) -> Self:
52
+ r"""Create tensor from a string :class:`pyarrow.Array`.
53
+
54
+ .. testcode::
55
+
56
+ import pyarrow as pa
57
+ from sdm import StringTensor
58
+
59
+ array = pa.array(["foo", "bar", "hello world", ""])
60
+ tensor = StringTensor.from_arrow(array, size=(2, 2))
61
+
62
+ Args:
63
+ array: The string :class:`pyarrow.Array` or
64
+ :class:`pyarrow.ChunkedArray`.
65
+ size: The shape of the tensor.
66
+ device: The device.
67
+ """
68
+ if isinstance(array, pa.ChunkedArray):
69
+ array = combine_arrow_chunks(array)
70
+
71
+ if size is None:
72
+ size = (len(array),)
73
+ elif math.prod(size) != len(array):
74
+ raise ValueError(
75
+ f"Expected 'size' in '{cls.__name__}.from_arrow' to contain "
76
+ f"{len(array)} elements (got {math.prod(size)})"
77
+ )
78
+
79
+ is_string = pa.types.is_string(array.type)
80
+ is_large_string = pa.types.is_large_string(array.type)
81
+ if not is_string and not is_large_string:
82
+ raise TypeError(
83
+ f"Expected 'array' in '{cls.__name__}.from_arrow' to have "
84
+ f"'string' or 'large_string' type (got '{array.type}')"
85
+ )
86
+
87
+ buffers = array.buffers()
88
+ offset = torch.frombuffer(
89
+ buffers[1],
90
+ dtype=torch.int32 if is_string else torch.int64,
91
+ )
92
+ valid: Tensor | None = None
93
+ storage_offset = array.offset
94
+ if array.null_count > 0:
95
+ offset = offset[array.offset : array.offset + len(array) + 1]
96
+ valid = arrow_as_tensor(
97
+ array.is_valid(),
98
+ dtype=torch.bool,
99
+ device=device,
100
+ )
101
+ storage_offset = 0
102
+
103
+ return cls(
104
+ data=torch.frombuffer(buffers[2], dtype=torch.uint8).to(device)
105
+ if buffers[2] is not None and buffers[2].size > 0
106
+ else torch.empty(0, dtype=torch.uint8, device=device),
107
+ offset=offset.to(device),
108
+ valid=valid,
109
+ size=size,
110
+ storage_offset=storage_offset,
111
+ )
112
+
113
+ @override
114
+ def to_arrow(self) -> pa.Array:
115
+ r"""Convert this tensor to a flat :class:`pyarrow.Array`."""
116
+ tensor = cast(StringTensor, self.contiguous().cpu())
117
+
118
+ return pa.Array.from_buffers(
119
+ pa.string()
120
+ if tensor._offset.dtype == torch.int32
121
+ else pa.large_string(),
122
+ length=tensor.numel(),
123
+ buffers=[
124
+ pa.array(tensor._valid.numpy(), type=pa.bool_()).buffers()[1]
125
+ if tensor._valid is not None
126
+ else None,
127
+ pa.py_buffer(tensor._offset.numpy()),
128
+ pa.py_buffer(tensor._data.numpy()),
129
+ ],
130
+ null_count=-1,
131
+ offset=int(tensor.storage_offset()),
132
+ )
133
+
134
+ @classmethod
135
+ def from_cudf(
136
+ cls,
137
+ ser: cudf.Series | cudf.Index,
138
+ *,
139
+ size: Sequence[int] | None = None,
140
+ device: torch.device | str | None = None,
141
+ ) -> Self:
142
+ r"""Create tensor from a string :class:`cudf.Series`.
143
+
144
+ Args:
145
+ ser: The string :class:`cudf.Series` or :class:`cudf.Index`.
146
+ size: The shape of the tensor.
147
+ device: The device.
148
+ """
149
+ import cupy as cp
150
+ import pylibcudf as plc
151
+ from cudf.api.types import is_string_dtype
152
+
153
+ if size is None:
154
+ size = (len(ser),)
155
+ elif math.prod(size) != len(ser):
156
+ raise ValueError(
157
+ f"Expected 'size' in '{cls.__name__}.from_cudf' to contain "
158
+ f"{len(ser)} elements (got {math.prod(size)})"
159
+ )
160
+
161
+ if not is_string_dtype(ser.dtype):
162
+ raise TypeError(
163
+ f"Expected 'values' in '{cls.__name__}.from_cudf' to have "
164
+ f"string type (got '{ser.dtype}')"
165
+ )
166
+
167
+ # `Series.to_pylibcudf` returns a zero-copy Arrow-style view: base
168
+ # character/offset buffers plus a row offset into the offsets.
169
+ column, _ = ser.to_pylibcudf()
170
+
171
+ # `None` or zero-length when the column holds no characters:
172
+ chars = column.data()
173
+ data = torch.from_dlpack(
174
+ cp.asarray(chars) if chars is not None else cp.empty(0, cp.uint8)
175
+ ).to(device)
176
+
177
+ if len(ser) == 0:
178
+ return cls(
179
+ data=data,
180
+ offset=torch.zeros(1, dtype=torch.int32, device=data.device),
181
+ valid=None,
182
+ size=size,
183
+ )
184
+
185
+ offsets = column.children()[0] # (base_size + 1,) INT32/INT64 values
186
+ offset_dtype = (
187
+ cp.int32
188
+ if offsets.type().id() == plc.types.TypeId.INT32
189
+ else cp.int64
190
+ )
191
+ offset = torch.from_dlpack(
192
+ cp.asarray(offsets.data()).view(offset_dtype)
193
+ )
194
+
195
+ valid: Tensor | None = None
196
+ storage_offset = column.offset()
197
+ if ser.hasnans:
198
+ offset = offset[column.offset() : column.offset() + len(ser) + 1]
199
+ valid = torch.from_dlpack(ser.notnull().to_cupy()).to(device)
200
+ storage_offset = 0
201
+
202
+ return cls(
203
+ data=data,
204
+ offset=offset.to(device),
205
+ valid=valid,
206
+ size=size,
207
+ storage_offset=storage_offset,
208
+ )
209
+
210
+ def to_cudf(self) -> cudf.Series:
211
+ r"""Convert this CUDA tensor to a flat :class:`cudf.Series`."""
212
+ if not self.is_cuda:
213
+ raise RuntimeError(
214
+ f"Expected tensor to be on a CUDA device (got '{self.device}')"
215
+ )
216
+
217
+ tensor = cast(StringTensor, self.contiguous())
218
+
219
+ with torch.cuda.device(self.device):
220
+ import cudf
221
+ import pylibcudf as plc
222
+
223
+ mask = None
224
+ null_count = 0
225
+ start = int(tensor.storage_offset())
226
+ if tensor._valid is not None:
227
+ mask = cudf.Series(tensor._valid, copy=False)._column.as_mask()
228
+ if isinstance(mask, tuple):
229
+ mask = mask[0]
230
+ null_count = plc.null_mask.null_count(
231
+ mask, start, start + tensor.numel()
232
+ )
233
+
234
+ plc_column = plc.Column(
235
+ data_type=plc.DataType(plc.TypeId.STRING),
236
+ size=tensor.numel(),
237
+ data=plc.gpumemoryview(tensor._data),
238
+ mask=mask,
239
+ null_count=null_count,
240
+ offset=start,
241
+ children=[plc.Column.from_array(obj=tensor._offset)],
242
+ )
243
+ return cudf.Series.from_pylibcudf(plc_column)
244
+
245
+ @classmethod
246
+ @override
247
+ def from_list(
248
+ cls,
249
+ values: str | Sequence[Any] | None,
250
+ *,
251
+ dtype: torch.dtype | None = None,
252
+ device: torch.device | str | None = None,
253
+ offset_dtype: torch.dtype = torch.int64,
254
+ ) -> Self:
255
+ r"""Create tensor from a rectangular Python list of strings.
256
+
257
+ .. testcode::
258
+
259
+ from sdm import StringTensor
260
+
261
+ tensor = StringTensor.from_list([
262
+ ["foo", "bar"],
263
+ ["hello world", ""],
264
+ [None, "xzy"],
265
+ ])
266
+
267
+ Args:
268
+ values: The rectangular Python list of strings.
269
+ dtype: The dtype of the ``value`` tensor.
270
+ device: The device.
271
+ offset_dtype: The dtype of the ``offset`` tensor.
272
+ """
273
+ if dtype is not None and dtype != torch.uint8:
274
+ raise ValueError(
275
+ f"Expected 'dtype' in '{cls.__name__}.from_list' to be "
276
+ f"'torch.uint8' (got '{dtype}')"
277
+ )
278
+ if offset_dtype not in (torch.int32, torch.int64):
279
+ raise ValueError(
280
+ f"Expected 'offset_dtype' in '{cls.__name__}.from_list' "
281
+ f"to be 'torch.int32' or 'torch.int64' "
282
+ f"(got '{offset_dtype}')"
283
+ )
284
+
285
+ data: list[str | None] = []
286
+
287
+ def flatten(value: Any) -> tuple[int, ...]:
288
+ if value is None or isinstance(value, str):
289
+ data.append(value)
290
+ return ()
291
+
292
+ if not isinstance(value, Sequence):
293
+ raise ValueError(f"{cls.__name__!r} data must be rectangular")
294
+
295
+ if len(value) == 0:
296
+ return (0,)
297
+
298
+ if not isinstance(value[0], Sequence) or isinstance(value[0], str):
299
+ data.extend(value)
300
+ return (len(value),)
301
+
302
+ child_size: tuple[int, ...] | None = None
303
+ for item in value:
304
+ item_size = flatten(item)
305
+ if child_size is None:
306
+ child_size = item_size
307
+ elif item_size != child_size:
308
+ raise ValueError(
309
+ f"{cls.__name__!r} data must be rectangular"
310
+ )
311
+
312
+ assert child_size is not None
313
+ return (len(value), *child_size)
314
+
315
+ size = flatten(values)
316
+
317
+ pa_type = pa.large_string()
318
+ if offset_dtype == torch.int32:
319
+ pa_type = pa.string()
320
+
321
+ return cls.from_arrow(
322
+ array=pa.array(data, type=pa_type),
323
+ device=device,
324
+ size=size,
325
+ )
326
+
327
+ @override
328
+ def item(self) -> str | None: # type: ignore
329
+ return cast(str | None, super().item())
330
+
331
+ def __eq__(self, other: object) -> Tensor: # type: ignore
332
+ if isinstance(other, str):
333
+ return _eq(self, other)
334
+ return cast(Tensor, super().__eq__(other))
335
+
336
+ def __ne__(self, other: object) -> Tensor: # type: ignore
337
+ if isinstance(other, str):
338
+ return _ne(self, other)
339
+ return cast(Tensor, super().__ne__(other))
340
+
341
+ def __repr__(self, *, tensor_contents: Any = None) -> str:
342
+ # TODO Support tensor content printing.
343
+ out = f"{self.__class__.__name__}("
344
+ out += f"size={tuple(self.size())}"
345
+ if self.valid is not None:
346
+ out += f", null_count={int((~self.valid).sum())}"
347
+ if not self.is_cpu:
348
+ out += f", device={self.device}"
349
+ out += ")"
350
+ return out
351
+
352
+
353
+ @StringTensor.implements(aten.eq.Tensor)
354
+ @StringTensor.implements(aten.eq.str)
355
+ def _eq(inp: StringTensor, other: Tensor | str) -> Tensor:
356
+ if isinstance(other, Tensor) and inp.device != other.device:
357
+ raise RuntimeError(
358
+ f"Expected both tensors to be on the same device "
359
+ f"(got '{inp.device}' and '{other.device}')"
360
+ )
361
+
362
+ if not isinstance(other, StringTensor | str):
363
+ return torch.zeros(
364
+ torch.broadcast_shapes(inp.size(), other.size()),
365
+ dtype=torch.bool,
366
+ device=inp.device,
367
+ )
368
+
369
+ if isinstance(other, Tensor):
370
+ size = torch.broadcast_shapes(inp.size(), other.size())
371
+ inp = cast(StringTensor, inp.expand(size))
372
+ other = cast(StringTensor, other.expand(size))
373
+ else:
374
+ size = inp.size()
375
+
376
+ backend: Literal["arrow", "cudf"] = "arrow"
377
+ if inp.is_cuda:
378
+ if importlib.util.find_spec("cudf") is not None:
379
+ backend = "cudf"
380
+ else:
381
+ warn_once(
382
+ key="missing-cudf-eq",
383
+ message=(
384
+ "Falling back to a CPU-based string comparison because "
385
+ "cuDF is not installed. Install cuDF to enable faster "
386
+ "CUDA-based string comparisons without device "
387
+ "synchronization."
388
+ ),
389
+ )
390
+
391
+ if backend == "arrow":
392
+ out = pc.call_function(
393
+ "equal",
394
+ [
395
+ inp.to_arrow(),
396
+ other.to_arrow() if isinstance(other, StringTensor) else other,
397
+ ],
398
+ )
399
+ if out.null_count > 0:
400
+ out = out.fill_null(False)
401
+ mask = arrow_as_tensor(out, dtype=torch.bool, device=inp.device)
402
+ else:
403
+ assert backend == "cudf"
404
+ with torch.cuda.device(inp.device):
405
+ if isinstance(other, StringTensor):
406
+ out = inp.to_cudf() == other.to_cudf()
407
+ else:
408
+ out = inp.to_cudf() == other
409
+ if out.hasnans:
410
+ out = out.fillna(False)
411
+ mask = torch.from_dlpack(out.to_cupy()).view(size)
412
+
413
+ return mask.view(size)
414
+
415
+
416
+ @StringTensor.implements(aten.ne.Tensor)
417
+ @StringTensor.implements(aten.ne.str)
418
+ def _ne(inp: StringTensor, other: Tensor | str) -> Tensor:
419
+ if isinstance(other, Tensor) and inp.device != other.device:
420
+ raise RuntimeError(
421
+ f"Expected both tensors to be on the same device "
422
+ f"(got '{inp.device}' and '{other.device}')"
423
+ )
424
+
425
+ if not isinstance(other, StringTensor | str):
426
+ return torch.ones(
427
+ torch.broadcast_shapes(inp.size(), other.size()),
428
+ dtype=torch.bool,
429
+ device=inp.device,
430
+ )
431
+
432
+ if isinstance(other, Tensor):
433
+ size = torch.broadcast_shapes(inp.size(), other.size())
434
+ inp = cast(StringTensor, inp.expand(size))
435
+ other = cast(StringTensor, other.expand(size))
436
+ else:
437
+ size = inp.size()
438
+
439
+ backend: Literal["arrow", "cudf"] = "arrow"
440
+ if inp.is_cuda:
441
+ if importlib.util.find_spec("cudf") is not None:
442
+ backend = "cudf"
443
+ else:
444
+ warn_once(
445
+ key="missing-cudf-eq",
446
+ message=(
447
+ "Falling back to a CPU-based string comparison because "
448
+ "cuDF is not installed. Install cuDF to enable faster "
449
+ "CUDA-based string comparisons without device "
450
+ "synchronization."
451
+ ),
452
+ )
453
+
454
+ if backend == "arrow":
455
+ out = pc.call_function(
456
+ "not_equal",
457
+ [
458
+ inp.to_arrow(),
459
+ other.to_arrow() if isinstance(other, StringTensor) else other,
460
+ ],
461
+ )
462
+ if out.null_count > 0:
463
+ out = out.fill_null(False)
464
+ mask = arrow_as_tensor(out, dtype=torch.bool, device=inp.device)
465
+ else:
466
+ assert backend == "cudf"
467
+ with torch.cuda.device(inp.device):
468
+ if isinstance(other, StringTensor):
469
+ out = inp.to_cudf() != other.to_cudf()
470
+ else:
471
+ out = inp.to_cudf() != other
472
+ if out.hasnans:
473
+ out = out.fillna(False)
474
+ mask = torch.from_dlpack(out.to_cupy()).view(size)
475
+
476
+ return mask.view(size)
477
+
478
+
479
+ @StringTensor.implements(aten.sort.default)
480
+ @StringTensor.implements(aten.sort.stable)
481
+ def _sort(
482
+ inp: StringTensor,
483
+ dim: int = -1,
484
+ descending: bool = False,
485
+ *,
486
+ stable: bool | None = None,
487
+ ) -> tuple[StringTensor, Tensor]:
488
+ if dim < -inp.dim() or dim >= inp.dim():
489
+ raise IndexError(
490
+ f"Dimension out of range (expected to be in range of "
491
+ f"[{-inp.dim()}, {inp.dim() - 1}], but got {dim})"
492
+ )
493
+
494
+ if inp.dim() != 1:
495
+ raise NotImplementedError("'sort' only supports one-dimensional input")
496
+
497
+ backend: Literal["arrow", "cudf"] = "arrow"
498
+ if inp.is_cuda:
499
+ if importlib.util.find_spec("cudf") is not None:
500
+ backend = "cudf"
501
+ else:
502
+ warn_once(
503
+ key="missing-cudf-sort",
504
+ message=(
505
+ "Falling back to a CPU-based string sort because cuDF is "
506
+ "not installed. Install cuDF to enable faster CUDA-based "
507
+ "string sorting without device synchronization."
508
+ ),
509
+ )
510
+
511
+ if backend == "arrow":
512
+ out = pc.call_function(
513
+ "array_sort_indices",
514
+ [inp.to_arrow()],
515
+ options=pc.ArraySortOptions(
516
+ order="descending" if descending else "ascending",
517
+ ),
518
+ )
519
+ perm = arrow_as_tensor(out, dtype=torch.int64, device=inp.device)
520
+ else:
521
+ assert backend == "cudf"
522
+ with torch.cuda.device(inp.device):
523
+ perm_ser = inp.to_cudf().argsort(ascending=not descending)
524
+ perm = torch.from_dlpack(perm_ser.astype("int64").to_cupy())
525
+
526
+ return cast(StringTensor, inp[perm]), perm