structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/tensor/table.py
ADDED
|
@@ -0,0 +1,2035 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import functools
|
|
7
|
+
import math
|
|
8
|
+
import warnings
|
|
9
|
+
from collections import defaultdict
|
|
10
|
+
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from itertools import chain
|
|
13
|
+
from typing import TYPE_CHECKING, Any, ClassVar, Self, SupportsIndex, cast
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import pyarrow as pa
|
|
17
|
+
import torch
|
|
18
|
+
from torch import Tensor
|
|
19
|
+
from typing_extensions import override
|
|
20
|
+
|
|
21
|
+
from sdm import NaT, Stype, StypeLike
|
|
22
|
+
from sdm.tensor import CategoricalTensor, ColumnarTensor, StringTensor
|
|
23
|
+
from sdm.tensor.io import arrow_as_tensor, to_arrow, to_cudf
|
|
24
|
+
from sdm.tensor.mixin import _resolve_device
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
import cudf
|
|
28
|
+
import pandas as pd
|
|
29
|
+
|
|
30
|
+
aten = torch.ops.aten
|
|
31
|
+
|
|
32
|
+
_NUMPY_TORCH_DTYPES = {
|
|
33
|
+
np.dtype(np.float16): torch.float16,
|
|
34
|
+
np.dtype(np.float32): torch.float32,
|
|
35
|
+
np.dtype(np.float64): torch.float64,
|
|
36
|
+
}
|
|
37
|
+
_TORCH_NUMPY_DTYPES = {
|
|
38
|
+
torch_dtype: numpy_dtype
|
|
39
|
+
for numpy_dtype, torch_dtype in _NUMPY_TORCH_DTYPES.items()
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def preserve_view_inference_mode(fn: Callable) -> Callable:
|
|
44
|
+
r"""Preserve input inference state for tensor view operations."""
|
|
45
|
+
|
|
46
|
+
@functools.wraps(fn)
|
|
47
|
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
48
|
+
with torch.inference_mode(args[0].is_inference()):
|
|
49
|
+
return fn(*args, **kwargs)
|
|
50
|
+
|
|
51
|
+
return wrapper
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def preserve_autograd_state(fn: Callable) -> Callable:
|
|
55
|
+
r"""Re-apply the caller's autograd/inference-mode state for non-view ops.
|
|
56
|
+
|
|
57
|
+
``__torch_dispatch__`` redispatch does not automatically retain the
|
|
58
|
+
caller's inference-mode and grad-enabled state for non-view ops, so a
|
|
59
|
+
real (non-view) copy performed here would otherwise silently lose
|
|
60
|
+
``requires_grad``/``grad_fn`` even though gradients are enabled in the
|
|
61
|
+
caller's scope.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
@functools.wraps(fn)
|
|
65
|
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
66
|
+
inference_mode_enabled = torch.is_inference_mode_enabled()
|
|
67
|
+
grad_enabled = torch.is_grad_enabled()
|
|
68
|
+
with (
|
|
69
|
+
torch.inference_mode(inference_mode_enabled),
|
|
70
|
+
torch.set_grad_enabled(grad_enabled),
|
|
71
|
+
):
|
|
72
|
+
return fn(*args, **kwargs)
|
|
73
|
+
|
|
74
|
+
return wrapper
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _move_categories(
|
|
78
|
+
categories: Sequence[Tensor],
|
|
79
|
+
device: torch.device | str | None,
|
|
80
|
+
) -> tuple[Tensor, ...]:
|
|
81
|
+
device = _resolve_device(device)
|
|
82
|
+
if device is None or device.type == "cpu":
|
|
83
|
+
return tuple(categories)
|
|
84
|
+
|
|
85
|
+
groups: dict[
|
|
86
|
+
tuple[type[Tensor], torch.dtype, torch.dtype | None], list[int]
|
|
87
|
+
] = defaultdict(list)
|
|
88
|
+
moved: list[Tensor | None] = [None] * len(categories)
|
|
89
|
+
for i, category in enumerate(categories):
|
|
90
|
+
if type(category) is Tensor or isinstance(category, StringTensor):
|
|
91
|
+
offset_dtype = (
|
|
92
|
+
category.data_offset[1].dtype
|
|
93
|
+
if isinstance(category, StringTensor)
|
|
94
|
+
else None
|
|
95
|
+
)
|
|
96
|
+
groups[(type(category), category.dtype, offset_dtype)].append(i)
|
|
97
|
+
else:
|
|
98
|
+
moved[i] = category.to(device)
|
|
99
|
+
|
|
100
|
+
for indices in groups.values():
|
|
101
|
+
tensors = [categories[i] for i in indices]
|
|
102
|
+
if len(tensors) == 1:
|
|
103
|
+
parts = (tensors[0].to(device),)
|
|
104
|
+
else:
|
|
105
|
+
sizes = [tensor.numel() for tensor in tensors]
|
|
106
|
+
parts = torch.cat(tensors).to(device).split(sizes)
|
|
107
|
+
for i, part in zip(indices, parts):
|
|
108
|
+
moved[i] = part
|
|
109
|
+
|
|
110
|
+
assert all(category is not None for category in moved)
|
|
111
|
+
return cast(tuple[Tensor, ...], tuple(moved))
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@dataclass(frozen=True)
|
|
115
|
+
class TableSchema:
|
|
116
|
+
r"""The schema of a :class:`TableTensor`.
|
|
117
|
+
|
|
118
|
+
Args:
|
|
119
|
+
columns: Column names grouped by semantic type.
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
columns: Mapping[Stype, tuple[str, ...]]
|
|
123
|
+
|
|
124
|
+
def __hash__(self) -> int:
|
|
125
|
+
return hash(frozenset(self.columns.items()))
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class TableTensor(Tensor):
|
|
129
|
+
r"""A :class:`torch.Tensor` for tensorized, lossless table data.
|
|
130
|
+
|
|
131
|
+
A :class:`TableTensor` stores column blocks separately per semantic type,
|
|
132
|
+
while exposing a single tensor-shaped table interface.
|
|
133
|
+
The last dimension represents named columns.
|
|
134
|
+
|
|
135
|
+
.. testcode:: drop_stypes, select_columns, drop_columns
|
|
136
|
+
|
|
137
|
+
import torch
|
|
138
|
+
from sdm import CategoricalTensor, StringTensor, Stype, TableTensor
|
|
139
|
+
|
|
140
|
+
table = TableTensor(
|
|
141
|
+
columns={
|
|
142
|
+
"numerical": ["age", "income"],
|
|
143
|
+
"categorical": ["country", "segment"],
|
|
144
|
+
},
|
|
145
|
+
numerical=torch.randn(10, 2),
|
|
146
|
+
categorical=CategoricalTensor(
|
|
147
|
+
code=torch.randint(0, 2, size=(10, 2)),
|
|
148
|
+
categories=(
|
|
149
|
+
StringTensor.from_list(["USA", "Germany"]),
|
|
150
|
+
StringTensor.from_list(["enterprise", "startup"]),
|
|
151
|
+
),
|
|
152
|
+
),
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
print(table)
|
|
156
|
+
# TableTensor(
|
|
157
|
+
# size=(10, 4),
|
|
158
|
+
# blocks={
|
|
159
|
+
# numerical (2): ['age', 'income'],
|
|
160
|
+
# categorical (2): ['country', 'segment'],
|
|
161
|
+
# },
|
|
162
|
+
# )
|
|
163
|
+
|
|
164
|
+
# DataFrame-like column selection, but still tensor-native:
|
|
165
|
+
features = table[["age", "country"]]
|
|
166
|
+
assert features.size() == (10, 2)
|
|
167
|
+
|
|
168
|
+
# Normal PyTorch indexing still works on row/batch dimensions:
|
|
169
|
+
batch = table[[1, 0, 2], ["income", "segment"]]
|
|
170
|
+
assert batch.size() == (3, 2)
|
|
171
|
+
|
|
172
|
+
# Semantic blocks stay separate for model input:
|
|
173
|
+
x_num = table.numerical
|
|
174
|
+
x_cat = table.categorical
|
|
175
|
+
|
|
176
|
+
# Tensor ops preserve the table container:
|
|
177
|
+
stacked = torch.stack([table, table], dim=0)
|
|
178
|
+
assert stacked.size() == (2, 10, 4)
|
|
179
|
+
|
|
180
|
+
# Column-wise cat extends the schema:
|
|
181
|
+
wide = torch.cat([table[["age"]], table[["country"]]], dim=-1)
|
|
182
|
+
|
|
183
|
+
.. testoutput:: drop_stypes, select_columns, drop_columns
|
|
184
|
+
:hide:
|
|
185
|
+
:options: +ELLIPSIS
|
|
186
|
+
|
|
187
|
+
...
|
|
188
|
+
|
|
189
|
+
Args:
|
|
190
|
+
size: The shape of the tensor ``[..., C]``.
|
|
191
|
+
columns: Column names grouped by semantic type.
|
|
192
|
+
numerical: The numerical column block of shape ``[..., C_num]``.
|
|
193
|
+
categorical: The categorical column block of shape ``[..., C_cat]``.
|
|
194
|
+
datetime: The ``datetime64[us]`` column block of shape ``[..., C_dt]``.
|
|
195
|
+
text: The text column block of shape ``[..., C_text]``.
|
|
196
|
+
id: The identifier column block of shape ``[..., C_id]``.
|
|
197
|
+
device: The device.
|
|
198
|
+
"""
|
|
199
|
+
|
|
200
|
+
HANDLED_FUNCTIONS: ClassVar[
|
|
201
|
+
dict[Callable[..., Any], Callable[..., Any]]
|
|
202
|
+
] = {}
|
|
203
|
+
|
|
204
|
+
_numerical: Tensor
|
|
205
|
+
_categorical: CategoricalTensor
|
|
206
|
+
_datetime: Tensor
|
|
207
|
+
_text: StringTensor
|
|
208
|
+
_id: ColumnarTensor
|
|
209
|
+
_columns: dict[Stype, tuple[str, ...]]
|
|
210
|
+
_column_to_loc: dict[str, tuple[Stype, int]]
|
|
211
|
+
|
|
212
|
+
# Route tensor operations through `__torch_dispatch__` only.
|
|
213
|
+
__torch_function__ = torch._C._disabled_torch_function_impl # type: ignore
|
|
214
|
+
|
|
215
|
+
# Constructors ############################################################
|
|
216
|
+
|
|
217
|
+
def __init__(
|
|
218
|
+
self,
|
|
219
|
+
size: Sequence[int] | None = None,
|
|
220
|
+
columns: Mapping[StypeLike, Sequence[str]] | None = None,
|
|
221
|
+
numerical: Tensor | None = None,
|
|
222
|
+
categorical: CategoricalTensor | None = None,
|
|
223
|
+
datetime: Tensor | None = None,
|
|
224
|
+
text: StringTensor | None = None,
|
|
225
|
+
id: ColumnarTensor | None = None,
|
|
226
|
+
device: torch.device | str | None = None,
|
|
227
|
+
) -> None:
|
|
228
|
+
pass
|
|
229
|
+
|
|
230
|
+
def __new__(
|
|
231
|
+
cls,
|
|
232
|
+
size: Sequence[int] | None = None,
|
|
233
|
+
columns: Mapping[StypeLike, Sequence[str]] | None = None,
|
|
234
|
+
numerical: Tensor | None = None,
|
|
235
|
+
categorical: CategoricalTensor | None = None,
|
|
236
|
+
datetime: Tensor | None = None,
|
|
237
|
+
text: StringTensor | None = None,
|
|
238
|
+
id: ColumnarTensor | None = None,
|
|
239
|
+
device: torch.device | str | None = None,
|
|
240
|
+
) -> Self:
|
|
241
|
+
r"""Create a tensor wrapper."""
|
|
242
|
+
if size is not None and len(size) == 0:
|
|
243
|
+
raise ValueError("Expected 'size' to be non-empty")
|
|
244
|
+
|
|
245
|
+
size = tuple(size) if size is not None else size
|
|
246
|
+
device = _resolve_device(device)
|
|
247
|
+
|
|
248
|
+
for stype, block in (
|
|
249
|
+
(Stype.numerical, numerical),
|
|
250
|
+
(Stype.categorical, categorical),
|
|
251
|
+
(Stype.datetime, datetime),
|
|
252
|
+
(Stype.text, text),
|
|
253
|
+
(Stype.id, id),
|
|
254
|
+
):
|
|
255
|
+
if block is None:
|
|
256
|
+
continue
|
|
257
|
+
|
|
258
|
+
if stype == Stype.datetime and block.dtype != torch.int64:
|
|
259
|
+
raise ValueError(
|
|
260
|
+
f"Expected {str(stype)!r} block to have dtype "
|
|
261
|
+
f"'{torch.int64}' (got '{block.dtype}')"
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
size = tuple(block.size()[:-1]) if size is None else size
|
|
265
|
+
device = block.device if device is None else device
|
|
266
|
+
|
|
267
|
+
if block.dim() < 2:
|
|
268
|
+
raise ValueError(
|
|
269
|
+
f"Expected {str(stype)!r} block to be at least 2D "
|
|
270
|
+
f"(got {block.dim()}D)"
|
|
271
|
+
)
|
|
272
|
+
if size != block.size()[:-1]:
|
|
273
|
+
raise ValueError(
|
|
274
|
+
f"Expected {str(stype)!r} block size of "
|
|
275
|
+
f"{_block_size_repr(size)} (got {tuple(block.size())})"
|
|
276
|
+
)
|
|
277
|
+
if device != block.device:
|
|
278
|
+
raise ValueError(
|
|
279
|
+
f"Expected {str(stype)!r} block to be on device "
|
|
280
|
+
f"'{device}' (got '{block.device}')"
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
if size is None:
|
|
284
|
+
raise ValueError(
|
|
285
|
+
"Expected 'size' to be given when all blocks are 'None'"
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
if numerical is None:
|
|
289
|
+
numerical = torch.empty((*size, 0), device=device)
|
|
290
|
+
if categorical is None:
|
|
291
|
+
categorical = CategoricalTensor(
|
|
292
|
+
code=torch.empty((*size, 0), dtype=torch.int32, device=device),
|
|
293
|
+
categories=(),
|
|
294
|
+
)
|
|
295
|
+
if datetime is None:
|
|
296
|
+
datetime = torch.empty((*size, 0), dtype=torch.long, device=device)
|
|
297
|
+
if text is None:
|
|
298
|
+
text = StringTensor(
|
|
299
|
+
data=torch.empty(0, dtype=torch.uint8, device=device),
|
|
300
|
+
offset=torch.zeros(1, dtype=torch.int32, device=device),
|
|
301
|
+
valid=None,
|
|
302
|
+
size=(*size, 0),
|
|
303
|
+
)
|
|
304
|
+
if id is None:
|
|
305
|
+
id = ColumnarTensor((), size=size, device=device)
|
|
306
|
+
|
|
307
|
+
columns = {
|
|
308
|
+
Stype(stype): tuple(names)
|
|
309
|
+
for stype, names in (columns or {}).items()
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
for stype, block, name in (
|
|
313
|
+
(Stype.numerical, numerical, "num"),
|
|
314
|
+
(Stype.categorical, categorical, "cat"),
|
|
315
|
+
(Stype.datetime, datetime, "dt"),
|
|
316
|
+
(Stype.text, text, "text"),
|
|
317
|
+
(Stype.id, id, "id"),
|
|
318
|
+
):
|
|
319
|
+
if stype not in columns:
|
|
320
|
+
columns[stype] = tuple(
|
|
321
|
+
f"{name}_{i}" for i in range(block.size(-1))
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
if block.size(-1) != len(columns[stype]):
|
|
325
|
+
_columns = "column" if len(columns[stype]) == 1 else "columns"
|
|
326
|
+
raise ValueError(
|
|
327
|
+
f"Expected {str(stype)!r} block to hold "
|
|
328
|
+
f"{len(columns[stype])} {_columns} (got {block.size(-1)})"
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
column_names = list(chain.from_iterable(columns.values()))
|
|
332
|
+
column_to_loc: dict[str, tuple[Stype, int]] = {}
|
|
333
|
+
for stype_like, names in columns.items():
|
|
334
|
+
stype = Stype(stype_like)
|
|
335
|
+
for i, name in enumerate(names):
|
|
336
|
+
column_to_loc[name] = (stype, i)
|
|
337
|
+
if len(column_names) != len(column_to_loc):
|
|
338
|
+
raise ValueError("Expected column names to be unique")
|
|
339
|
+
|
|
340
|
+
out = Tensor._make_wrapper_subclass(
|
|
341
|
+
cls,
|
|
342
|
+
size=(*size, len(column_names)),
|
|
343
|
+
dtype=numerical.dtype,
|
|
344
|
+
device=numerical.device,
|
|
345
|
+
requires_grad=False,
|
|
346
|
+
)
|
|
347
|
+
|
|
348
|
+
out._numerical = numerical
|
|
349
|
+
out._categorical = categorical
|
|
350
|
+
out._datetime = datetime
|
|
351
|
+
out._text = text
|
|
352
|
+
out._id = id
|
|
353
|
+
out._columns = columns
|
|
354
|
+
out._column_to_loc = column_to_loc
|
|
355
|
+
|
|
356
|
+
return out
|
|
357
|
+
|
|
358
|
+
@classmethod
|
|
359
|
+
def from_arrow(
|
|
360
|
+
cls,
|
|
361
|
+
table: pa.Table,
|
|
362
|
+
stypes: Mapping[str, StypeLike],
|
|
363
|
+
*,
|
|
364
|
+
device: torch.device | str | None = None,
|
|
365
|
+
) -> Self:
|
|
366
|
+
r"""Create a tensor from a :class:`pyarrow.Table`.
|
|
367
|
+
|
|
368
|
+
.. testcode::
|
|
369
|
+
|
|
370
|
+
import pyarrow as pa
|
|
371
|
+
from sdm import TableTensor
|
|
372
|
+
|
|
373
|
+
arrow_table = pa.table({
|
|
374
|
+
"age": pa.array([25, 31, 42], type=pa.int64()),
|
|
375
|
+
"city": pa.array(["SF", "NYC", "SF"], type=pa.string()),
|
|
376
|
+
})
|
|
377
|
+
tensor = TableTensor.from_arrow(
|
|
378
|
+
table=arrow_table,
|
|
379
|
+
stypes={"age": "numerical", "city": "categorical"},
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
Args:
|
|
383
|
+
table: The table.
|
|
384
|
+
stypes: The semantic type for each column. Columns that are present
|
|
385
|
+
in ``table`` but not included in ``stypes`` will be ignored.
|
|
386
|
+
device: The device.
|
|
387
|
+
"""
|
|
388
|
+
columns: dict[Stype, list[str]] = defaultdict(list)
|
|
389
|
+
for column, stype in stypes.items():
|
|
390
|
+
columns[Stype(stype)].append(column)
|
|
391
|
+
|
|
392
|
+
blocks: dict[Stype, Tensor] = {}
|
|
393
|
+
for stype in columns:
|
|
394
|
+
tensors: list[Tensor] = []
|
|
395
|
+
for column in columns[stype]:
|
|
396
|
+
array = table.column(column)
|
|
397
|
+
if stype == Stype.numerical:
|
|
398
|
+
tensor = arrow_as_tensor(
|
|
399
|
+
array,
|
|
400
|
+
dtype=torch.get_default_dtype(),
|
|
401
|
+
).unsqueeze(-1)
|
|
402
|
+
elif stype == Stype.categorical:
|
|
403
|
+
tensor = CategoricalTensor.from_arrow(array)
|
|
404
|
+
elif stype == Stype.datetime:
|
|
405
|
+
array = array.cast(pa.timestamp("us"))
|
|
406
|
+
values = array.to_numpy(zero_copy_only=False)
|
|
407
|
+
values = values.astype("int64")
|
|
408
|
+
tensor = torch.from_numpy(values).unsqueeze(-1)
|
|
409
|
+
elif stype == Stype.text:
|
|
410
|
+
tensor = StringTensor.from_arrow(array).unsqueeze(-1)
|
|
411
|
+
elif stype == Stype.id:
|
|
412
|
+
tensor = ColumnarTensor.from_arrow(array)
|
|
413
|
+
else:
|
|
414
|
+
raise NotImplementedError
|
|
415
|
+
tensors.append(tensor)
|
|
416
|
+
|
|
417
|
+
blocks[stype] = torch.cat(tensors, dim=-1).to(device)
|
|
418
|
+
|
|
419
|
+
return cls(
|
|
420
|
+
columns=cast(Mapping[StypeLike, Sequence[str]], columns),
|
|
421
|
+
**blocks,
|
|
422
|
+
)
|
|
423
|
+
|
|
424
|
+
def to_arrow(self) -> pa.Table:
|
|
425
|
+
r"""Convert this tensor to a :class:`pyarrow.Table`."""
|
|
426
|
+
arrays: list[pa.Array] = []
|
|
427
|
+
columns: list[str] = []
|
|
428
|
+
for stype, tensor in self.items():
|
|
429
|
+
if tensor.size(-1) == 0:
|
|
430
|
+
continue
|
|
431
|
+
|
|
432
|
+
columns.extend(self._columns[stype])
|
|
433
|
+
|
|
434
|
+
if isinstance(tensor, CategoricalTensor | ColumnarTensor):
|
|
435
|
+
arrays.extend(tensor.to_arrow().itercolumns())
|
|
436
|
+
elif isinstance(tensor, StringTensor):
|
|
437
|
+
tensor = cast(
|
|
438
|
+
StringTensor,
|
|
439
|
+
tensor.movedim(-1, 0).contiguous().cpu(),
|
|
440
|
+
)
|
|
441
|
+
arrays.extend(cast(StringTensor, t).to_arrow() for t in tensor)
|
|
442
|
+
elif stype == Stype.datetime:
|
|
443
|
+
tensor = tensor.movedim(-1, 0).contiguous().cpu()
|
|
444
|
+
array = tensor.numpy().reshape(tensor.size(0), -1)
|
|
445
|
+
array = array.view("datetime64[us]")
|
|
446
|
+
arrays.extend(pa.array(a) for a in array)
|
|
447
|
+
else:
|
|
448
|
+
tensor = tensor.detach().movedim(-1, 0).contiguous().cpu()
|
|
449
|
+
arrays.extend(to_arrow(t) for t in tensor)
|
|
450
|
+
|
|
451
|
+
return pa.Table.from_arrays(arrays, names=columns)
|
|
452
|
+
|
|
453
|
+
@classmethod
|
|
454
|
+
def from_pandas(
|
|
455
|
+
cls,
|
|
456
|
+
df: pd.DataFrame,
|
|
457
|
+
stypes: Mapping[str, StypeLike],
|
|
458
|
+
*,
|
|
459
|
+
device: torch.device | str | None = None,
|
|
460
|
+
) -> Self:
|
|
461
|
+
r"""Create a tensor from a :class:`pandas.DataFrame`.
|
|
462
|
+
|
|
463
|
+
Args:
|
|
464
|
+
df: The dataframe.
|
|
465
|
+
stypes: The semantic type for each column. Columns that are present
|
|
466
|
+
in ``df`` but not included in ``stypes`` will be ignored.
|
|
467
|
+
device: The device.
|
|
468
|
+
"""
|
|
469
|
+
import pandas as pd
|
|
470
|
+
|
|
471
|
+
columns: dict[Stype, list[str]] = defaultdict(list)
|
|
472
|
+
for name, stype in stypes.items():
|
|
473
|
+
columns[Stype(stype)].append(name)
|
|
474
|
+
|
|
475
|
+
if columns and set(columns).issubset(
|
|
476
|
+
{Stype.numerical, Stype.categorical}
|
|
477
|
+
):
|
|
478
|
+
target_device = (
|
|
479
|
+
torch.get_default_device() if device is None else device
|
|
480
|
+
)
|
|
481
|
+
blocks: dict[Stype, Tensor] = {}
|
|
482
|
+
|
|
483
|
+
if numerical_columns := columns.get(Stype.numerical):
|
|
484
|
+
dtype = torch.get_default_dtype()
|
|
485
|
+
numpy_dtype = _TORCH_NUMPY_DTYPES.get(
|
|
486
|
+
dtype, np.dtype(np.float32)
|
|
487
|
+
)
|
|
488
|
+
numerical_df = (
|
|
489
|
+
df
|
|
490
|
+
if len(numerical_columns) == len(df.columns)
|
|
491
|
+
and numerical_columns == df.columns.tolist()
|
|
492
|
+
else df[numerical_columns]
|
|
493
|
+
)
|
|
494
|
+
if len(df) == 0:
|
|
495
|
+
if any(
|
|
496
|
+
pd.api.types.is_complex_dtype(dtype)
|
|
497
|
+
for dtype in numerical_df.dtypes
|
|
498
|
+
):
|
|
499
|
+
raise TypeError(
|
|
500
|
+
"Expected numerical columns to contain real values"
|
|
501
|
+
)
|
|
502
|
+
blocks[Stype.numerical] = torch.empty(
|
|
503
|
+
(0, len(numerical_columns)),
|
|
504
|
+
device=target_device,
|
|
505
|
+
dtype=dtype,
|
|
506
|
+
)
|
|
507
|
+
else:
|
|
508
|
+
try:
|
|
509
|
+
with warnings.catch_warnings():
|
|
510
|
+
warnings.simplefilter(
|
|
511
|
+
"error", np.exceptions.ComplexWarning
|
|
512
|
+
)
|
|
513
|
+
values = numerical_df.to_numpy(
|
|
514
|
+
dtype=numpy_dtype,
|
|
515
|
+
na_value=np.nan,
|
|
516
|
+
)
|
|
517
|
+
except np.exceptions.ComplexWarning as error:
|
|
518
|
+
raise TypeError(
|
|
519
|
+
"Expected numerical columns to contain real values"
|
|
520
|
+
) from error
|
|
521
|
+
values = values.copy(order="C")
|
|
522
|
+
blocks[Stype.numerical] = torch.as_tensor(
|
|
523
|
+
values,
|
|
524
|
+
device=target_device,
|
|
525
|
+
dtype=dtype,
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
if categorical_columns := columns.get(Stype.categorical):
|
|
529
|
+
categorical_series = [df[name] for name in categorical_columns]
|
|
530
|
+
all_categorical = all(
|
|
531
|
+
isinstance(ser.dtype, pd.CategoricalDtype)
|
|
532
|
+
for ser in categorical_series
|
|
533
|
+
)
|
|
534
|
+
use_int64_code = not all_categorical and any(
|
|
535
|
+
isinstance(ser.dtype, pd.ArrowDtype)
|
|
536
|
+
and pa.types.is_dictionary(ser.dtype.pyarrow_dtype)
|
|
537
|
+
and pa.types.is_int64(ser.dtype.pyarrow_dtype.index_type)
|
|
538
|
+
for ser in categorical_series
|
|
539
|
+
)
|
|
540
|
+
numpy_code_dtype = np.int64 if use_int64_code else np.int32
|
|
541
|
+
torch_code_dtype = (
|
|
542
|
+
torch.int64 if use_int64_code else torch.int32
|
|
543
|
+
)
|
|
544
|
+
if all_categorical:
|
|
545
|
+
code = np.stack(
|
|
546
|
+
[ser.array.codes for ser in categorical_series], axis=1
|
|
547
|
+
).astype(
|
|
548
|
+
numpy_code_dtype,
|
|
549
|
+
copy=False,
|
|
550
|
+
)
|
|
551
|
+
else:
|
|
552
|
+
code = np.empty(
|
|
553
|
+
(len(df), len(categorical_columns)),
|
|
554
|
+
dtype=numpy_code_dtype,
|
|
555
|
+
)
|
|
556
|
+
categories: list[Tensor] = []
|
|
557
|
+
for i, ser in enumerate(categorical_series):
|
|
558
|
+
if all_categorical:
|
|
559
|
+
categorical = ser.array
|
|
560
|
+
values = categorical.categories
|
|
561
|
+
elif isinstance(ser.dtype, pd.CategoricalDtype):
|
|
562
|
+
categorical = ser.array
|
|
563
|
+
code[:, i] = categorical.codes
|
|
564
|
+
values = categorical.categories
|
|
565
|
+
elif ser.dtype.kind == "f" or (
|
|
566
|
+
isinstance(ser.dtype, pd.ArrowDtype)
|
|
567
|
+
and pa.types.is_dictionary(ser.dtype.pyarrow_dtype)
|
|
568
|
+
):
|
|
569
|
+
categorical = CategoricalTensor.from_arrow(
|
|
570
|
+
pa.array(ser, from_pandas=True)
|
|
571
|
+
)
|
|
572
|
+
code[:, i] = categorical.code[:, 0].numpy()
|
|
573
|
+
categories.append(categorical.categories[0])
|
|
574
|
+
continue
|
|
575
|
+
else:
|
|
576
|
+
column_code, values = pd.factorize(
|
|
577
|
+
ser,
|
|
578
|
+
sort=False,
|
|
579
|
+
use_na_sentinel=True,
|
|
580
|
+
)
|
|
581
|
+
code[:, i] = column_code
|
|
582
|
+
|
|
583
|
+
category_dtype = values.dtype
|
|
584
|
+
if category_dtype.kind in "biuf":
|
|
585
|
+
category = torch.from_numpy(np.asarray(values).copy())
|
|
586
|
+
else:
|
|
587
|
+
array = pa.array(values, from_pandas=True)
|
|
588
|
+
is_string = pa.types.is_string(array.type)
|
|
589
|
+
is_large_string = pa.types.is_large_string(array.type)
|
|
590
|
+
if is_string or is_large_string:
|
|
591
|
+
category = StringTensor.from_arrow(
|
|
592
|
+
array,
|
|
593
|
+
device="cpu",
|
|
594
|
+
)
|
|
595
|
+
elif pa.types.is_null(array.type):
|
|
596
|
+
category = torch.empty(
|
|
597
|
+
0,
|
|
598
|
+
dtype=torch.int64,
|
|
599
|
+
device="cpu",
|
|
600
|
+
)
|
|
601
|
+
elif ser.dtype.kind == "O" and (
|
|
602
|
+
pa.types.is_boolean(array.type)
|
|
603
|
+
or pa.types.is_integer(array.type)
|
|
604
|
+
or pa.types.is_floating(array.type)
|
|
605
|
+
):
|
|
606
|
+
categorical = CategoricalTensor.from_arrow(
|
|
607
|
+
pa.array(ser, from_pandas=True)
|
|
608
|
+
)
|
|
609
|
+
code[:, i] = categorical.code[:, 0].numpy()
|
|
610
|
+
category = categorical.categories[0]
|
|
611
|
+
else:
|
|
612
|
+
category = arrow_as_tensor(array, device="cpu")
|
|
613
|
+
categories.append(category)
|
|
614
|
+
|
|
615
|
+
code_tensor = (
|
|
616
|
+
torch.from_numpy(code).to(target_device)
|
|
617
|
+
if len(df) > 0
|
|
618
|
+
else torch.empty(
|
|
619
|
+
(0, len(categorical_columns)),
|
|
620
|
+
dtype=torch_code_dtype,
|
|
621
|
+
device=target_device,
|
|
622
|
+
)
|
|
623
|
+
)
|
|
624
|
+
blocks[Stype.categorical] = CategoricalTensor(
|
|
625
|
+
code=code_tensor,
|
|
626
|
+
categories=_move_categories(categories, target_device),
|
|
627
|
+
)
|
|
628
|
+
|
|
629
|
+
return cls(
|
|
630
|
+
size=(len(df),),
|
|
631
|
+
columns=cast(Mapping[StypeLike, Sequence[str]], columns),
|
|
632
|
+
**blocks,
|
|
633
|
+
)
|
|
634
|
+
|
|
635
|
+
df = df[stypes.keys()]
|
|
636
|
+
# Resolve period ordinals before Arrow casts them as timestamps, which
|
|
637
|
+
# would interpret the ordinals as microseconds since the Unix epoch.
|
|
638
|
+
period_timestamps = {}
|
|
639
|
+
for name, stype in stypes.items():
|
|
640
|
+
if Stype(stype) != Stype.datetime:
|
|
641
|
+
continue
|
|
642
|
+
if not isinstance(df[name].dtype, pd.PeriodDtype):
|
|
643
|
+
continue
|
|
644
|
+
period_timestamps[name] = df[name].dt.to_timestamp(how="start")
|
|
645
|
+
if period_timestamps:
|
|
646
|
+
df = df.assign(**period_timestamps)
|
|
647
|
+
|
|
648
|
+
return cls.from_arrow(
|
|
649
|
+
table=pa.Table.from_pandas(df, preserve_index=False),
|
|
650
|
+
stypes=stypes,
|
|
651
|
+
device=device,
|
|
652
|
+
)
|
|
653
|
+
|
|
654
|
+
@classmethod
|
|
655
|
+
def from_columns(
|
|
656
|
+
cls,
|
|
657
|
+
data: Mapping[str, Sequence[Any]],
|
|
658
|
+
stypes: Mapping[str, StypeLike],
|
|
659
|
+
*,
|
|
660
|
+
device: torch.device | str | None = None,
|
|
661
|
+
) -> Self:
|
|
662
|
+
r"""Create a tensor from column data.
|
|
663
|
+
|
|
664
|
+
Args:
|
|
665
|
+
data: Column data keyed by column name.
|
|
666
|
+
stypes: The semantic type for each column. Columns that are present
|
|
667
|
+
in ``data`` but not included in ``stypes`` will be ignored.
|
|
668
|
+
device: The device.
|
|
669
|
+
"""
|
|
670
|
+
import pandas as pd
|
|
671
|
+
|
|
672
|
+
return cls.from_pandas(
|
|
673
|
+
df=pd.DataFrame(data),
|
|
674
|
+
stypes=stypes,
|
|
675
|
+
device=device,
|
|
676
|
+
)
|
|
677
|
+
|
|
678
|
+
def to_pandas(self) -> pd.DataFrame:
|
|
679
|
+
r"""Convert this tensor to a :class:`pandas.DataFrame`."""
|
|
680
|
+
return self.to_arrow().to_pandas()
|
|
681
|
+
|
|
682
|
+
@classmethod
|
|
683
|
+
def from_tensor(
|
|
684
|
+
cls,
|
|
685
|
+
tensor: Tensor,
|
|
686
|
+
columns: Sequence[str] | None = None,
|
|
687
|
+
) -> Self:
|
|
688
|
+
r"""Create a table from a :class:`torch.Tensor`.
|
|
689
|
+
|
|
690
|
+
Args:
|
|
691
|
+
tensor: The input tensor with shape ``[..., C]``, interpreted as:
|
|
692
|
+
|
|
693
|
+
* A floating-point :class:`torch.Tensor` becomes numerical
|
|
694
|
+
columns.
|
|
695
|
+
* An integer :class:`torch.Tensor` becomes categorical columns.
|
|
696
|
+
* A :class:`StringTensor` becomes text columns.
|
|
697
|
+
columns: The ``C`` column names.
|
|
698
|
+
"""
|
|
699
|
+
if columns is None:
|
|
700
|
+
columns = [str(i) for i in range(tensor.size(-1))]
|
|
701
|
+
|
|
702
|
+
if isinstance(tensor, StringTensor):
|
|
703
|
+
return cls(
|
|
704
|
+
columns={Stype.text: columns},
|
|
705
|
+
text=tensor,
|
|
706
|
+
)
|
|
707
|
+
|
|
708
|
+
if not tensor.is_floating_point():
|
|
709
|
+
return cls(
|
|
710
|
+
columns={Stype.categorical: columns},
|
|
711
|
+
categorical=CategoricalTensor.from_tensor(tensor),
|
|
712
|
+
)
|
|
713
|
+
|
|
714
|
+
return cls(
|
|
715
|
+
columns={Stype.numerical: columns},
|
|
716
|
+
numerical=tensor,
|
|
717
|
+
)
|
|
718
|
+
|
|
719
|
+
@classmethod
|
|
720
|
+
def from_cudf(
|
|
721
|
+
cls,
|
|
722
|
+
df: cudf.DataFrame,
|
|
723
|
+
stypes: Mapping[str, StypeLike],
|
|
724
|
+
*,
|
|
725
|
+
device: torch.device | str | None = None,
|
|
726
|
+
) -> Self:
|
|
727
|
+
r"""Create a tensor from a :class:`cudf.DataFrame`.
|
|
728
|
+
|
|
729
|
+
Args:
|
|
730
|
+
df: The dataframe.
|
|
731
|
+
stypes: The semantic type for each column. Columns that are present
|
|
732
|
+
in ``df`` but not included in ``stypes`` will be ignored.
|
|
733
|
+
device: The device.
|
|
734
|
+
"""
|
|
735
|
+
columns: dict[Stype, list[str]] = defaultdict(list)
|
|
736
|
+
for column, stype in stypes.items():
|
|
737
|
+
columns[Stype(stype)].append(column)
|
|
738
|
+
|
|
739
|
+
blocks: dict[Stype, Tensor] = {}
|
|
740
|
+
for stype in columns:
|
|
741
|
+
tensors: list[Tensor] = []
|
|
742
|
+
for column in columns[stype]:
|
|
743
|
+
ser = df[column]
|
|
744
|
+
if stype == Stype.numerical:
|
|
745
|
+
ser = ser.astype("float32", copy=False)
|
|
746
|
+
if ser.null_count > 0:
|
|
747
|
+
ser = ser.fillna(float("nan"))
|
|
748
|
+
tensor = torch.from_dlpack(ser.to_dlpack()).unsqueeze(-1)
|
|
749
|
+
tensor = tensor.to(device)
|
|
750
|
+
elif stype == Stype.categorical:
|
|
751
|
+
tensor = CategoricalTensor.from_cudf(ser, device=device)
|
|
752
|
+
elif stype == Stype.datetime:
|
|
753
|
+
ser = ser.astype("datetime64[us]", copy=False)
|
|
754
|
+
ser = ser.astype("int64", copy=False)
|
|
755
|
+
if ser.null_count > 0:
|
|
756
|
+
ser = ser.fillna(NaT)
|
|
757
|
+
tensor = torch.from_dlpack(ser.to_dlpack()).unsqueeze(-1)
|
|
758
|
+
tensor = tensor.to(device)
|
|
759
|
+
elif stype == Stype.text:
|
|
760
|
+
tensor = StringTensor.from_cudf(ser, device=device)
|
|
761
|
+
tensor = tensor.unsqueeze(-1)
|
|
762
|
+
elif stype == Stype.id:
|
|
763
|
+
tensor = ColumnarTensor.from_cudf(ser, device=device)
|
|
764
|
+
else:
|
|
765
|
+
raise NotImplementedError
|
|
766
|
+
tensors.append(tensor)
|
|
767
|
+
|
|
768
|
+
blocks[stype] = torch.cat(tensors, dim=-1)
|
|
769
|
+
|
|
770
|
+
return cls(
|
|
771
|
+
columns=cast(Mapping[StypeLike, Sequence[str]], columns),
|
|
772
|
+
**blocks,
|
|
773
|
+
)
|
|
774
|
+
|
|
775
|
+
def to_cudf(self) -> cudf.DataFrame:
|
|
776
|
+
r"""Convert this tensor to a :class:`cudf.DataFrame`."""
|
|
777
|
+
import cudf
|
|
778
|
+
|
|
779
|
+
dfs: list[cudf.DataFrame] = []
|
|
780
|
+
for stype, tensor in self.items():
|
|
781
|
+
if tensor.size(-1) == 0:
|
|
782
|
+
continue
|
|
783
|
+
|
|
784
|
+
if isinstance(tensor, CategoricalTensor | ColumnarTensor):
|
|
785
|
+
dfs.append(tensor.to_cudf(self._columns[stype]))
|
|
786
|
+
elif isinstance(tensor, StringTensor):
|
|
787
|
+
tensor = cast(StringTensor, tensor.movedim(-1, 0).contiguous())
|
|
788
|
+
df = cudf.DataFrame(
|
|
789
|
+
{
|
|
790
|
+
name: cast(StringTensor, column).to_cudf()
|
|
791
|
+
for name, column in zip(self._columns[stype], tensor)
|
|
792
|
+
}
|
|
793
|
+
)
|
|
794
|
+
dfs.append(df)
|
|
795
|
+
elif stype == Stype.datetime:
|
|
796
|
+
tensor = tensor.movedim(-1, 0).contiguous()
|
|
797
|
+
df = cudf.DataFrame(
|
|
798
|
+
{
|
|
799
|
+
name: to_cudf(column, mask).astype(
|
|
800
|
+
"datetime64[us]", copy=False
|
|
801
|
+
)
|
|
802
|
+
for name, column, mask in zip(
|
|
803
|
+
self._columns[stype],
|
|
804
|
+
tensor,
|
|
805
|
+
tensor != NaT,
|
|
806
|
+
)
|
|
807
|
+
}
|
|
808
|
+
)
|
|
809
|
+
dfs.append(df)
|
|
810
|
+
else:
|
|
811
|
+
tensor = tensor.detach().movedim(-1, 0).contiguous()
|
|
812
|
+
valid = (
|
|
813
|
+
tensor.isnan().logical_not_()
|
|
814
|
+
if tensor.is_floating_point()
|
|
815
|
+
else None
|
|
816
|
+
)
|
|
817
|
+
col_dict: dict[str, cudf.Series] = {}
|
|
818
|
+
for idx, (name, column) in enumerate(
|
|
819
|
+
zip(self._columns[stype], tensor)
|
|
820
|
+
):
|
|
821
|
+
col_dict[name] = to_cudf(
|
|
822
|
+
tensor=column,
|
|
823
|
+
valid_mask=None if valid is None else valid[idx],
|
|
824
|
+
)
|
|
825
|
+
|
|
826
|
+
df = cudf.DataFrame(col_dict)
|
|
827
|
+
dfs.append(df)
|
|
828
|
+
|
|
829
|
+
if len(dfs) == 1:
|
|
830
|
+
return dfs[0]
|
|
831
|
+
|
|
832
|
+
return cudf.concat(dfs, axis=1)
|
|
833
|
+
|
|
834
|
+
# Properties ##############################################################
|
|
835
|
+
|
|
836
|
+
@property
|
|
837
|
+
def columns(self) -> Mapping[Stype, tuple[str, ...]]:
|
|
838
|
+
r"""Return column names grouped by semantic type."""
|
|
839
|
+
return self._columns.copy()
|
|
840
|
+
|
|
841
|
+
@property
|
|
842
|
+
def column_names(self) -> frozenset[str]:
|
|
843
|
+
r"""The column names of this tensor."""
|
|
844
|
+
return frozenset(self._column_to_loc)
|
|
845
|
+
|
|
846
|
+
@property
|
|
847
|
+
def stypes(self) -> Mapping[str, Stype]:
|
|
848
|
+
r"""Return the semantic type for each column."""
|
|
849
|
+
return {key: stype for key, (stype, _) in self._column_to_loc.items()}
|
|
850
|
+
|
|
851
|
+
def stype(self, column: str) -> Stype:
|
|
852
|
+
r"""Return the semantic type for a column.
|
|
853
|
+
|
|
854
|
+
Args:
|
|
855
|
+
column: The column name.
|
|
856
|
+
"""
|
|
857
|
+
return self._column_to_loc[column][0]
|
|
858
|
+
|
|
859
|
+
@property
|
|
860
|
+
def active_stypes(self) -> frozenset[Stype]:
|
|
861
|
+
r"""Semantic types with at least one column."""
|
|
862
|
+
return frozenset(
|
|
863
|
+
stype for stype, tensor in self.items() if tensor.size(-1) > 0
|
|
864
|
+
)
|
|
865
|
+
|
|
866
|
+
@property
|
|
867
|
+
def numerical(self) -> Tensor:
|
|
868
|
+
r"""Return the numerical column block."""
|
|
869
|
+
return self._numerical
|
|
870
|
+
|
|
871
|
+
@property
|
|
872
|
+
def categorical(self) -> CategoricalTensor:
|
|
873
|
+
r"""Return the categorical column block."""
|
|
874
|
+
return self._categorical
|
|
875
|
+
|
|
876
|
+
@property
|
|
877
|
+
def datetime(self) -> Tensor:
|
|
878
|
+
r"""Return the datetime column block."""
|
|
879
|
+
return self._datetime
|
|
880
|
+
|
|
881
|
+
@property
|
|
882
|
+
def text(self) -> StringTensor:
|
|
883
|
+
r"""Return the text column block."""
|
|
884
|
+
return self._text
|
|
885
|
+
|
|
886
|
+
@property
|
|
887
|
+
def id(self) -> ColumnarTensor:
|
|
888
|
+
r"""Return the identifier column block."""
|
|
889
|
+
return self._id
|
|
890
|
+
|
|
891
|
+
def items(self) -> Iterator[tuple[Stype, Tensor]]:
|
|
892
|
+
r"""Yield ``(stype, block)`` pairs for typed column blocks."""
|
|
893
|
+
yield Stype.numerical, self._numerical
|
|
894
|
+
yield Stype.categorical, self._categorical
|
|
895
|
+
yield Stype.datetime, self._datetime
|
|
896
|
+
yield Stype.text, self._text
|
|
897
|
+
yield Stype.id, self._id
|
|
898
|
+
|
|
899
|
+
@property
|
|
900
|
+
def blocks(self) -> Mapping[Stype, Tensor]:
|
|
901
|
+
r"""Return typed column blocks per semantic type."""
|
|
902
|
+
return dict(self.items())
|
|
903
|
+
|
|
904
|
+
@property
|
|
905
|
+
def schema(self) -> TableSchema:
|
|
906
|
+
r"""The schema of this table."""
|
|
907
|
+
return TableSchema(columns=self._columns)
|
|
908
|
+
|
|
909
|
+
def is_same_schema(self, other: TableTensor) -> bool:
|
|
910
|
+
r"""Whether ``other`` has the same schema layout.
|
|
911
|
+
|
|
912
|
+
Args:
|
|
913
|
+
other: The object to compare against.
|
|
914
|
+
"""
|
|
915
|
+
return self.schema == other.schema
|
|
916
|
+
|
|
917
|
+
def replace_blocks(
|
|
918
|
+
self,
|
|
919
|
+
*,
|
|
920
|
+
numerical: Tensor | None = None,
|
|
921
|
+
categorical: CategoricalTensor | None = None,
|
|
922
|
+
datetime: Tensor | None = None,
|
|
923
|
+
text: StringTensor | None = None,
|
|
924
|
+
id: ColumnarTensor | None = None,
|
|
925
|
+
) -> Self:
|
|
926
|
+
r"""Return a table with one or more semantic blocks replaced.
|
|
927
|
+
|
|
928
|
+
Provided blocks replace the corresponding semantic type while omitted
|
|
929
|
+
blocks are reused from this table. The returned table preserves the
|
|
930
|
+
current column schema and is validated by the ``TableTensor``
|
|
931
|
+
constructor.
|
|
932
|
+
|
|
933
|
+
Args:
|
|
934
|
+
numerical: Replacement numerical block with shape
|
|
935
|
+
``[..., C_num]``.
|
|
936
|
+
categorical: Replacement categorical block with shape
|
|
937
|
+
``[..., C_cat]``.
|
|
938
|
+
datetime: Replacement datetime block with shape ``[..., C_dt]``.
|
|
939
|
+
text: Replacement text block with shape ``[..., C_text]``.
|
|
940
|
+
id: Replacement identifier block with shape ``[..., C_id]``.
|
|
941
|
+
"""
|
|
942
|
+
return self.__class__(
|
|
943
|
+
columns=cast(Mapping[StypeLike, Sequence[str]], self.columns),
|
|
944
|
+
numerical=self.numerical if numerical is None else numerical,
|
|
945
|
+
categorical=(
|
|
946
|
+
self.categorical if categorical is None else categorical
|
|
947
|
+
),
|
|
948
|
+
datetime=self.datetime if datetime is None else datetime,
|
|
949
|
+
text=self.text if text is None else text,
|
|
950
|
+
id=self.id if id is None else id,
|
|
951
|
+
)
|
|
952
|
+
|
|
953
|
+
def select_stypes(
|
|
954
|
+
self,
|
|
955
|
+
stypes: StypeLike | Iterable[StypeLike],
|
|
956
|
+
) -> Self:
|
|
957
|
+
r"""Return a table containing only ``stypes`` columns.
|
|
958
|
+
|
|
959
|
+
Args:
|
|
960
|
+
stypes: The semantic type or semantic types to select.
|
|
961
|
+
"""
|
|
962
|
+
if isinstance(stypes, (str, Stype)):
|
|
963
|
+
stypes = (stypes,)
|
|
964
|
+
stypes = tuple(Stype(stype) for stype in stypes)
|
|
965
|
+
|
|
966
|
+
return self.__class__(
|
|
967
|
+
columns={stype: self._columns[stype] for stype in stypes},
|
|
968
|
+
**{stype: getattr(self, stype) for stype in stypes},
|
|
969
|
+
)
|
|
970
|
+
|
|
971
|
+
def drop_stypes(
|
|
972
|
+
self,
|
|
973
|
+
stypes: StypeLike | Iterable[StypeLike],
|
|
974
|
+
) -> Self:
|
|
975
|
+
r"""Return a table with ``stypes`` columns removed.
|
|
976
|
+
|
|
977
|
+
.. testcode:: drop_stypes
|
|
978
|
+
|
|
979
|
+
assert table.columns[Stype.categorical] == ("country", "segment")
|
|
980
|
+
table = table.drop_stypes("categorical")
|
|
981
|
+
assert table.columns[Stype.categorical] == ()
|
|
982
|
+
assert table.columns[Stype.numerical] == ("age", "income")
|
|
983
|
+
|
|
984
|
+
Args:
|
|
985
|
+
stypes: The semantic type or semantic types to drop.
|
|
986
|
+
"""
|
|
987
|
+
if isinstance(stypes, (str, Stype)):
|
|
988
|
+
stypes = (stypes,)
|
|
989
|
+
stypes = {Stype(stype) for stype in stypes}
|
|
990
|
+
|
|
991
|
+
keep = tuple(stype for stype in self._columns if stype not in stypes)
|
|
992
|
+
return self.__class__(
|
|
993
|
+
size=self.size()[:-1],
|
|
994
|
+
columns={stype: self._columns[stype] for stype in keep},
|
|
995
|
+
device=self.device,
|
|
996
|
+
**{stype: getattr(self, stype) for stype in keep},
|
|
997
|
+
)
|
|
998
|
+
|
|
999
|
+
def select_columns(self, columns: str | Iterable[str]) -> Self:
|
|
1000
|
+
r"""Return a table containing only ``columns``.
|
|
1001
|
+
|
|
1002
|
+
.. testcode:: select_columns
|
|
1003
|
+
|
|
1004
|
+
assert table.size() == (10, 4)
|
|
1005
|
+
table = table.select_columns(["age", "country"])
|
|
1006
|
+
assert table.size() == (10, 2)
|
|
1007
|
+
|
|
1008
|
+
Args:
|
|
1009
|
+
columns: The columns to select.
|
|
1010
|
+
"""
|
|
1011
|
+
columns = {columns} if isinstance(columns, str) else set(columns)
|
|
1012
|
+
|
|
1013
|
+
for column in columns:
|
|
1014
|
+
if column not in self._column_to_loc:
|
|
1015
|
+
raise KeyError(column)
|
|
1016
|
+
|
|
1017
|
+
index_dict: dict[Stype, list[int]] = defaultdict(list)
|
|
1018
|
+
columns_dict: dict[StypeLike, list[str]] = defaultdict(list)
|
|
1019
|
+
for stype, stype_columns in self._columns.items():
|
|
1020
|
+
for i, column in enumerate(stype_columns):
|
|
1021
|
+
if column in columns:
|
|
1022
|
+
index_dict[stype].append(i)
|
|
1023
|
+
columns_dict[stype].append(column)
|
|
1024
|
+
|
|
1025
|
+
blocks: dict[Stype, Tensor] = {}
|
|
1026
|
+
for stype, tensor in self.items():
|
|
1027
|
+
indices = index_dict[stype]
|
|
1028
|
+
if len(indices) == 0:
|
|
1029
|
+
blocks[stype] = tensor.narrow(-1, 0, 0)
|
|
1030
|
+
elif len(indices) == len(self._columns[stype]):
|
|
1031
|
+
blocks[stype] = tensor
|
|
1032
|
+
elif len(indices) == 1:
|
|
1033
|
+
blocks[stype] = tensor.narrow(-1, indices[0], 1)
|
|
1034
|
+
else:
|
|
1035
|
+
blocks[stype] = torch.cat(
|
|
1036
|
+
[tensor.narrow(-1, index, 1) for index in indices],
|
|
1037
|
+
dim=-1,
|
|
1038
|
+
)
|
|
1039
|
+
|
|
1040
|
+
return self.__class__(columns=columns_dict, **blocks)
|
|
1041
|
+
|
|
1042
|
+
def drop_columns(self, columns: str | Iterable[str]) -> Self:
|
|
1043
|
+
r"""Return a table with ``columns`` removed.
|
|
1044
|
+
|
|
1045
|
+
.. testcode:: drop_columns
|
|
1046
|
+
|
|
1047
|
+
assert table.size() == (10, 4)
|
|
1048
|
+
table = table.drop_columns(["age", "country"])
|
|
1049
|
+
assert table.size() == (10, 2)
|
|
1050
|
+
|
|
1051
|
+
Args:
|
|
1052
|
+
columns: The columns to drop.
|
|
1053
|
+
"""
|
|
1054
|
+
columns = {columns} if isinstance(columns, str) else set(columns)
|
|
1055
|
+
|
|
1056
|
+
for column in columns:
|
|
1057
|
+
if column not in self._column_to_loc:
|
|
1058
|
+
raise KeyError(column)
|
|
1059
|
+
|
|
1060
|
+
columns = set(chain.from_iterable(self._columns.values())) - columns
|
|
1061
|
+
return self.select_columns(columns)
|
|
1062
|
+
|
|
1063
|
+
# Decorators ##############################################################
|
|
1064
|
+
|
|
1065
|
+
@classmethod
|
|
1066
|
+
def implements(
|
|
1067
|
+
cls,
|
|
1068
|
+
torch_function: Callable[..., Any],
|
|
1069
|
+
) -> Callable[..., Any]:
|
|
1070
|
+
r"""Register a ``__torch_dispatch__`` implementation.
|
|
1071
|
+
|
|
1072
|
+
See PyTorch's
|
|
1073
|
+
:ref:`calling convention <torch-dispatch-calling-convention>`.
|
|
1074
|
+
"""
|
|
1075
|
+
if "HANDLED_FUNCTIONS" not in cls.__dict__:
|
|
1076
|
+
cls.HANDLED_FUNCTIONS = cls.HANDLED_FUNCTIONS.copy()
|
|
1077
|
+
|
|
1078
|
+
def decorator(my_function: Callable[..., Any]) -> Callable[..., Any]:
|
|
1079
|
+
cls.HANDLED_FUNCTIONS[torch_function] = my_function
|
|
1080
|
+
return my_function
|
|
1081
|
+
|
|
1082
|
+
return decorator
|
|
1083
|
+
|
|
1084
|
+
# PyTorch/Python builtins #################################################
|
|
1085
|
+
|
|
1086
|
+
def __reduce_ex__(self, proto: SupportsIndex) -> Any:
|
|
1087
|
+
args = (
|
|
1088
|
+
tuple(self.size()[:-1]),
|
|
1089
|
+
self._columns,
|
|
1090
|
+
self._numerical,
|
|
1091
|
+
self._categorical,
|
|
1092
|
+
self._datetime,
|
|
1093
|
+
self._text,
|
|
1094
|
+
self._id,
|
|
1095
|
+
)
|
|
1096
|
+
return (self.__class__, args)
|
|
1097
|
+
|
|
1098
|
+
@classmethod
|
|
1099
|
+
def __torch_dispatch__( # type: ignore
|
|
1100
|
+
cls,
|
|
1101
|
+
func: Callable[..., Any],
|
|
1102
|
+
types: tuple[type[Any], ...],
|
|
1103
|
+
args: tuple[Any, ...] = (),
|
|
1104
|
+
kwargs: dict[str, Any] | None = None,
|
|
1105
|
+
) -> Any:
|
|
1106
|
+
if (handler := cls.HANDLED_FUNCTIONS.get(func)) is not None:
|
|
1107
|
+
return handler(*args, **(kwargs or {}))
|
|
1108
|
+
|
|
1109
|
+
raise NotImplementedError(
|
|
1110
|
+
f"'{func}' is not supported for {cls.__name__!r}"
|
|
1111
|
+
)
|
|
1112
|
+
|
|
1113
|
+
def __getitem__(self, indices: Any) -> TableTensor:
|
|
1114
|
+
def is_column_index(index: Any) -> bool:
|
|
1115
|
+
return isinstance(index, str) or (
|
|
1116
|
+
isinstance(index, list)
|
|
1117
|
+
and all(isinstance(value, str) for value in index)
|
|
1118
|
+
)
|
|
1119
|
+
|
|
1120
|
+
if is_column_index(indices):
|
|
1121
|
+
return self.select_columns(indices)
|
|
1122
|
+
if not isinstance(indices, tuple):
|
|
1123
|
+
return cast(TableTensor, Tensor.__getitem__(self, indices))
|
|
1124
|
+
if not any(is_column_index(index) for index in indices):
|
|
1125
|
+
return cast(TableTensor, Tensor.__getitem__(self, indices))
|
|
1126
|
+
if any(is_column_index(index) for index in indices[:-1]):
|
|
1127
|
+
raise IndexError(
|
|
1128
|
+
"Column names can only index the column dimension"
|
|
1129
|
+
)
|
|
1130
|
+
|
|
1131
|
+
out = Tensor.__getitem__(self, (*indices[:-1], slice(None)))
|
|
1132
|
+
return cast(TableTensor, out).select_columns(indices[-1])
|
|
1133
|
+
|
|
1134
|
+
@override
|
|
1135
|
+
def is_shared(self) -> bool:
|
|
1136
|
+
return all(tensor.is_shared() for _, tensor in self.items())
|
|
1137
|
+
|
|
1138
|
+
@override
|
|
1139
|
+
def share_memory_(self) -> Self:
|
|
1140
|
+
for _, tensor in self.items():
|
|
1141
|
+
tensor.share_memory_()
|
|
1142
|
+
return self
|
|
1143
|
+
|
|
1144
|
+
@override
|
|
1145
|
+
def is_contiguous(
|
|
1146
|
+
self,
|
|
1147
|
+
memory_format: torch.memory_format = torch.contiguous_format,
|
|
1148
|
+
) -> bool:
|
|
1149
|
+
return all(
|
|
1150
|
+
tensor.is_contiguous(memory_format=memory_format)
|
|
1151
|
+
for _, tensor in self.items()
|
|
1152
|
+
)
|
|
1153
|
+
|
|
1154
|
+
@override
|
|
1155
|
+
def contiguous(
|
|
1156
|
+
self,
|
|
1157
|
+
memory_format: torch.memory_format = torch.contiguous_format,
|
|
1158
|
+
) -> Self:
|
|
1159
|
+
if self.is_contiguous(memory_format=memory_format):
|
|
1160
|
+
return self
|
|
1161
|
+
return _contiguous(self, memory_format=memory_format)
|
|
1162
|
+
|
|
1163
|
+
@override
|
|
1164
|
+
def tolist() -> Any:
|
|
1165
|
+
raise NotImplementedError("'tolist() is not yet implemented") # TODO
|
|
1166
|
+
|
|
1167
|
+
def __repr__(self, *, indent: int = 0) -> str: # type: ignore
|
|
1168
|
+
def _columns_repr(
|
|
1169
|
+
columns: Sequence[str],
|
|
1170
|
+
max_cols: int = 3,
|
|
1171
|
+
max_item_len: int = 24,
|
|
1172
|
+
) -> str:
|
|
1173
|
+
columns = [
|
|
1174
|
+
column
|
|
1175
|
+
if len(column) <= max_item_len
|
|
1176
|
+
else column[: max_item_len - 1] + "…"
|
|
1177
|
+
for column in columns
|
|
1178
|
+
]
|
|
1179
|
+
if len(columns) > max_cols:
|
|
1180
|
+
columns = [*columns[: max_cols - 1], "...", columns[-1]]
|
|
1181
|
+
return "[" + ", ".join(column for column in columns) + "]"
|
|
1182
|
+
|
|
1183
|
+
stype_repr = [
|
|
1184
|
+
(
|
|
1185
|
+
f"{' ' * (indent + 4)}{stype} ({tensor.size(-1):,}): "
|
|
1186
|
+
f"{_columns_repr(self._columns[stype])},"
|
|
1187
|
+
)
|
|
1188
|
+
for stype, tensor in self.items()
|
|
1189
|
+
if tensor.size(-1) > 0
|
|
1190
|
+
]
|
|
1191
|
+
|
|
1192
|
+
out = f"{' ' * indent}{self.__class__.__name__}(\n"
|
|
1193
|
+
out += f"{' ' * (indent + 2)}size={tuple(self.size())},\n"
|
|
1194
|
+
if len(stype_repr) > 0:
|
|
1195
|
+
out += f"{' ' * (indent + 2)}blocks={{\n"
|
|
1196
|
+
out += "\n".join(stype_repr) + "\n"
|
|
1197
|
+
out += f"{' ' * (indent + 2)}}},\n"
|
|
1198
|
+
if not self.is_cpu:
|
|
1199
|
+
out += f"{' ' * (indent + 2)}device={self.device},\n"
|
|
1200
|
+
out += f"{' ' * indent})"
|
|
1201
|
+
return out
|
|
1202
|
+
|
|
1203
|
+
def _repr_html_(self) -> str:
|
|
1204
|
+
import pandas as pd
|
|
1205
|
+
|
|
1206
|
+
max_columns = 10
|
|
1207
|
+
rows = [
|
|
1208
|
+
[column, str(stype)]
|
|
1209
|
+
for stype, columns in self._columns.items()
|
|
1210
|
+
for column in columns
|
|
1211
|
+
]
|
|
1212
|
+
if len(rows) > max_columns + 1:
|
|
1213
|
+
rows = [
|
|
1214
|
+
*rows[: max_columns // 2],
|
|
1215
|
+
["...", "..."],
|
|
1216
|
+
*rows[-max_columns // 2 :],
|
|
1217
|
+
]
|
|
1218
|
+
df = pd.DataFrame(
|
|
1219
|
+
data=rows,
|
|
1220
|
+
columns=pd.Index(["Column", "Stype"]),
|
|
1221
|
+
)
|
|
1222
|
+
|
|
1223
|
+
size = f"{self.size(-2)} rows x {self.size(-1)} columns"
|
|
1224
|
+
if self.dim() > 2:
|
|
1225
|
+
examples = " x ".join(str(dim) for dim in self.size()[:-2])
|
|
1226
|
+
size = f"{examples} examples x {size}"
|
|
1227
|
+
if not self.is_cpu:
|
|
1228
|
+
size += f", device={self.device}"
|
|
1229
|
+
|
|
1230
|
+
return df.to_html(index=False, escape=True) + f"<p>{size}</p>"
|
|
1231
|
+
|
|
1232
|
+
|
|
1233
|
+
@TableTensor.implements(aten.alias.default)
|
|
1234
|
+
@preserve_view_inference_mode
|
|
1235
|
+
def _alias(inp: TableTensor) -> TableTensor:
|
|
1236
|
+
blocks = {
|
|
1237
|
+
stype: aten.alias.default(tensor) for stype, tensor in inp.items()
|
|
1238
|
+
}
|
|
1239
|
+
return inp.__class__(
|
|
1240
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1241
|
+
**blocks,
|
|
1242
|
+
)
|
|
1243
|
+
|
|
1244
|
+
|
|
1245
|
+
@TableTensor.implements(aten.record_stream.default)
|
|
1246
|
+
def _record_stream(inp: TableTensor, stream: torch.Stream) -> None:
|
|
1247
|
+
for _, tensor in inp.items():
|
|
1248
|
+
tensor.record_stream(stream)
|
|
1249
|
+
|
|
1250
|
+
|
|
1251
|
+
@TableTensor.implements(aten.to.dtype_layout)
|
|
1252
|
+
@preserve_autograd_state
|
|
1253
|
+
def _to_dtype_layout(
|
|
1254
|
+
inp: TableTensor,
|
|
1255
|
+
*,
|
|
1256
|
+
dtype: torch.dtype | None = None,
|
|
1257
|
+
layout: torch.layout | None = None,
|
|
1258
|
+
device: torch.device | str | None = None,
|
|
1259
|
+
pin_memory: bool | None = None, # Ignored by PyTorch.
|
|
1260
|
+
non_blocking: bool = False,
|
|
1261
|
+
copy: bool = False,
|
|
1262
|
+
memory_format: torch.memory_format | None = None,
|
|
1263
|
+
) -> TableTensor:
|
|
1264
|
+
|
|
1265
|
+
if (
|
|
1266
|
+
not copy
|
|
1267
|
+
and (dtype is None or dtype == inp.dtype)
|
|
1268
|
+
and (device is None or torch.device(device) == inp.device)
|
|
1269
|
+
and (layout is None or layout == inp.layout)
|
|
1270
|
+
and (
|
|
1271
|
+
memory_format is None
|
|
1272
|
+
or memory_format == torch.preserve_format
|
|
1273
|
+
or (
|
|
1274
|
+
memory_format == torch.contiguous_format
|
|
1275
|
+
and inp.is_contiguous()
|
|
1276
|
+
)
|
|
1277
|
+
)
|
|
1278
|
+
):
|
|
1279
|
+
return inp
|
|
1280
|
+
|
|
1281
|
+
blocks = {
|
|
1282
|
+
stype: aten.to.dtype_layout(
|
|
1283
|
+
tensor,
|
|
1284
|
+
dtype=dtype
|
|
1285
|
+
if (
|
|
1286
|
+
stype not in (Stype.categorical,)
|
|
1287
|
+
or dtype in (torch.int32, torch.int64)
|
|
1288
|
+
)
|
|
1289
|
+
and stype not in (Stype.datetime, Stype.text, Stype.id)
|
|
1290
|
+
else None,
|
|
1291
|
+
layout=layout,
|
|
1292
|
+
device=device,
|
|
1293
|
+
pin_memory=pin_memory,
|
|
1294
|
+
non_blocking=non_blocking,
|
|
1295
|
+
copy=copy,
|
|
1296
|
+
memory_format=memory_format,
|
|
1297
|
+
)
|
|
1298
|
+
for stype, tensor in inp.items()
|
|
1299
|
+
}
|
|
1300
|
+
return inp.__class__(
|
|
1301
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1302
|
+
**blocks,
|
|
1303
|
+
)
|
|
1304
|
+
|
|
1305
|
+
|
|
1306
|
+
@TableTensor.implements(aten.to.dtype)
|
|
1307
|
+
def _to_dtype(
|
|
1308
|
+
inp: TableTensor,
|
|
1309
|
+
dtype: torch.dtype,
|
|
1310
|
+
non_blocking: bool = False,
|
|
1311
|
+
copy: bool = False,
|
|
1312
|
+
memory_format: torch.memory_format | None = None,
|
|
1313
|
+
) -> TableTensor:
|
|
1314
|
+
return _to_dtype_layout(
|
|
1315
|
+
inp,
|
|
1316
|
+
dtype=dtype,
|
|
1317
|
+
non_blocking=non_blocking,
|
|
1318
|
+
copy=copy,
|
|
1319
|
+
memory_format=memory_format,
|
|
1320
|
+
)
|
|
1321
|
+
|
|
1322
|
+
|
|
1323
|
+
@TableTensor.implements(aten.to.device)
|
|
1324
|
+
def _to_device(
|
|
1325
|
+
inp: TableTensor,
|
|
1326
|
+
device: torch.device,
|
|
1327
|
+
dtype: torch.dtype,
|
|
1328
|
+
non_blocking: bool = False,
|
|
1329
|
+
copy: bool = False,
|
|
1330
|
+
memory_format: torch.memory_format | None = None,
|
|
1331
|
+
) -> TableTensor:
|
|
1332
|
+
return _to_dtype_layout(
|
|
1333
|
+
inp,
|
|
1334
|
+
dtype=dtype,
|
|
1335
|
+
device=device,
|
|
1336
|
+
non_blocking=non_blocking,
|
|
1337
|
+
copy=copy,
|
|
1338
|
+
memory_format=memory_format,
|
|
1339
|
+
)
|
|
1340
|
+
|
|
1341
|
+
|
|
1342
|
+
@TableTensor.implements(aten.to.other)
|
|
1343
|
+
def _to_other(
|
|
1344
|
+
inp: TableTensor,
|
|
1345
|
+
other: Tensor,
|
|
1346
|
+
non_blocking: bool = False,
|
|
1347
|
+
copy: bool = False,
|
|
1348
|
+
memory_format: torch.memory_format | None = None,
|
|
1349
|
+
) -> TableTensor:
|
|
1350
|
+
return _to_dtype_layout(
|
|
1351
|
+
inp,
|
|
1352
|
+
dtype=other.dtype,
|
|
1353
|
+
layout=other.layout,
|
|
1354
|
+
device=other.device,
|
|
1355
|
+
non_blocking=non_blocking,
|
|
1356
|
+
copy=copy,
|
|
1357
|
+
memory_format=memory_format,
|
|
1358
|
+
)
|
|
1359
|
+
|
|
1360
|
+
|
|
1361
|
+
@TableTensor.implements(aten._to_copy.default)
|
|
1362
|
+
def _to_copy(
|
|
1363
|
+
inp: TableTensor,
|
|
1364
|
+
*,
|
|
1365
|
+
dtype: torch.dtype | None = None,
|
|
1366
|
+
layout: torch.layout | None = None,
|
|
1367
|
+
device: torch.device | str | None = None,
|
|
1368
|
+
pin_memory: bool = False, # Ignored by PyTorch.
|
|
1369
|
+
non_blocking: bool = False,
|
|
1370
|
+
memory_format: torch.memory_format | None = None,
|
|
1371
|
+
) -> TableTensor:
|
|
1372
|
+
return _to_dtype_layout(
|
|
1373
|
+
inp,
|
|
1374
|
+
dtype=dtype,
|
|
1375
|
+
layout=layout,
|
|
1376
|
+
device=device,
|
|
1377
|
+
pin_memory=pin_memory,
|
|
1378
|
+
non_blocking=non_blocking,
|
|
1379
|
+
copy=True,
|
|
1380
|
+
memory_format=memory_format,
|
|
1381
|
+
)
|
|
1382
|
+
|
|
1383
|
+
|
|
1384
|
+
@TableTensor.implements(aten.clone.default)
|
|
1385
|
+
def _clone(
|
|
1386
|
+
inp: TableTensor,
|
|
1387
|
+
*,
|
|
1388
|
+
memory_format: torch.memory_format | None = None,
|
|
1389
|
+
) -> TableTensor:
|
|
1390
|
+
return _to_dtype_layout(inp, copy=True, memory_format=memory_format)
|
|
1391
|
+
|
|
1392
|
+
|
|
1393
|
+
@TableTensor.implements(aten.contiguous.default)
|
|
1394
|
+
def _contiguous(
|
|
1395
|
+
inp: TableTensor,
|
|
1396
|
+
*,
|
|
1397
|
+
memory_format: torch.memory_format = torch.contiguous_format,
|
|
1398
|
+
) -> TableTensor:
|
|
1399
|
+
blocks = {
|
|
1400
|
+
stype: tensor.contiguous(memory_format=memory_format)
|
|
1401
|
+
for stype, tensor in inp.items()
|
|
1402
|
+
}
|
|
1403
|
+
return inp.__class__(
|
|
1404
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1405
|
+
**blocks,
|
|
1406
|
+
)
|
|
1407
|
+
|
|
1408
|
+
|
|
1409
|
+
@TableTensor.implements(aten.is_pinned.default)
|
|
1410
|
+
def _is_pinned(inp: TableTensor) -> bool:
|
|
1411
|
+
return all(
|
|
1412
|
+
tensor.is_pinned() for _, tensor in inp.items() if tensor.numel() > 0
|
|
1413
|
+
)
|
|
1414
|
+
|
|
1415
|
+
|
|
1416
|
+
@TableTensor.implements(aten._pin_memory.default)
|
|
1417
|
+
def _pin_memory(inp: TableTensor) -> TableTensor:
|
|
1418
|
+
blocks = {
|
|
1419
|
+
stype: tensor.pin_memory() if tensor.numel() > 0 else tensor
|
|
1420
|
+
for stype, tensor in inp.items()
|
|
1421
|
+
}
|
|
1422
|
+
return inp.__class__(
|
|
1423
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1424
|
+
**blocks,
|
|
1425
|
+
)
|
|
1426
|
+
|
|
1427
|
+
|
|
1428
|
+
@TableTensor.implements(aten.pin_memory.default)
|
|
1429
|
+
def _pin_memory_composite(
|
|
1430
|
+
inp: TableTensor,
|
|
1431
|
+
device: torch.device | None = None,
|
|
1432
|
+
) -> TableTensor:
|
|
1433
|
+
if _is_pinned(inp):
|
|
1434
|
+
return inp
|
|
1435
|
+
return _pin_memory(inp)
|
|
1436
|
+
|
|
1437
|
+
|
|
1438
|
+
@TableTensor.implements(aten.equal.default)
|
|
1439
|
+
def _equal(inp: TableTensor, other: Tensor) -> bool:
|
|
1440
|
+
if inp.__class__ is not other.__class__:
|
|
1441
|
+
return False
|
|
1442
|
+
if inp.size() != other.size():
|
|
1443
|
+
return False
|
|
1444
|
+
if inp.stypes != other.stypes:
|
|
1445
|
+
return False
|
|
1446
|
+
|
|
1447
|
+
for stype, block in _align_like(inp, other).items():
|
|
1448
|
+
if not block.equal(other.blocks[stype]):
|
|
1449
|
+
return False
|
|
1450
|
+
|
|
1451
|
+
return True
|
|
1452
|
+
|
|
1453
|
+
|
|
1454
|
+
@TableTensor.implements(aten.allclose.default)
|
|
1455
|
+
def _allclose(
|
|
1456
|
+
inp: TableTensor,
|
|
1457
|
+
other: Tensor,
|
|
1458
|
+
rtol: float = 1e-05,
|
|
1459
|
+
atol: float = 1e-08,
|
|
1460
|
+
equal_nan: bool = False,
|
|
1461
|
+
) -> bool:
|
|
1462
|
+
if inp.__class__ is not other.__class__:
|
|
1463
|
+
return False
|
|
1464
|
+
if inp.size() != other.size():
|
|
1465
|
+
return False
|
|
1466
|
+
if inp.stypes != other.stypes:
|
|
1467
|
+
return False
|
|
1468
|
+
|
|
1469
|
+
for stype, block in _align_like(inp, other).items():
|
|
1470
|
+
if stype != Stype.numerical: # Tolerance only applies for numerical:
|
|
1471
|
+
if not block.equal(other.blocks[stype]):
|
|
1472
|
+
return False
|
|
1473
|
+
elif not block.allclose(other.blocks[stype], rtol, atol, equal_nan):
|
|
1474
|
+
return False
|
|
1475
|
+
|
|
1476
|
+
return True
|
|
1477
|
+
|
|
1478
|
+
|
|
1479
|
+
@TableTensor.implements(aten.view.default)
|
|
1480
|
+
@preserve_view_inference_mode
|
|
1481
|
+
def _view(inp: TableTensor, size: Sequence[int]) -> TableTensor:
|
|
1482
|
+
size = tuple(size)
|
|
1483
|
+
for i, dim_size in enumerate(size):
|
|
1484
|
+
if dim_size < -1:
|
|
1485
|
+
raise RuntimeError(
|
|
1486
|
+
f"Invalid shape dimension {dim_size} at index {i} of shape "
|
|
1487
|
+
f"{size}"
|
|
1488
|
+
)
|
|
1489
|
+
|
|
1490
|
+
if size.count(-1) > 1:
|
|
1491
|
+
raise RuntimeError("Only one dimension can be inferred")
|
|
1492
|
+
|
|
1493
|
+
if -1 in size:
|
|
1494
|
+
known = math.prod(dim_size for dim_size in size if dim_size != -1)
|
|
1495
|
+
if known == 0:
|
|
1496
|
+
raise RuntimeError(
|
|
1497
|
+
f"Cannot reshape tensor of {inp.numel()} elements into "
|
|
1498
|
+
f"shape {size} because the unspecified dimension size -1 can "
|
|
1499
|
+
f"be any value and is ambiguous"
|
|
1500
|
+
)
|
|
1501
|
+
if inp.numel() % known != 0:
|
|
1502
|
+
raise RuntimeError(
|
|
1503
|
+
f"Shape {size} is invalid for input of size {inp.numel()}"
|
|
1504
|
+
)
|
|
1505
|
+
dim = size.index(-1)
|
|
1506
|
+
size = (*size[:dim], inp.numel() // known, *size[dim + 1 :])
|
|
1507
|
+
|
|
1508
|
+
if len(size) == 0 or size[-1] != inp.size(-1):
|
|
1509
|
+
_columns = "column" if inp.size(-1) == 1 else "columns"
|
|
1510
|
+
raise RuntimeError(
|
|
1511
|
+
f"Can't reshape {inp.__class__.__name__!r} with "
|
|
1512
|
+
f"{inp.size(-1)} {_columns} into shape {size}"
|
|
1513
|
+
)
|
|
1514
|
+
|
|
1515
|
+
blocks = {
|
|
1516
|
+
stype: tensor.view((*size[:-1], tensor.size(-1)))
|
|
1517
|
+
for stype, tensor in inp.items()
|
|
1518
|
+
}
|
|
1519
|
+
return inp.__class__(
|
|
1520
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1521
|
+
**blocks,
|
|
1522
|
+
)
|
|
1523
|
+
|
|
1524
|
+
|
|
1525
|
+
@TableTensor.implements(aten._unsafe_view.default)
|
|
1526
|
+
@preserve_view_inference_mode
|
|
1527
|
+
def _unsafe_view(inp: TableTensor, size: Sequence[int]) -> TableTensor:
|
|
1528
|
+
return _view(inp, size)
|
|
1529
|
+
|
|
1530
|
+
|
|
1531
|
+
@TableTensor.implements(aten.reshape.default)
|
|
1532
|
+
def _reshape(inp: TableTensor, size: Sequence[int]) -> TableTensor:
|
|
1533
|
+
return cast(
|
|
1534
|
+
TableTensor,
|
|
1535
|
+
aten.reshape.default.decompose(inp, size),
|
|
1536
|
+
)
|
|
1537
|
+
|
|
1538
|
+
|
|
1539
|
+
@TableTensor.implements(aten.flatten.using_ints)
|
|
1540
|
+
def _flatten(
|
|
1541
|
+
inp: TableTensor,
|
|
1542
|
+
start_dim: int = 0,
|
|
1543
|
+
end_dim: int = -1,
|
|
1544
|
+
) -> TableTensor:
|
|
1545
|
+
return cast(
|
|
1546
|
+
TableTensor,
|
|
1547
|
+
aten.flatten.using_ints.decompose(inp, start_dim, end_dim),
|
|
1548
|
+
)
|
|
1549
|
+
|
|
1550
|
+
|
|
1551
|
+
@TableTensor.implements(aten.squeeze.default)
|
|
1552
|
+
@preserve_view_inference_mode
|
|
1553
|
+
def _squeeze(inp: TableTensor) -> TableTensor:
|
|
1554
|
+
return _squeeze_dims(inp, range(inp.dim() - 1))
|
|
1555
|
+
|
|
1556
|
+
|
|
1557
|
+
@TableTensor.implements(aten.squeeze.dim)
|
|
1558
|
+
@preserve_view_inference_mode
|
|
1559
|
+
def _squeeze_dim(inp: TableTensor, dim: int) -> TableTensor:
|
|
1560
|
+
return _squeeze_dims(inp, (dim,))
|
|
1561
|
+
|
|
1562
|
+
|
|
1563
|
+
@TableTensor.implements(aten.squeeze.dims)
|
|
1564
|
+
@preserve_view_inference_mode
|
|
1565
|
+
def _squeeze_dims(inp: TableTensor, dim: Sequence[int]) -> TableTensor:
|
|
1566
|
+
dims = tuple(dim)
|
|
1567
|
+
blocks = {stype: tensor.squeeze(dims) for stype, tensor in inp.items()}
|
|
1568
|
+
|
|
1569
|
+
if inp.dim() - 1 in tuple(dim % inp.dim() for dim in dims):
|
|
1570
|
+
raise RuntimeError(
|
|
1571
|
+
f"Can't squeeze the column dimension of {inp.__class__.__name__!r}"
|
|
1572
|
+
)
|
|
1573
|
+
|
|
1574
|
+
return inp.__class__(
|
|
1575
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1576
|
+
**blocks,
|
|
1577
|
+
)
|
|
1578
|
+
|
|
1579
|
+
|
|
1580
|
+
@TableTensor.implements(aten.unsqueeze.default)
|
|
1581
|
+
@preserve_view_inference_mode
|
|
1582
|
+
def _unsqueeze(inp: TableTensor, dim: int) -> TableTensor:
|
|
1583
|
+
blocks = {stype: tensor.unsqueeze(dim) for stype, tensor in inp.items()}
|
|
1584
|
+
|
|
1585
|
+
if dim % (inp.dim() + 1) == inp.dim():
|
|
1586
|
+
raise RuntimeError(
|
|
1587
|
+
f"Can't unsqueeze after the column dimension of "
|
|
1588
|
+
f"{inp.__class__.__name__!r}"
|
|
1589
|
+
)
|
|
1590
|
+
|
|
1591
|
+
return inp.__class__(
|
|
1592
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1593
|
+
**blocks,
|
|
1594
|
+
)
|
|
1595
|
+
|
|
1596
|
+
|
|
1597
|
+
@TableTensor.implements(aten.expand.default)
|
|
1598
|
+
@preserve_view_inference_mode
|
|
1599
|
+
def _expand(
|
|
1600
|
+
inp: TableTensor,
|
|
1601
|
+
size: Sequence[int],
|
|
1602
|
+
*,
|
|
1603
|
+
implicit: bool = False,
|
|
1604
|
+
) -> TableTensor:
|
|
1605
|
+
size = tuple(size)
|
|
1606
|
+
if len(size) == 0 or size[-1] not in (-1, inp.size(-1)):
|
|
1607
|
+
_columns = "column" if inp.size(-1) == 1 else "columns"
|
|
1608
|
+
raise RuntimeError(
|
|
1609
|
+
f"Can't expand {inp.__class__.__name__!r} with "
|
|
1610
|
+
f"{inp.size(-1)} {_columns} to shape {size}"
|
|
1611
|
+
)
|
|
1612
|
+
|
|
1613
|
+
blocks = {
|
|
1614
|
+
stype: aten.expand.default(
|
|
1615
|
+
tensor,
|
|
1616
|
+
(*size[:-1], tensor.size(-1)),
|
|
1617
|
+
implicit=implicit,
|
|
1618
|
+
)
|
|
1619
|
+
for stype, tensor in inp.items()
|
|
1620
|
+
}
|
|
1621
|
+
return inp.__class__(
|
|
1622
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1623
|
+
**blocks,
|
|
1624
|
+
)
|
|
1625
|
+
|
|
1626
|
+
|
|
1627
|
+
@TableTensor.implements(aten.transpose.int)
|
|
1628
|
+
@preserve_view_inference_mode
|
|
1629
|
+
def _transpose(inp: TableTensor, dim0: int, dim1: int) -> TableTensor:
|
|
1630
|
+
blocks = {
|
|
1631
|
+
stype: tensor.transpose(dim0, dim1) for stype, tensor in inp.items()
|
|
1632
|
+
}
|
|
1633
|
+
|
|
1634
|
+
dim0 %= inp.dim()
|
|
1635
|
+
dim1 %= inp.dim()
|
|
1636
|
+
if dim0 != dim1 and inp.dim() - 1 in (dim0, dim1):
|
|
1637
|
+
raise RuntimeError(
|
|
1638
|
+
f"Can't transpose the column dimension of "
|
|
1639
|
+
f"{inp.__class__.__name__!r}"
|
|
1640
|
+
)
|
|
1641
|
+
|
|
1642
|
+
return inp.__class__(
|
|
1643
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1644
|
+
**blocks,
|
|
1645
|
+
)
|
|
1646
|
+
|
|
1647
|
+
|
|
1648
|
+
@TableTensor.implements(aten.permute.default)
|
|
1649
|
+
@preserve_view_inference_mode
|
|
1650
|
+
def _permute(inp: TableTensor, dims: Sequence[int]) -> TableTensor:
|
|
1651
|
+
dims = tuple(dims)
|
|
1652
|
+
blocks = {stype: tensor.permute(dims) for stype, tensor in inp.items()}
|
|
1653
|
+
|
|
1654
|
+
dims = tuple(dim % inp.dim() for dim in dims)
|
|
1655
|
+
if dims[-1] != inp.dim() - 1:
|
|
1656
|
+
raise RuntimeError(
|
|
1657
|
+
f"Can't permute the column dimension of {inp.__class__.__name__!r}"
|
|
1658
|
+
)
|
|
1659
|
+
|
|
1660
|
+
return inp.__class__(
|
|
1661
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1662
|
+
**blocks,
|
|
1663
|
+
)
|
|
1664
|
+
|
|
1665
|
+
|
|
1666
|
+
@TableTensor.implements(aten.movedim.int)
|
|
1667
|
+
def _movedim_int(
|
|
1668
|
+
inp: TableTensor,
|
|
1669
|
+
source: int,
|
|
1670
|
+
destination: int,
|
|
1671
|
+
) -> TableTensor:
|
|
1672
|
+
return cast(
|
|
1673
|
+
TableTensor,
|
|
1674
|
+
aten.movedim.int.decompose(inp, source, destination),
|
|
1675
|
+
)
|
|
1676
|
+
|
|
1677
|
+
|
|
1678
|
+
@TableTensor.implements(aten.movedim.intlist)
|
|
1679
|
+
def _movedim_intlist(
|
|
1680
|
+
inp: TableTensor,
|
|
1681
|
+
source: Sequence[int],
|
|
1682
|
+
destination: Sequence[int],
|
|
1683
|
+
) -> TableTensor:
|
|
1684
|
+
return cast(
|
|
1685
|
+
TableTensor,
|
|
1686
|
+
aten.movedim.intlist.decompose(inp, source, destination),
|
|
1687
|
+
)
|
|
1688
|
+
|
|
1689
|
+
|
|
1690
|
+
@TableTensor.implements(aten.select.int)
|
|
1691
|
+
@preserve_view_inference_mode
|
|
1692
|
+
def _select(inp: TableTensor, dim: int, index: int) -> TableTensor:
|
|
1693
|
+
if _is_column_dim(inp, dim):
|
|
1694
|
+
raise RuntimeError(
|
|
1695
|
+
f"Can't select the column dimension of {inp.__class__.__name__!r}"
|
|
1696
|
+
)
|
|
1697
|
+
|
|
1698
|
+
blocks = {
|
|
1699
|
+
stype: tensor.select(dim, index) for stype, tensor in inp.items()
|
|
1700
|
+
}
|
|
1701
|
+
|
|
1702
|
+
return inp.__class__(
|
|
1703
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1704
|
+
**blocks,
|
|
1705
|
+
)
|
|
1706
|
+
|
|
1707
|
+
|
|
1708
|
+
@TableTensor.implements(aten.slice.Tensor)
|
|
1709
|
+
@preserve_view_inference_mode
|
|
1710
|
+
def _slice(
|
|
1711
|
+
inp: TableTensor,
|
|
1712
|
+
dim: int = 0,
|
|
1713
|
+
start: int | None = None,
|
|
1714
|
+
end: int | None = None,
|
|
1715
|
+
step: int = 1,
|
|
1716
|
+
) -> TableTensor:
|
|
1717
|
+
if _is_column_dim(inp, dim):
|
|
1718
|
+
if (
|
|
1719
|
+
(start is None or start == 0 or start <= -inp.size(-1))
|
|
1720
|
+
and (end is None or end >= inp.size(dim))
|
|
1721
|
+
and step == 1
|
|
1722
|
+
):
|
|
1723
|
+
return _alias(inp)
|
|
1724
|
+
raise RuntimeError(
|
|
1725
|
+
f"Can't slice the column dimension of '{inp.__class__.__name__}'"
|
|
1726
|
+
)
|
|
1727
|
+
|
|
1728
|
+
blocks = {
|
|
1729
|
+
stype: aten.slice.Tensor(tensor, dim, start, end, step)
|
|
1730
|
+
for stype, tensor in inp.items()
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1733
|
+
return inp.__class__(
|
|
1734
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1735
|
+
**blocks,
|
|
1736
|
+
)
|
|
1737
|
+
|
|
1738
|
+
|
|
1739
|
+
@TableTensor.implements(aten.narrow.default)
|
|
1740
|
+
@preserve_view_inference_mode
|
|
1741
|
+
def _narrow(
|
|
1742
|
+
inp: TableTensor,
|
|
1743
|
+
dim: int,
|
|
1744
|
+
start: int,
|
|
1745
|
+
length: int,
|
|
1746
|
+
) -> TableTensor:
|
|
1747
|
+
if _is_column_dim(inp, dim):
|
|
1748
|
+
if (start == 0 or start == -inp.size(-1)) and length == inp.size(-1):
|
|
1749
|
+
return _alias(inp)
|
|
1750
|
+
raise RuntimeError(
|
|
1751
|
+
f"Can't narrow the column dimension of '{inp.__class__.__name__}'"
|
|
1752
|
+
)
|
|
1753
|
+
|
|
1754
|
+
blocks = {
|
|
1755
|
+
stype: tensor.narrow(dim, start, length)
|
|
1756
|
+
for stype, tensor in inp.items()
|
|
1757
|
+
}
|
|
1758
|
+
|
|
1759
|
+
return inp.__class__(
|
|
1760
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1761
|
+
**blocks,
|
|
1762
|
+
)
|
|
1763
|
+
|
|
1764
|
+
|
|
1765
|
+
@TableTensor.implements(aten.unbind.int)
|
|
1766
|
+
@preserve_view_inference_mode
|
|
1767
|
+
def _unbind(inp: TableTensor, dim: int = 0) -> tuple[TableTensor, ...]:
|
|
1768
|
+
if _is_column_dim(inp, dim):
|
|
1769
|
+
return _split(inp, split_size=1, dim=dim)
|
|
1770
|
+
|
|
1771
|
+
tensors_dict: dict[Stype, tuple[Tensor, ...]] = {
|
|
1772
|
+
stype: tensor.unbind(dim) for stype, tensor in inp.items()
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
stypes = tuple(tensors_dict.keys())
|
|
1776
|
+
return tuple(
|
|
1777
|
+
inp.__class__(
|
|
1778
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1779
|
+
**dict(zip(stypes, blocks)),
|
|
1780
|
+
)
|
|
1781
|
+
for blocks in zip(*tensors_dict.values())
|
|
1782
|
+
)
|
|
1783
|
+
|
|
1784
|
+
|
|
1785
|
+
@TableTensor.implements(aten.split.Tensor)
|
|
1786
|
+
@preserve_view_inference_mode
|
|
1787
|
+
def _split(
|
|
1788
|
+
inp: TableTensor,
|
|
1789
|
+
split_size: int,
|
|
1790
|
+
dim: int = 0,
|
|
1791
|
+
) -> tuple[TableTensor, ...]:
|
|
1792
|
+
if _is_column_dim(inp, dim):
|
|
1793
|
+
if split_size != 1:
|
|
1794
|
+
raise RuntimeError(
|
|
1795
|
+
f"Can only split the column dimension of "
|
|
1796
|
+
f"{inp.__class__.__name__!r} with split size 1"
|
|
1797
|
+
)
|
|
1798
|
+
return tuple(
|
|
1799
|
+
inp.__class__(
|
|
1800
|
+
columns={stype: (name,)},
|
|
1801
|
+
**{stype: tensor.narrow(-1, i, 1)},
|
|
1802
|
+
)
|
|
1803
|
+
for stype, tensor in inp.items()
|
|
1804
|
+
for i, name in enumerate(inp._columns[stype])
|
|
1805
|
+
)
|
|
1806
|
+
|
|
1807
|
+
tensors_dict: dict[Stype, tuple[Tensor, ...]] = {
|
|
1808
|
+
stype: tensor.split(split_size, dim) for stype, tensor in inp.items()
|
|
1809
|
+
}
|
|
1810
|
+
|
|
1811
|
+
stypes = tuple(tensors_dict.keys())
|
|
1812
|
+
return tuple(
|
|
1813
|
+
inp.__class__(
|
|
1814
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1815
|
+
**dict(zip(stypes, blocks)),
|
|
1816
|
+
)
|
|
1817
|
+
for blocks in zip(*tensors_dict.values())
|
|
1818
|
+
)
|
|
1819
|
+
|
|
1820
|
+
|
|
1821
|
+
@TableTensor.implements(aten.split.sizes)
|
|
1822
|
+
@TableTensor.implements(aten.split.default)
|
|
1823
|
+
@TableTensor.implements(aten.split_with_sizes.default)
|
|
1824
|
+
@preserve_view_inference_mode
|
|
1825
|
+
def _split_with_sizes(
|
|
1826
|
+
inp: TableTensor,
|
|
1827
|
+
split_sizes: Sequence[int],
|
|
1828
|
+
dim: int = 0,
|
|
1829
|
+
) -> tuple[TableTensor, ...]:
|
|
1830
|
+
if _is_column_dim(inp, dim):
|
|
1831
|
+
raise RuntimeError(
|
|
1832
|
+
f"Can't split the column dimension of {inp.__class__.__name__!r}"
|
|
1833
|
+
)
|
|
1834
|
+
|
|
1835
|
+
split_sizes = tuple(split_sizes)
|
|
1836
|
+
blocks_dict: dict[Stype, tuple[Tensor, ...]] = {
|
|
1837
|
+
stype: tensor.split(split_sizes, dim) for stype, tensor in inp.items()
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
stypes = tuple(blocks_dict.keys())
|
|
1841
|
+
return tuple(
|
|
1842
|
+
inp.__class__(
|
|
1843
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1844
|
+
**dict(zip(stypes, blocks)),
|
|
1845
|
+
)
|
|
1846
|
+
for blocks in zip(*blocks_dict.values())
|
|
1847
|
+
)
|
|
1848
|
+
|
|
1849
|
+
|
|
1850
|
+
@TableTensor.implements(aten.index_select.default)
|
|
1851
|
+
def _index_select(
|
|
1852
|
+
inp: TableTensor,
|
|
1853
|
+
dim: int,
|
|
1854
|
+
index: Tensor,
|
|
1855
|
+
) -> TableTensor:
|
|
1856
|
+
if _is_column_dim(inp, dim):
|
|
1857
|
+
raise RuntimeError(
|
|
1858
|
+
f"Can't index the column dimension of {inp.__class__.__name__!r}"
|
|
1859
|
+
)
|
|
1860
|
+
|
|
1861
|
+
blocks = {
|
|
1862
|
+
stype: tensor.index_select(dim, index) for stype, tensor in inp.items()
|
|
1863
|
+
}
|
|
1864
|
+
|
|
1865
|
+
return inp.__class__(
|
|
1866
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1867
|
+
**blocks,
|
|
1868
|
+
)
|
|
1869
|
+
|
|
1870
|
+
|
|
1871
|
+
@TableTensor.implements(aten.index.Tensor)
|
|
1872
|
+
def _index(
|
|
1873
|
+
inp: TableTensor,
|
|
1874
|
+
indices: Sequence[Tensor | None],
|
|
1875
|
+
) -> TableTensor:
|
|
1876
|
+
|
|
1877
|
+
current_dim = 0
|
|
1878
|
+
for index in indices:
|
|
1879
|
+
if index is None:
|
|
1880
|
+
current_dim += 1
|
|
1881
|
+
continue
|
|
1882
|
+
|
|
1883
|
+
# Check whether we index the column dimension:
|
|
1884
|
+
num_indexed_dims = index.dim() if index.dtype == torch.bool else 1
|
|
1885
|
+
if current_dim <= inp.dim() - 1 < current_dim + num_indexed_dims:
|
|
1886
|
+
raise RuntimeError(
|
|
1887
|
+
f"Can't index the column dimension of "
|
|
1888
|
+
f"{inp.__class__.__name__!r}"
|
|
1889
|
+
)
|
|
1890
|
+
current_dim += num_indexed_dims
|
|
1891
|
+
|
|
1892
|
+
blocks = {
|
|
1893
|
+
stype: aten.index.Tensor(tensor, indices)
|
|
1894
|
+
for stype, tensor in inp.items()
|
|
1895
|
+
}
|
|
1896
|
+
|
|
1897
|
+
return inp.__class__(
|
|
1898
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
|
|
1899
|
+
**blocks,
|
|
1900
|
+
)
|
|
1901
|
+
|
|
1902
|
+
|
|
1903
|
+
@TableTensor.implements(aten.cat.default)
|
|
1904
|
+
@preserve_autograd_state
|
|
1905
|
+
def _cat(tensors: Sequence[Tensor], dim: int = 0) -> TableTensor:
|
|
1906
|
+
if len(tensors) == 0:
|
|
1907
|
+
raise ValueError("torch.cat(): expected a non-empty list of Tensors")
|
|
1908
|
+
|
|
1909
|
+
if not all(isinstance(tensor, TableTensor) for tensor in tensors):
|
|
1910
|
+
raise TypeError(
|
|
1911
|
+
f"Expected all tensors to be {TableTensor.__name__!r} instances"
|
|
1912
|
+
)
|
|
1913
|
+
tensors = cast(Sequence[TableTensor], tensors)
|
|
1914
|
+
|
|
1915
|
+
ref = tensors[0]
|
|
1916
|
+
if not _is_column_dim(ref, dim):
|
|
1917
|
+
tensors = (ref, *(_align_like(tensor, ref) for tensor in tensors[1:]))
|
|
1918
|
+
|
|
1919
|
+
blocks: dict[Stype, Tensor] = {}
|
|
1920
|
+
for stype, _ in ref.items():
|
|
1921
|
+
block_list = [tensor.blocks[stype] for tensor in tensors]
|
|
1922
|
+
block_list = [block for block in block_list if block.size(-1) > 0]
|
|
1923
|
+
if len(block_list) == 1:
|
|
1924
|
+
blocks[stype] = block_list[0]
|
|
1925
|
+
elif len(block_list) > 1:
|
|
1926
|
+
blocks[stype] = torch.cat(block_list, dim=dim)
|
|
1927
|
+
|
|
1928
|
+
size: Sequence[int] | None = None
|
|
1929
|
+
if not _is_column_dim(ref, dim):
|
|
1930
|
+
columns = ref._columns
|
|
1931
|
+
if len(blocks) == 0:
|
|
1932
|
+
size = list(tensors[0].size())
|
|
1933
|
+
for tensor in tensors[1:]:
|
|
1934
|
+
size[dim] += tensor.size(dim)
|
|
1935
|
+
else:
|
|
1936
|
+
columns = {
|
|
1937
|
+
stype: tuple(
|
|
1938
|
+
chain.from_iterable(t._columns[stype] for t in tensors)
|
|
1939
|
+
)
|
|
1940
|
+
for stype, _ in ref.items()
|
|
1941
|
+
}
|
|
1942
|
+
size = ref.size()
|
|
1943
|
+
|
|
1944
|
+
return ref.__class__(
|
|
1945
|
+
size=size[:-1] if size is not None else None,
|
|
1946
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], columns),
|
|
1947
|
+
device=ref.device if len(blocks) == 0 else None,
|
|
1948
|
+
**blocks,
|
|
1949
|
+
)
|
|
1950
|
+
|
|
1951
|
+
|
|
1952
|
+
@TableTensor.implements(aten.stack.default)
|
|
1953
|
+
@preserve_autograd_state
|
|
1954
|
+
def _stack(tensors: Sequence[Tensor], dim: int = 0) -> TableTensor:
|
|
1955
|
+
if len(tensors) == 0:
|
|
1956
|
+
raise RuntimeError("stack expects a non-empty TensorList")
|
|
1957
|
+
|
|
1958
|
+
if not all(isinstance(tensor, TableTensor) for tensor in tensors):
|
|
1959
|
+
raise TypeError(
|
|
1960
|
+
f"Expected all tensors to be {TableTensor.__name__!r} instances"
|
|
1961
|
+
)
|
|
1962
|
+
|
|
1963
|
+
dim %= tensors[0].dim() + 1
|
|
1964
|
+
if dim >= tensors[0].dim():
|
|
1965
|
+
raise RuntimeError(
|
|
1966
|
+
f"Can't stack after the column dimension of "
|
|
1967
|
+
f"{tensors[0].__class__.__name__!r}"
|
|
1968
|
+
)
|
|
1969
|
+
|
|
1970
|
+
tensors = cast(Sequence[TableTensor], tensors)
|
|
1971
|
+
|
|
1972
|
+
ref = tensors[0]
|
|
1973
|
+
tensors = (ref, *(_align_like(tensor, ref) for tensor in tensors[1:]))
|
|
1974
|
+
|
|
1975
|
+
blocks = {
|
|
1976
|
+
stype: torch.stack([tensor.blocks[stype] for tensor in tensors], dim)
|
|
1977
|
+
for stype in ref._columns
|
|
1978
|
+
}
|
|
1979
|
+
|
|
1980
|
+
return ref.__class__(
|
|
1981
|
+
columns=cast(dict[StypeLike, tuple[str, ...]], ref._columns),
|
|
1982
|
+
**blocks,
|
|
1983
|
+
)
|
|
1984
|
+
|
|
1985
|
+
|
|
1986
|
+
# Helpers #####################################################################
|
|
1987
|
+
|
|
1988
|
+
|
|
1989
|
+
def _block_size_repr(size: Sequence[int]) -> str:
|
|
1990
|
+
if len(size) == 0:
|
|
1991
|
+
return "(*,)"
|
|
1992
|
+
if len(size) == 1:
|
|
1993
|
+
return f"({size[0]}, *)"
|
|
1994
|
+
return f"{str(tuple(size))[:-1]}, *)"
|
|
1995
|
+
|
|
1996
|
+
|
|
1997
|
+
def _is_column_dim(inp: Tensor, dim: int) -> bool:
|
|
1998
|
+
if dim < -inp.dim() or dim >= inp.dim():
|
|
1999
|
+
return False
|
|
2000
|
+
return dim % inp.dim() == inp.dim() - 1
|
|
2001
|
+
|
|
2002
|
+
|
|
2003
|
+
def _align_like(inp: TableTensor, ref: TableTensor) -> TableTensor:
|
|
2004
|
+
if inp._columns == ref._columns:
|
|
2005
|
+
return inp
|
|
2006
|
+
|
|
2007
|
+
if inp.stypes != ref.stypes:
|
|
2008
|
+
raise ValueError(
|
|
2009
|
+
"Expected tensors to have the same column names and stypes"
|
|
2010
|
+
)
|
|
2011
|
+
|
|
2012
|
+
blocks: dict[Stype, Tensor] = {}
|
|
2013
|
+
for stype, ref_columns in ref._columns.items():
|
|
2014
|
+
if len(ref_columns) > 0:
|
|
2015
|
+
column_to_index = {
|
|
2016
|
+
column: i for i, column in enumerate(inp._columns[stype])
|
|
2017
|
+
}
|
|
2018
|
+
indices = [column_to_index[column] for column in ref_columns]
|
|
2019
|
+
block = inp.blocks[stype]
|
|
2020
|
+
if indices == list(range(len(inp._columns[stype]))):
|
|
2021
|
+
blocks[stype] = block
|
|
2022
|
+
elif len(indices) == 1:
|
|
2023
|
+
blocks[stype] = block.narrow(-1, indices[0], 1)
|
|
2024
|
+
else:
|
|
2025
|
+
blocks[stype] = torch.cat(
|
|
2026
|
+
[block.narrow(-1, index, 1) for index in indices],
|
|
2027
|
+
dim=-1,
|
|
2028
|
+
)
|
|
2029
|
+
else:
|
|
2030
|
+
blocks[stype] = inp.blocks[stype]
|
|
2031
|
+
|
|
2032
|
+
return inp.__class__(
|
|
2033
|
+
columns=cast(Mapping[StypeLike, Sequence[str]], ref._columns),
|
|
2034
|
+
**blocks,
|
|
2035
|
+
)
|