structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/tensor/table.py ADDED
@@ -0,0 +1,2035 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from __future__ import annotations
5
+
6
+ import functools
7
+ import math
8
+ import warnings
9
+ from collections import defaultdict
10
+ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
11
+ from dataclasses import dataclass
12
+ from itertools import chain
13
+ from typing import TYPE_CHECKING, Any, ClassVar, Self, SupportsIndex, cast
14
+
15
+ import numpy as np
16
+ import pyarrow as pa
17
+ import torch
18
+ from torch import Tensor
19
+ from typing_extensions import override
20
+
21
+ from sdm import NaT, Stype, StypeLike
22
+ from sdm.tensor import CategoricalTensor, ColumnarTensor, StringTensor
23
+ from sdm.tensor.io import arrow_as_tensor, to_arrow, to_cudf
24
+ from sdm.tensor.mixin import _resolve_device
25
+
26
+ if TYPE_CHECKING:
27
+ import cudf
28
+ import pandas as pd
29
+
30
+ aten = torch.ops.aten
31
+
32
+ _NUMPY_TORCH_DTYPES = {
33
+ np.dtype(np.float16): torch.float16,
34
+ np.dtype(np.float32): torch.float32,
35
+ np.dtype(np.float64): torch.float64,
36
+ }
37
+ _TORCH_NUMPY_DTYPES = {
38
+ torch_dtype: numpy_dtype
39
+ for numpy_dtype, torch_dtype in _NUMPY_TORCH_DTYPES.items()
40
+ }
41
+
42
+
43
+ def preserve_view_inference_mode(fn: Callable) -> Callable:
44
+ r"""Preserve input inference state for tensor view operations."""
45
+
46
+ @functools.wraps(fn)
47
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
48
+ with torch.inference_mode(args[0].is_inference()):
49
+ return fn(*args, **kwargs)
50
+
51
+ return wrapper
52
+
53
+
54
+ def preserve_autograd_state(fn: Callable) -> Callable:
55
+ r"""Re-apply the caller's autograd/inference-mode state for non-view ops.
56
+
57
+ ``__torch_dispatch__`` redispatch does not automatically retain the
58
+ caller's inference-mode and grad-enabled state for non-view ops, so a
59
+ real (non-view) copy performed here would otherwise silently lose
60
+ ``requires_grad``/``grad_fn`` even though gradients are enabled in the
61
+ caller's scope.
62
+ """
63
+
64
+ @functools.wraps(fn)
65
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
66
+ inference_mode_enabled = torch.is_inference_mode_enabled()
67
+ grad_enabled = torch.is_grad_enabled()
68
+ with (
69
+ torch.inference_mode(inference_mode_enabled),
70
+ torch.set_grad_enabled(grad_enabled),
71
+ ):
72
+ return fn(*args, **kwargs)
73
+
74
+ return wrapper
75
+
76
+
77
+ def _move_categories(
78
+ categories: Sequence[Tensor],
79
+ device: torch.device | str | None,
80
+ ) -> tuple[Tensor, ...]:
81
+ device = _resolve_device(device)
82
+ if device is None or device.type == "cpu":
83
+ return tuple(categories)
84
+
85
+ groups: dict[
86
+ tuple[type[Tensor], torch.dtype, torch.dtype | None], list[int]
87
+ ] = defaultdict(list)
88
+ moved: list[Tensor | None] = [None] * len(categories)
89
+ for i, category in enumerate(categories):
90
+ if type(category) is Tensor or isinstance(category, StringTensor):
91
+ offset_dtype = (
92
+ category.data_offset[1].dtype
93
+ if isinstance(category, StringTensor)
94
+ else None
95
+ )
96
+ groups[(type(category), category.dtype, offset_dtype)].append(i)
97
+ else:
98
+ moved[i] = category.to(device)
99
+
100
+ for indices in groups.values():
101
+ tensors = [categories[i] for i in indices]
102
+ if len(tensors) == 1:
103
+ parts = (tensors[0].to(device),)
104
+ else:
105
+ sizes = [tensor.numel() for tensor in tensors]
106
+ parts = torch.cat(tensors).to(device).split(sizes)
107
+ for i, part in zip(indices, parts):
108
+ moved[i] = part
109
+
110
+ assert all(category is not None for category in moved)
111
+ return cast(tuple[Tensor, ...], tuple(moved))
112
+
113
+
114
+ @dataclass(frozen=True)
115
+ class TableSchema:
116
+ r"""The schema of a :class:`TableTensor`.
117
+
118
+ Args:
119
+ columns: Column names grouped by semantic type.
120
+ """
121
+
122
+ columns: Mapping[Stype, tuple[str, ...]]
123
+
124
+ def __hash__(self) -> int:
125
+ return hash(frozenset(self.columns.items()))
126
+
127
+
128
+ class TableTensor(Tensor):
129
+ r"""A :class:`torch.Tensor` for tensorized, lossless table data.
130
+
131
+ A :class:`TableTensor` stores column blocks separately per semantic type,
132
+ while exposing a single tensor-shaped table interface.
133
+ The last dimension represents named columns.
134
+
135
+ .. testcode:: drop_stypes, select_columns, drop_columns
136
+
137
+ import torch
138
+ from sdm import CategoricalTensor, StringTensor, Stype, TableTensor
139
+
140
+ table = TableTensor(
141
+ columns={
142
+ "numerical": ["age", "income"],
143
+ "categorical": ["country", "segment"],
144
+ },
145
+ numerical=torch.randn(10, 2),
146
+ categorical=CategoricalTensor(
147
+ code=torch.randint(0, 2, size=(10, 2)),
148
+ categories=(
149
+ StringTensor.from_list(["USA", "Germany"]),
150
+ StringTensor.from_list(["enterprise", "startup"]),
151
+ ),
152
+ ),
153
+ )
154
+
155
+ print(table)
156
+ # TableTensor(
157
+ # size=(10, 4),
158
+ # blocks={
159
+ # numerical (2): ['age', 'income'],
160
+ # categorical (2): ['country', 'segment'],
161
+ # },
162
+ # )
163
+
164
+ # DataFrame-like column selection, but still tensor-native:
165
+ features = table[["age", "country"]]
166
+ assert features.size() == (10, 2)
167
+
168
+ # Normal PyTorch indexing still works on row/batch dimensions:
169
+ batch = table[[1, 0, 2], ["income", "segment"]]
170
+ assert batch.size() == (3, 2)
171
+
172
+ # Semantic blocks stay separate for model input:
173
+ x_num = table.numerical
174
+ x_cat = table.categorical
175
+
176
+ # Tensor ops preserve the table container:
177
+ stacked = torch.stack([table, table], dim=0)
178
+ assert stacked.size() == (2, 10, 4)
179
+
180
+ # Column-wise cat extends the schema:
181
+ wide = torch.cat([table[["age"]], table[["country"]]], dim=-1)
182
+
183
+ .. testoutput:: drop_stypes, select_columns, drop_columns
184
+ :hide:
185
+ :options: +ELLIPSIS
186
+
187
+ ...
188
+
189
+ Args:
190
+ size: The shape of the tensor ``[..., C]``.
191
+ columns: Column names grouped by semantic type.
192
+ numerical: The numerical column block of shape ``[..., C_num]``.
193
+ categorical: The categorical column block of shape ``[..., C_cat]``.
194
+ datetime: The ``datetime64[us]`` column block of shape ``[..., C_dt]``.
195
+ text: The text column block of shape ``[..., C_text]``.
196
+ id: The identifier column block of shape ``[..., C_id]``.
197
+ device: The device.
198
+ """
199
+
200
+ HANDLED_FUNCTIONS: ClassVar[
201
+ dict[Callable[..., Any], Callable[..., Any]]
202
+ ] = {}
203
+
204
+ _numerical: Tensor
205
+ _categorical: CategoricalTensor
206
+ _datetime: Tensor
207
+ _text: StringTensor
208
+ _id: ColumnarTensor
209
+ _columns: dict[Stype, tuple[str, ...]]
210
+ _column_to_loc: dict[str, tuple[Stype, int]]
211
+
212
+ # Route tensor operations through `__torch_dispatch__` only.
213
+ __torch_function__ = torch._C._disabled_torch_function_impl # type: ignore
214
+
215
+ # Constructors ############################################################
216
+
217
+ def __init__(
218
+ self,
219
+ size: Sequence[int] | None = None,
220
+ columns: Mapping[StypeLike, Sequence[str]] | None = None,
221
+ numerical: Tensor | None = None,
222
+ categorical: CategoricalTensor | None = None,
223
+ datetime: Tensor | None = None,
224
+ text: StringTensor | None = None,
225
+ id: ColumnarTensor | None = None,
226
+ device: torch.device | str | None = None,
227
+ ) -> None:
228
+ pass
229
+
230
+ def __new__(
231
+ cls,
232
+ size: Sequence[int] | None = None,
233
+ columns: Mapping[StypeLike, Sequence[str]] | None = None,
234
+ numerical: Tensor | None = None,
235
+ categorical: CategoricalTensor | None = None,
236
+ datetime: Tensor | None = None,
237
+ text: StringTensor | None = None,
238
+ id: ColumnarTensor | None = None,
239
+ device: torch.device | str | None = None,
240
+ ) -> Self:
241
+ r"""Create a tensor wrapper."""
242
+ if size is not None and len(size) == 0:
243
+ raise ValueError("Expected 'size' to be non-empty")
244
+
245
+ size = tuple(size) if size is not None else size
246
+ device = _resolve_device(device)
247
+
248
+ for stype, block in (
249
+ (Stype.numerical, numerical),
250
+ (Stype.categorical, categorical),
251
+ (Stype.datetime, datetime),
252
+ (Stype.text, text),
253
+ (Stype.id, id),
254
+ ):
255
+ if block is None:
256
+ continue
257
+
258
+ if stype == Stype.datetime and block.dtype != torch.int64:
259
+ raise ValueError(
260
+ f"Expected {str(stype)!r} block to have dtype "
261
+ f"'{torch.int64}' (got '{block.dtype}')"
262
+ )
263
+
264
+ size = tuple(block.size()[:-1]) if size is None else size
265
+ device = block.device if device is None else device
266
+
267
+ if block.dim() < 2:
268
+ raise ValueError(
269
+ f"Expected {str(stype)!r} block to be at least 2D "
270
+ f"(got {block.dim()}D)"
271
+ )
272
+ if size != block.size()[:-1]:
273
+ raise ValueError(
274
+ f"Expected {str(stype)!r} block size of "
275
+ f"{_block_size_repr(size)} (got {tuple(block.size())})"
276
+ )
277
+ if device != block.device:
278
+ raise ValueError(
279
+ f"Expected {str(stype)!r} block to be on device "
280
+ f"'{device}' (got '{block.device}')"
281
+ )
282
+
283
+ if size is None:
284
+ raise ValueError(
285
+ "Expected 'size' to be given when all blocks are 'None'"
286
+ )
287
+
288
+ if numerical is None:
289
+ numerical = torch.empty((*size, 0), device=device)
290
+ if categorical is None:
291
+ categorical = CategoricalTensor(
292
+ code=torch.empty((*size, 0), dtype=torch.int32, device=device),
293
+ categories=(),
294
+ )
295
+ if datetime is None:
296
+ datetime = torch.empty((*size, 0), dtype=torch.long, device=device)
297
+ if text is None:
298
+ text = StringTensor(
299
+ data=torch.empty(0, dtype=torch.uint8, device=device),
300
+ offset=torch.zeros(1, dtype=torch.int32, device=device),
301
+ valid=None,
302
+ size=(*size, 0),
303
+ )
304
+ if id is None:
305
+ id = ColumnarTensor((), size=size, device=device)
306
+
307
+ columns = {
308
+ Stype(stype): tuple(names)
309
+ for stype, names in (columns or {}).items()
310
+ }
311
+
312
+ for stype, block, name in (
313
+ (Stype.numerical, numerical, "num"),
314
+ (Stype.categorical, categorical, "cat"),
315
+ (Stype.datetime, datetime, "dt"),
316
+ (Stype.text, text, "text"),
317
+ (Stype.id, id, "id"),
318
+ ):
319
+ if stype not in columns:
320
+ columns[stype] = tuple(
321
+ f"{name}_{i}" for i in range(block.size(-1))
322
+ )
323
+
324
+ if block.size(-1) != len(columns[stype]):
325
+ _columns = "column" if len(columns[stype]) == 1 else "columns"
326
+ raise ValueError(
327
+ f"Expected {str(stype)!r} block to hold "
328
+ f"{len(columns[stype])} {_columns} (got {block.size(-1)})"
329
+ )
330
+
331
+ column_names = list(chain.from_iterable(columns.values()))
332
+ column_to_loc: dict[str, tuple[Stype, int]] = {}
333
+ for stype_like, names in columns.items():
334
+ stype = Stype(stype_like)
335
+ for i, name in enumerate(names):
336
+ column_to_loc[name] = (stype, i)
337
+ if len(column_names) != len(column_to_loc):
338
+ raise ValueError("Expected column names to be unique")
339
+
340
+ out = Tensor._make_wrapper_subclass(
341
+ cls,
342
+ size=(*size, len(column_names)),
343
+ dtype=numerical.dtype,
344
+ device=numerical.device,
345
+ requires_grad=False,
346
+ )
347
+
348
+ out._numerical = numerical
349
+ out._categorical = categorical
350
+ out._datetime = datetime
351
+ out._text = text
352
+ out._id = id
353
+ out._columns = columns
354
+ out._column_to_loc = column_to_loc
355
+
356
+ return out
357
+
358
+ @classmethod
359
+ def from_arrow(
360
+ cls,
361
+ table: pa.Table,
362
+ stypes: Mapping[str, StypeLike],
363
+ *,
364
+ device: torch.device | str | None = None,
365
+ ) -> Self:
366
+ r"""Create a tensor from a :class:`pyarrow.Table`.
367
+
368
+ .. testcode::
369
+
370
+ import pyarrow as pa
371
+ from sdm import TableTensor
372
+
373
+ arrow_table = pa.table({
374
+ "age": pa.array([25, 31, 42], type=pa.int64()),
375
+ "city": pa.array(["SF", "NYC", "SF"], type=pa.string()),
376
+ })
377
+ tensor = TableTensor.from_arrow(
378
+ table=arrow_table,
379
+ stypes={"age": "numerical", "city": "categorical"},
380
+ )
381
+
382
+ Args:
383
+ table: The table.
384
+ stypes: The semantic type for each column. Columns that are present
385
+ in ``table`` but not included in ``stypes`` will be ignored.
386
+ device: The device.
387
+ """
388
+ columns: dict[Stype, list[str]] = defaultdict(list)
389
+ for column, stype in stypes.items():
390
+ columns[Stype(stype)].append(column)
391
+
392
+ blocks: dict[Stype, Tensor] = {}
393
+ for stype in columns:
394
+ tensors: list[Tensor] = []
395
+ for column in columns[stype]:
396
+ array = table.column(column)
397
+ if stype == Stype.numerical:
398
+ tensor = arrow_as_tensor(
399
+ array,
400
+ dtype=torch.get_default_dtype(),
401
+ ).unsqueeze(-1)
402
+ elif stype == Stype.categorical:
403
+ tensor = CategoricalTensor.from_arrow(array)
404
+ elif stype == Stype.datetime:
405
+ array = array.cast(pa.timestamp("us"))
406
+ values = array.to_numpy(zero_copy_only=False)
407
+ values = values.astype("int64")
408
+ tensor = torch.from_numpy(values).unsqueeze(-1)
409
+ elif stype == Stype.text:
410
+ tensor = StringTensor.from_arrow(array).unsqueeze(-1)
411
+ elif stype == Stype.id:
412
+ tensor = ColumnarTensor.from_arrow(array)
413
+ else:
414
+ raise NotImplementedError
415
+ tensors.append(tensor)
416
+
417
+ blocks[stype] = torch.cat(tensors, dim=-1).to(device)
418
+
419
+ return cls(
420
+ columns=cast(Mapping[StypeLike, Sequence[str]], columns),
421
+ **blocks,
422
+ )
423
+
424
+ def to_arrow(self) -> pa.Table:
425
+ r"""Convert this tensor to a :class:`pyarrow.Table`."""
426
+ arrays: list[pa.Array] = []
427
+ columns: list[str] = []
428
+ for stype, tensor in self.items():
429
+ if tensor.size(-1) == 0:
430
+ continue
431
+
432
+ columns.extend(self._columns[stype])
433
+
434
+ if isinstance(tensor, CategoricalTensor | ColumnarTensor):
435
+ arrays.extend(tensor.to_arrow().itercolumns())
436
+ elif isinstance(tensor, StringTensor):
437
+ tensor = cast(
438
+ StringTensor,
439
+ tensor.movedim(-1, 0).contiguous().cpu(),
440
+ )
441
+ arrays.extend(cast(StringTensor, t).to_arrow() for t in tensor)
442
+ elif stype == Stype.datetime:
443
+ tensor = tensor.movedim(-1, 0).contiguous().cpu()
444
+ array = tensor.numpy().reshape(tensor.size(0), -1)
445
+ array = array.view("datetime64[us]")
446
+ arrays.extend(pa.array(a) for a in array)
447
+ else:
448
+ tensor = tensor.detach().movedim(-1, 0).contiguous().cpu()
449
+ arrays.extend(to_arrow(t) for t in tensor)
450
+
451
+ return pa.Table.from_arrays(arrays, names=columns)
452
+
453
+ @classmethod
454
+ def from_pandas(
455
+ cls,
456
+ df: pd.DataFrame,
457
+ stypes: Mapping[str, StypeLike],
458
+ *,
459
+ device: torch.device | str | None = None,
460
+ ) -> Self:
461
+ r"""Create a tensor from a :class:`pandas.DataFrame`.
462
+
463
+ Args:
464
+ df: The dataframe.
465
+ stypes: The semantic type for each column. Columns that are present
466
+ in ``df`` but not included in ``stypes`` will be ignored.
467
+ device: The device.
468
+ """
469
+ import pandas as pd
470
+
471
+ columns: dict[Stype, list[str]] = defaultdict(list)
472
+ for name, stype in stypes.items():
473
+ columns[Stype(stype)].append(name)
474
+
475
+ if columns and set(columns).issubset(
476
+ {Stype.numerical, Stype.categorical}
477
+ ):
478
+ target_device = (
479
+ torch.get_default_device() if device is None else device
480
+ )
481
+ blocks: dict[Stype, Tensor] = {}
482
+
483
+ if numerical_columns := columns.get(Stype.numerical):
484
+ dtype = torch.get_default_dtype()
485
+ numpy_dtype = _TORCH_NUMPY_DTYPES.get(
486
+ dtype, np.dtype(np.float32)
487
+ )
488
+ numerical_df = (
489
+ df
490
+ if len(numerical_columns) == len(df.columns)
491
+ and numerical_columns == df.columns.tolist()
492
+ else df[numerical_columns]
493
+ )
494
+ if len(df) == 0:
495
+ if any(
496
+ pd.api.types.is_complex_dtype(dtype)
497
+ for dtype in numerical_df.dtypes
498
+ ):
499
+ raise TypeError(
500
+ "Expected numerical columns to contain real values"
501
+ )
502
+ blocks[Stype.numerical] = torch.empty(
503
+ (0, len(numerical_columns)),
504
+ device=target_device,
505
+ dtype=dtype,
506
+ )
507
+ else:
508
+ try:
509
+ with warnings.catch_warnings():
510
+ warnings.simplefilter(
511
+ "error", np.exceptions.ComplexWarning
512
+ )
513
+ values = numerical_df.to_numpy(
514
+ dtype=numpy_dtype,
515
+ na_value=np.nan,
516
+ )
517
+ except np.exceptions.ComplexWarning as error:
518
+ raise TypeError(
519
+ "Expected numerical columns to contain real values"
520
+ ) from error
521
+ values = values.copy(order="C")
522
+ blocks[Stype.numerical] = torch.as_tensor(
523
+ values,
524
+ device=target_device,
525
+ dtype=dtype,
526
+ )
527
+
528
+ if categorical_columns := columns.get(Stype.categorical):
529
+ categorical_series = [df[name] for name in categorical_columns]
530
+ all_categorical = all(
531
+ isinstance(ser.dtype, pd.CategoricalDtype)
532
+ for ser in categorical_series
533
+ )
534
+ use_int64_code = not all_categorical and any(
535
+ isinstance(ser.dtype, pd.ArrowDtype)
536
+ and pa.types.is_dictionary(ser.dtype.pyarrow_dtype)
537
+ and pa.types.is_int64(ser.dtype.pyarrow_dtype.index_type)
538
+ for ser in categorical_series
539
+ )
540
+ numpy_code_dtype = np.int64 if use_int64_code else np.int32
541
+ torch_code_dtype = (
542
+ torch.int64 if use_int64_code else torch.int32
543
+ )
544
+ if all_categorical:
545
+ code = np.stack(
546
+ [ser.array.codes for ser in categorical_series], axis=1
547
+ ).astype(
548
+ numpy_code_dtype,
549
+ copy=False,
550
+ )
551
+ else:
552
+ code = np.empty(
553
+ (len(df), len(categorical_columns)),
554
+ dtype=numpy_code_dtype,
555
+ )
556
+ categories: list[Tensor] = []
557
+ for i, ser in enumerate(categorical_series):
558
+ if all_categorical:
559
+ categorical = ser.array
560
+ values = categorical.categories
561
+ elif isinstance(ser.dtype, pd.CategoricalDtype):
562
+ categorical = ser.array
563
+ code[:, i] = categorical.codes
564
+ values = categorical.categories
565
+ elif ser.dtype.kind == "f" or (
566
+ isinstance(ser.dtype, pd.ArrowDtype)
567
+ and pa.types.is_dictionary(ser.dtype.pyarrow_dtype)
568
+ ):
569
+ categorical = CategoricalTensor.from_arrow(
570
+ pa.array(ser, from_pandas=True)
571
+ )
572
+ code[:, i] = categorical.code[:, 0].numpy()
573
+ categories.append(categorical.categories[0])
574
+ continue
575
+ else:
576
+ column_code, values = pd.factorize(
577
+ ser,
578
+ sort=False,
579
+ use_na_sentinel=True,
580
+ )
581
+ code[:, i] = column_code
582
+
583
+ category_dtype = values.dtype
584
+ if category_dtype.kind in "biuf":
585
+ category = torch.from_numpy(np.asarray(values).copy())
586
+ else:
587
+ array = pa.array(values, from_pandas=True)
588
+ is_string = pa.types.is_string(array.type)
589
+ is_large_string = pa.types.is_large_string(array.type)
590
+ if is_string or is_large_string:
591
+ category = StringTensor.from_arrow(
592
+ array,
593
+ device="cpu",
594
+ )
595
+ elif pa.types.is_null(array.type):
596
+ category = torch.empty(
597
+ 0,
598
+ dtype=torch.int64,
599
+ device="cpu",
600
+ )
601
+ elif ser.dtype.kind == "O" and (
602
+ pa.types.is_boolean(array.type)
603
+ or pa.types.is_integer(array.type)
604
+ or pa.types.is_floating(array.type)
605
+ ):
606
+ categorical = CategoricalTensor.from_arrow(
607
+ pa.array(ser, from_pandas=True)
608
+ )
609
+ code[:, i] = categorical.code[:, 0].numpy()
610
+ category = categorical.categories[0]
611
+ else:
612
+ category = arrow_as_tensor(array, device="cpu")
613
+ categories.append(category)
614
+
615
+ code_tensor = (
616
+ torch.from_numpy(code).to(target_device)
617
+ if len(df) > 0
618
+ else torch.empty(
619
+ (0, len(categorical_columns)),
620
+ dtype=torch_code_dtype,
621
+ device=target_device,
622
+ )
623
+ )
624
+ blocks[Stype.categorical] = CategoricalTensor(
625
+ code=code_tensor,
626
+ categories=_move_categories(categories, target_device),
627
+ )
628
+
629
+ return cls(
630
+ size=(len(df),),
631
+ columns=cast(Mapping[StypeLike, Sequence[str]], columns),
632
+ **blocks,
633
+ )
634
+
635
+ df = df[stypes.keys()]
636
+ # Resolve period ordinals before Arrow casts them as timestamps, which
637
+ # would interpret the ordinals as microseconds since the Unix epoch.
638
+ period_timestamps = {}
639
+ for name, stype in stypes.items():
640
+ if Stype(stype) != Stype.datetime:
641
+ continue
642
+ if not isinstance(df[name].dtype, pd.PeriodDtype):
643
+ continue
644
+ period_timestamps[name] = df[name].dt.to_timestamp(how="start")
645
+ if period_timestamps:
646
+ df = df.assign(**period_timestamps)
647
+
648
+ return cls.from_arrow(
649
+ table=pa.Table.from_pandas(df, preserve_index=False),
650
+ stypes=stypes,
651
+ device=device,
652
+ )
653
+
654
+ @classmethod
655
+ def from_columns(
656
+ cls,
657
+ data: Mapping[str, Sequence[Any]],
658
+ stypes: Mapping[str, StypeLike],
659
+ *,
660
+ device: torch.device | str | None = None,
661
+ ) -> Self:
662
+ r"""Create a tensor from column data.
663
+
664
+ Args:
665
+ data: Column data keyed by column name.
666
+ stypes: The semantic type for each column. Columns that are present
667
+ in ``data`` but not included in ``stypes`` will be ignored.
668
+ device: The device.
669
+ """
670
+ import pandas as pd
671
+
672
+ return cls.from_pandas(
673
+ df=pd.DataFrame(data),
674
+ stypes=stypes,
675
+ device=device,
676
+ )
677
+
678
+ def to_pandas(self) -> pd.DataFrame:
679
+ r"""Convert this tensor to a :class:`pandas.DataFrame`."""
680
+ return self.to_arrow().to_pandas()
681
+
682
+ @classmethod
683
+ def from_tensor(
684
+ cls,
685
+ tensor: Tensor,
686
+ columns: Sequence[str] | None = None,
687
+ ) -> Self:
688
+ r"""Create a table from a :class:`torch.Tensor`.
689
+
690
+ Args:
691
+ tensor: The input tensor with shape ``[..., C]``, interpreted as:
692
+
693
+ * A floating-point :class:`torch.Tensor` becomes numerical
694
+ columns.
695
+ * An integer :class:`torch.Tensor` becomes categorical columns.
696
+ * A :class:`StringTensor` becomes text columns.
697
+ columns: The ``C`` column names.
698
+ """
699
+ if columns is None:
700
+ columns = [str(i) for i in range(tensor.size(-1))]
701
+
702
+ if isinstance(tensor, StringTensor):
703
+ return cls(
704
+ columns={Stype.text: columns},
705
+ text=tensor,
706
+ )
707
+
708
+ if not tensor.is_floating_point():
709
+ return cls(
710
+ columns={Stype.categorical: columns},
711
+ categorical=CategoricalTensor.from_tensor(tensor),
712
+ )
713
+
714
+ return cls(
715
+ columns={Stype.numerical: columns},
716
+ numerical=tensor,
717
+ )
718
+
719
+ @classmethod
720
+ def from_cudf(
721
+ cls,
722
+ df: cudf.DataFrame,
723
+ stypes: Mapping[str, StypeLike],
724
+ *,
725
+ device: torch.device | str | None = None,
726
+ ) -> Self:
727
+ r"""Create a tensor from a :class:`cudf.DataFrame`.
728
+
729
+ Args:
730
+ df: The dataframe.
731
+ stypes: The semantic type for each column. Columns that are present
732
+ in ``df`` but not included in ``stypes`` will be ignored.
733
+ device: The device.
734
+ """
735
+ columns: dict[Stype, list[str]] = defaultdict(list)
736
+ for column, stype in stypes.items():
737
+ columns[Stype(stype)].append(column)
738
+
739
+ blocks: dict[Stype, Tensor] = {}
740
+ for stype in columns:
741
+ tensors: list[Tensor] = []
742
+ for column in columns[stype]:
743
+ ser = df[column]
744
+ if stype == Stype.numerical:
745
+ ser = ser.astype("float32", copy=False)
746
+ if ser.null_count > 0:
747
+ ser = ser.fillna(float("nan"))
748
+ tensor = torch.from_dlpack(ser.to_dlpack()).unsqueeze(-1)
749
+ tensor = tensor.to(device)
750
+ elif stype == Stype.categorical:
751
+ tensor = CategoricalTensor.from_cudf(ser, device=device)
752
+ elif stype == Stype.datetime:
753
+ ser = ser.astype("datetime64[us]", copy=False)
754
+ ser = ser.astype("int64", copy=False)
755
+ if ser.null_count > 0:
756
+ ser = ser.fillna(NaT)
757
+ tensor = torch.from_dlpack(ser.to_dlpack()).unsqueeze(-1)
758
+ tensor = tensor.to(device)
759
+ elif stype == Stype.text:
760
+ tensor = StringTensor.from_cudf(ser, device=device)
761
+ tensor = tensor.unsqueeze(-1)
762
+ elif stype == Stype.id:
763
+ tensor = ColumnarTensor.from_cudf(ser, device=device)
764
+ else:
765
+ raise NotImplementedError
766
+ tensors.append(tensor)
767
+
768
+ blocks[stype] = torch.cat(tensors, dim=-1)
769
+
770
+ return cls(
771
+ columns=cast(Mapping[StypeLike, Sequence[str]], columns),
772
+ **blocks,
773
+ )
774
+
775
+ def to_cudf(self) -> cudf.DataFrame:
776
+ r"""Convert this tensor to a :class:`cudf.DataFrame`."""
777
+ import cudf
778
+
779
+ dfs: list[cudf.DataFrame] = []
780
+ for stype, tensor in self.items():
781
+ if tensor.size(-1) == 0:
782
+ continue
783
+
784
+ if isinstance(tensor, CategoricalTensor | ColumnarTensor):
785
+ dfs.append(tensor.to_cudf(self._columns[stype]))
786
+ elif isinstance(tensor, StringTensor):
787
+ tensor = cast(StringTensor, tensor.movedim(-1, 0).contiguous())
788
+ df = cudf.DataFrame(
789
+ {
790
+ name: cast(StringTensor, column).to_cudf()
791
+ for name, column in zip(self._columns[stype], tensor)
792
+ }
793
+ )
794
+ dfs.append(df)
795
+ elif stype == Stype.datetime:
796
+ tensor = tensor.movedim(-1, 0).contiguous()
797
+ df = cudf.DataFrame(
798
+ {
799
+ name: to_cudf(column, mask).astype(
800
+ "datetime64[us]", copy=False
801
+ )
802
+ for name, column, mask in zip(
803
+ self._columns[stype],
804
+ tensor,
805
+ tensor != NaT,
806
+ )
807
+ }
808
+ )
809
+ dfs.append(df)
810
+ else:
811
+ tensor = tensor.detach().movedim(-1, 0).contiguous()
812
+ valid = (
813
+ tensor.isnan().logical_not_()
814
+ if tensor.is_floating_point()
815
+ else None
816
+ )
817
+ col_dict: dict[str, cudf.Series] = {}
818
+ for idx, (name, column) in enumerate(
819
+ zip(self._columns[stype], tensor)
820
+ ):
821
+ col_dict[name] = to_cudf(
822
+ tensor=column,
823
+ valid_mask=None if valid is None else valid[idx],
824
+ )
825
+
826
+ df = cudf.DataFrame(col_dict)
827
+ dfs.append(df)
828
+
829
+ if len(dfs) == 1:
830
+ return dfs[0]
831
+
832
+ return cudf.concat(dfs, axis=1)
833
+
834
+ # Properties ##############################################################
835
+
836
+ @property
837
+ def columns(self) -> Mapping[Stype, tuple[str, ...]]:
838
+ r"""Return column names grouped by semantic type."""
839
+ return self._columns.copy()
840
+
841
+ @property
842
+ def column_names(self) -> frozenset[str]:
843
+ r"""The column names of this tensor."""
844
+ return frozenset(self._column_to_loc)
845
+
846
+ @property
847
+ def stypes(self) -> Mapping[str, Stype]:
848
+ r"""Return the semantic type for each column."""
849
+ return {key: stype for key, (stype, _) in self._column_to_loc.items()}
850
+
851
+ def stype(self, column: str) -> Stype:
852
+ r"""Return the semantic type for a column.
853
+
854
+ Args:
855
+ column: The column name.
856
+ """
857
+ return self._column_to_loc[column][0]
858
+
859
+ @property
860
+ def active_stypes(self) -> frozenset[Stype]:
861
+ r"""Semantic types with at least one column."""
862
+ return frozenset(
863
+ stype for stype, tensor in self.items() if tensor.size(-1) > 0
864
+ )
865
+
866
+ @property
867
+ def numerical(self) -> Tensor:
868
+ r"""Return the numerical column block."""
869
+ return self._numerical
870
+
871
+ @property
872
+ def categorical(self) -> CategoricalTensor:
873
+ r"""Return the categorical column block."""
874
+ return self._categorical
875
+
876
+ @property
877
+ def datetime(self) -> Tensor:
878
+ r"""Return the datetime column block."""
879
+ return self._datetime
880
+
881
+ @property
882
+ def text(self) -> StringTensor:
883
+ r"""Return the text column block."""
884
+ return self._text
885
+
886
+ @property
887
+ def id(self) -> ColumnarTensor:
888
+ r"""Return the identifier column block."""
889
+ return self._id
890
+
891
+ def items(self) -> Iterator[tuple[Stype, Tensor]]:
892
+ r"""Yield ``(stype, block)`` pairs for typed column blocks."""
893
+ yield Stype.numerical, self._numerical
894
+ yield Stype.categorical, self._categorical
895
+ yield Stype.datetime, self._datetime
896
+ yield Stype.text, self._text
897
+ yield Stype.id, self._id
898
+
899
+ @property
900
+ def blocks(self) -> Mapping[Stype, Tensor]:
901
+ r"""Return typed column blocks per semantic type."""
902
+ return dict(self.items())
903
+
904
+ @property
905
+ def schema(self) -> TableSchema:
906
+ r"""The schema of this table."""
907
+ return TableSchema(columns=self._columns)
908
+
909
+ def is_same_schema(self, other: TableTensor) -> bool:
910
+ r"""Whether ``other`` has the same schema layout.
911
+
912
+ Args:
913
+ other: The object to compare against.
914
+ """
915
+ return self.schema == other.schema
916
+
917
+ def replace_blocks(
918
+ self,
919
+ *,
920
+ numerical: Tensor | None = None,
921
+ categorical: CategoricalTensor | None = None,
922
+ datetime: Tensor | None = None,
923
+ text: StringTensor | None = None,
924
+ id: ColumnarTensor | None = None,
925
+ ) -> Self:
926
+ r"""Return a table with one or more semantic blocks replaced.
927
+
928
+ Provided blocks replace the corresponding semantic type while omitted
929
+ blocks are reused from this table. The returned table preserves the
930
+ current column schema and is validated by the ``TableTensor``
931
+ constructor.
932
+
933
+ Args:
934
+ numerical: Replacement numerical block with shape
935
+ ``[..., C_num]``.
936
+ categorical: Replacement categorical block with shape
937
+ ``[..., C_cat]``.
938
+ datetime: Replacement datetime block with shape ``[..., C_dt]``.
939
+ text: Replacement text block with shape ``[..., C_text]``.
940
+ id: Replacement identifier block with shape ``[..., C_id]``.
941
+ """
942
+ return self.__class__(
943
+ columns=cast(Mapping[StypeLike, Sequence[str]], self.columns),
944
+ numerical=self.numerical if numerical is None else numerical,
945
+ categorical=(
946
+ self.categorical if categorical is None else categorical
947
+ ),
948
+ datetime=self.datetime if datetime is None else datetime,
949
+ text=self.text if text is None else text,
950
+ id=self.id if id is None else id,
951
+ )
952
+
953
+ def select_stypes(
954
+ self,
955
+ stypes: StypeLike | Iterable[StypeLike],
956
+ ) -> Self:
957
+ r"""Return a table containing only ``stypes`` columns.
958
+
959
+ Args:
960
+ stypes: The semantic type or semantic types to select.
961
+ """
962
+ if isinstance(stypes, (str, Stype)):
963
+ stypes = (stypes,)
964
+ stypes = tuple(Stype(stype) for stype in stypes)
965
+
966
+ return self.__class__(
967
+ columns={stype: self._columns[stype] for stype in stypes},
968
+ **{stype: getattr(self, stype) for stype in stypes},
969
+ )
970
+
971
+ def drop_stypes(
972
+ self,
973
+ stypes: StypeLike | Iterable[StypeLike],
974
+ ) -> Self:
975
+ r"""Return a table with ``stypes`` columns removed.
976
+
977
+ .. testcode:: drop_stypes
978
+
979
+ assert table.columns[Stype.categorical] == ("country", "segment")
980
+ table = table.drop_stypes("categorical")
981
+ assert table.columns[Stype.categorical] == ()
982
+ assert table.columns[Stype.numerical] == ("age", "income")
983
+
984
+ Args:
985
+ stypes: The semantic type or semantic types to drop.
986
+ """
987
+ if isinstance(stypes, (str, Stype)):
988
+ stypes = (stypes,)
989
+ stypes = {Stype(stype) for stype in stypes}
990
+
991
+ keep = tuple(stype for stype in self._columns if stype not in stypes)
992
+ return self.__class__(
993
+ size=self.size()[:-1],
994
+ columns={stype: self._columns[stype] for stype in keep},
995
+ device=self.device,
996
+ **{stype: getattr(self, stype) for stype in keep},
997
+ )
998
+
999
+ def select_columns(self, columns: str | Iterable[str]) -> Self:
1000
+ r"""Return a table containing only ``columns``.
1001
+
1002
+ .. testcode:: select_columns
1003
+
1004
+ assert table.size() == (10, 4)
1005
+ table = table.select_columns(["age", "country"])
1006
+ assert table.size() == (10, 2)
1007
+
1008
+ Args:
1009
+ columns: The columns to select.
1010
+ """
1011
+ columns = {columns} if isinstance(columns, str) else set(columns)
1012
+
1013
+ for column in columns:
1014
+ if column not in self._column_to_loc:
1015
+ raise KeyError(column)
1016
+
1017
+ index_dict: dict[Stype, list[int]] = defaultdict(list)
1018
+ columns_dict: dict[StypeLike, list[str]] = defaultdict(list)
1019
+ for stype, stype_columns in self._columns.items():
1020
+ for i, column in enumerate(stype_columns):
1021
+ if column in columns:
1022
+ index_dict[stype].append(i)
1023
+ columns_dict[stype].append(column)
1024
+
1025
+ blocks: dict[Stype, Tensor] = {}
1026
+ for stype, tensor in self.items():
1027
+ indices = index_dict[stype]
1028
+ if len(indices) == 0:
1029
+ blocks[stype] = tensor.narrow(-1, 0, 0)
1030
+ elif len(indices) == len(self._columns[stype]):
1031
+ blocks[stype] = tensor
1032
+ elif len(indices) == 1:
1033
+ blocks[stype] = tensor.narrow(-1, indices[0], 1)
1034
+ else:
1035
+ blocks[stype] = torch.cat(
1036
+ [tensor.narrow(-1, index, 1) for index in indices],
1037
+ dim=-1,
1038
+ )
1039
+
1040
+ return self.__class__(columns=columns_dict, **blocks)
1041
+
1042
+ def drop_columns(self, columns: str | Iterable[str]) -> Self:
1043
+ r"""Return a table with ``columns`` removed.
1044
+
1045
+ .. testcode:: drop_columns
1046
+
1047
+ assert table.size() == (10, 4)
1048
+ table = table.drop_columns(["age", "country"])
1049
+ assert table.size() == (10, 2)
1050
+
1051
+ Args:
1052
+ columns: The columns to drop.
1053
+ """
1054
+ columns = {columns} if isinstance(columns, str) else set(columns)
1055
+
1056
+ for column in columns:
1057
+ if column not in self._column_to_loc:
1058
+ raise KeyError(column)
1059
+
1060
+ columns = set(chain.from_iterable(self._columns.values())) - columns
1061
+ return self.select_columns(columns)
1062
+
1063
+ # Decorators ##############################################################
1064
+
1065
+ @classmethod
1066
+ def implements(
1067
+ cls,
1068
+ torch_function: Callable[..., Any],
1069
+ ) -> Callable[..., Any]:
1070
+ r"""Register a ``__torch_dispatch__`` implementation.
1071
+
1072
+ See PyTorch's
1073
+ :ref:`calling convention <torch-dispatch-calling-convention>`.
1074
+ """
1075
+ if "HANDLED_FUNCTIONS" not in cls.__dict__:
1076
+ cls.HANDLED_FUNCTIONS = cls.HANDLED_FUNCTIONS.copy()
1077
+
1078
+ def decorator(my_function: Callable[..., Any]) -> Callable[..., Any]:
1079
+ cls.HANDLED_FUNCTIONS[torch_function] = my_function
1080
+ return my_function
1081
+
1082
+ return decorator
1083
+
1084
+ # PyTorch/Python builtins #################################################
1085
+
1086
+ def __reduce_ex__(self, proto: SupportsIndex) -> Any:
1087
+ args = (
1088
+ tuple(self.size()[:-1]),
1089
+ self._columns,
1090
+ self._numerical,
1091
+ self._categorical,
1092
+ self._datetime,
1093
+ self._text,
1094
+ self._id,
1095
+ )
1096
+ return (self.__class__, args)
1097
+
1098
+ @classmethod
1099
+ def __torch_dispatch__( # type: ignore
1100
+ cls,
1101
+ func: Callable[..., Any],
1102
+ types: tuple[type[Any], ...],
1103
+ args: tuple[Any, ...] = (),
1104
+ kwargs: dict[str, Any] | None = None,
1105
+ ) -> Any:
1106
+ if (handler := cls.HANDLED_FUNCTIONS.get(func)) is not None:
1107
+ return handler(*args, **(kwargs or {}))
1108
+
1109
+ raise NotImplementedError(
1110
+ f"'{func}' is not supported for {cls.__name__!r}"
1111
+ )
1112
+
1113
+ def __getitem__(self, indices: Any) -> TableTensor:
1114
+ def is_column_index(index: Any) -> bool:
1115
+ return isinstance(index, str) or (
1116
+ isinstance(index, list)
1117
+ and all(isinstance(value, str) for value in index)
1118
+ )
1119
+
1120
+ if is_column_index(indices):
1121
+ return self.select_columns(indices)
1122
+ if not isinstance(indices, tuple):
1123
+ return cast(TableTensor, Tensor.__getitem__(self, indices))
1124
+ if not any(is_column_index(index) for index in indices):
1125
+ return cast(TableTensor, Tensor.__getitem__(self, indices))
1126
+ if any(is_column_index(index) for index in indices[:-1]):
1127
+ raise IndexError(
1128
+ "Column names can only index the column dimension"
1129
+ )
1130
+
1131
+ out = Tensor.__getitem__(self, (*indices[:-1], slice(None)))
1132
+ return cast(TableTensor, out).select_columns(indices[-1])
1133
+
1134
+ @override
1135
+ def is_shared(self) -> bool:
1136
+ return all(tensor.is_shared() for _, tensor in self.items())
1137
+
1138
+ @override
1139
+ def share_memory_(self) -> Self:
1140
+ for _, tensor in self.items():
1141
+ tensor.share_memory_()
1142
+ return self
1143
+
1144
+ @override
1145
+ def is_contiguous(
1146
+ self,
1147
+ memory_format: torch.memory_format = torch.contiguous_format,
1148
+ ) -> bool:
1149
+ return all(
1150
+ tensor.is_contiguous(memory_format=memory_format)
1151
+ for _, tensor in self.items()
1152
+ )
1153
+
1154
+ @override
1155
+ def contiguous(
1156
+ self,
1157
+ memory_format: torch.memory_format = torch.contiguous_format,
1158
+ ) -> Self:
1159
+ if self.is_contiguous(memory_format=memory_format):
1160
+ return self
1161
+ return _contiguous(self, memory_format=memory_format)
1162
+
1163
+ @override
1164
+ def tolist() -> Any:
1165
+ raise NotImplementedError("'tolist() is not yet implemented") # TODO
1166
+
1167
+ def __repr__(self, *, indent: int = 0) -> str: # type: ignore
1168
+ def _columns_repr(
1169
+ columns: Sequence[str],
1170
+ max_cols: int = 3,
1171
+ max_item_len: int = 24,
1172
+ ) -> str:
1173
+ columns = [
1174
+ column
1175
+ if len(column) <= max_item_len
1176
+ else column[: max_item_len - 1] + "…"
1177
+ for column in columns
1178
+ ]
1179
+ if len(columns) > max_cols:
1180
+ columns = [*columns[: max_cols - 1], "...", columns[-1]]
1181
+ return "[" + ", ".join(column for column in columns) + "]"
1182
+
1183
+ stype_repr = [
1184
+ (
1185
+ f"{' ' * (indent + 4)}{stype} ({tensor.size(-1):,}): "
1186
+ f"{_columns_repr(self._columns[stype])},"
1187
+ )
1188
+ for stype, tensor in self.items()
1189
+ if tensor.size(-1) > 0
1190
+ ]
1191
+
1192
+ out = f"{' ' * indent}{self.__class__.__name__}(\n"
1193
+ out += f"{' ' * (indent + 2)}size={tuple(self.size())},\n"
1194
+ if len(stype_repr) > 0:
1195
+ out += f"{' ' * (indent + 2)}blocks={{\n"
1196
+ out += "\n".join(stype_repr) + "\n"
1197
+ out += f"{' ' * (indent + 2)}}},\n"
1198
+ if not self.is_cpu:
1199
+ out += f"{' ' * (indent + 2)}device={self.device},\n"
1200
+ out += f"{' ' * indent})"
1201
+ return out
1202
+
1203
+ def _repr_html_(self) -> str:
1204
+ import pandas as pd
1205
+
1206
+ max_columns = 10
1207
+ rows = [
1208
+ [column, str(stype)]
1209
+ for stype, columns in self._columns.items()
1210
+ for column in columns
1211
+ ]
1212
+ if len(rows) > max_columns + 1:
1213
+ rows = [
1214
+ *rows[: max_columns // 2],
1215
+ ["...", "..."],
1216
+ *rows[-max_columns // 2 :],
1217
+ ]
1218
+ df = pd.DataFrame(
1219
+ data=rows,
1220
+ columns=pd.Index(["Column", "Stype"]),
1221
+ )
1222
+
1223
+ size = f"{self.size(-2)} rows x {self.size(-1)} columns"
1224
+ if self.dim() > 2:
1225
+ examples = " x ".join(str(dim) for dim in self.size()[:-2])
1226
+ size = f"{examples} examples x {size}"
1227
+ if not self.is_cpu:
1228
+ size += f", device={self.device}"
1229
+
1230
+ return df.to_html(index=False, escape=True) + f"<p>{size}</p>"
1231
+
1232
+
1233
+ @TableTensor.implements(aten.alias.default)
1234
+ @preserve_view_inference_mode
1235
+ def _alias(inp: TableTensor) -> TableTensor:
1236
+ blocks = {
1237
+ stype: aten.alias.default(tensor) for stype, tensor in inp.items()
1238
+ }
1239
+ return inp.__class__(
1240
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1241
+ **blocks,
1242
+ )
1243
+
1244
+
1245
+ @TableTensor.implements(aten.record_stream.default)
1246
+ def _record_stream(inp: TableTensor, stream: torch.Stream) -> None:
1247
+ for _, tensor in inp.items():
1248
+ tensor.record_stream(stream)
1249
+
1250
+
1251
+ @TableTensor.implements(aten.to.dtype_layout)
1252
+ @preserve_autograd_state
1253
+ def _to_dtype_layout(
1254
+ inp: TableTensor,
1255
+ *,
1256
+ dtype: torch.dtype | None = None,
1257
+ layout: torch.layout | None = None,
1258
+ device: torch.device | str | None = None,
1259
+ pin_memory: bool | None = None, # Ignored by PyTorch.
1260
+ non_blocking: bool = False,
1261
+ copy: bool = False,
1262
+ memory_format: torch.memory_format | None = None,
1263
+ ) -> TableTensor:
1264
+
1265
+ if (
1266
+ not copy
1267
+ and (dtype is None or dtype == inp.dtype)
1268
+ and (device is None or torch.device(device) == inp.device)
1269
+ and (layout is None or layout == inp.layout)
1270
+ and (
1271
+ memory_format is None
1272
+ or memory_format == torch.preserve_format
1273
+ or (
1274
+ memory_format == torch.contiguous_format
1275
+ and inp.is_contiguous()
1276
+ )
1277
+ )
1278
+ ):
1279
+ return inp
1280
+
1281
+ blocks = {
1282
+ stype: aten.to.dtype_layout(
1283
+ tensor,
1284
+ dtype=dtype
1285
+ if (
1286
+ stype not in (Stype.categorical,)
1287
+ or dtype in (torch.int32, torch.int64)
1288
+ )
1289
+ and stype not in (Stype.datetime, Stype.text, Stype.id)
1290
+ else None,
1291
+ layout=layout,
1292
+ device=device,
1293
+ pin_memory=pin_memory,
1294
+ non_blocking=non_blocking,
1295
+ copy=copy,
1296
+ memory_format=memory_format,
1297
+ )
1298
+ for stype, tensor in inp.items()
1299
+ }
1300
+ return inp.__class__(
1301
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1302
+ **blocks,
1303
+ )
1304
+
1305
+
1306
+ @TableTensor.implements(aten.to.dtype)
1307
+ def _to_dtype(
1308
+ inp: TableTensor,
1309
+ dtype: torch.dtype,
1310
+ non_blocking: bool = False,
1311
+ copy: bool = False,
1312
+ memory_format: torch.memory_format | None = None,
1313
+ ) -> TableTensor:
1314
+ return _to_dtype_layout(
1315
+ inp,
1316
+ dtype=dtype,
1317
+ non_blocking=non_blocking,
1318
+ copy=copy,
1319
+ memory_format=memory_format,
1320
+ )
1321
+
1322
+
1323
+ @TableTensor.implements(aten.to.device)
1324
+ def _to_device(
1325
+ inp: TableTensor,
1326
+ device: torch.device,
1327
+ dtype: torch.dtype,
1328
+ non_blocking: bool = False,
1329
+ copy: bool = False,
1330
+ memory_format: torch.memory_format | None = None,
1331
+ ) -> TableTensor:
1332
+ return _to_dtype_layout(
1333
+ inp,
1334
+ dtype=dtype,
1335
+ device=device,
1336
+ non_blocking=non_blocking,
1337
+ copy=copy,
1338
+ memory_format=memory_format,
1339
+ )
1340
+
1341
+
1342
+ @TableTensor.implements(aten.to.other)
1343
+ def _to_other(
1344
+ inp: TableTensor,
1345
+ other: Tensor,
1346
+ non_blocking: bool = False,
1347
+ copy: bool = False,
1348
+ memory_format: torch.memory_format | None = None,
1349
+ ) -> TableTensor:
1350
+ return _to_dtype_layout(
1351
+ inp,
1352
+ dtype=other.dtype,
1353
+ layout=other.layout,
1354
+ device=other.device,
1355
+ non_blocking=non_blocking,
1356
+ copy=copy,
1357
+ memory_format=memory_format,
1358
+ )
1359
+
1360
+
1361
+ @TableTensor.implements(aten._to_copy.default)
1362
+ def _to_copy(
1363
+ inp: TableTensor,
1364
+ *,
1365
+ dtype: torch.dtype | None = None,
1366
+ layout: torch.layout | None = None,
1367
+ device: torch.device | str | None = None,
1368
+ pin_memory: bool = False, # Ignored by PyTorch.
1369
+ non_blocking: bool = False,
1370
+ memory_format: torch.memory_format | None = None,
1371
+ ) -> TableTensor:
1372
+ return _to_dtype_layout(
1373
+ inp,
1374
+ dtype=dtype,
1375
+ layout=layout,
1376
+ device=device,
1377
+ pin_memory=pin_memory,
1378
+ non_blocking=non_blocking,
1379
+ copy=True,
1380
+ memory_format=memory_format,
1381
+ )
1382
+
1383
+
1384
+ @TableTensor.implements(aten.clone.default)
1385
+ def _clone(
1386
+ inp: TableTensor,
1387
+ *,
1388
+ memory_format: torch.memory_format | None = None,
1389
+ ) -> TableTensor:
1390
+ return _to_dtype_layout(inp, copy=True, memory_format=memory_format)
1391
+
1392
+
1393
+ @TableTensor.implements(aten.contiguous.default)
1394
+ def _contiguous(
1395
+ inp: TableTensor,
1396
+ *,
1397
+ memory_format: torch.memory_format = torch.contiguous_format,
1398
+ ) -> TableTensor:
1399
+ blocks = {
1400
+ stype: tensor.contiguous(memory_format=memory_format)
1401
+ for stype, tensor in inp.items()
1402
+ }
1403
+ return inp.__class__(
1404
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1405
+ **blocks,
1406
+ )
1407
+
1408
+
1409
+ @TableTensor.implements(aten.is_pinned.default)
1410
+ def _is_pinned(inp: TableTensor) -> bool:
1411
+ return all(
1412
+ tensor.is_pinned() for _, tensor in inp.items() if tensor.numel() > 0
1413
+ )
1414
+
1415
+
1416
+ @TableTensor.implements(aten._pin_memory.default)
1417
+ def _pin_memory(inp: TableTensor) -> TableTensor:
1418
+ blocks = {
1419
+ stype: tensor.pin_memory() if tensor.numel() > 0 else tensor
1420
+ for stype, tensor in inp.items()
1421
+ }
1422
+ return inp.__class__(
1423
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1424
+ **blocks,
1425
+ )
1426
+
1427
+
1428
+ @TableTensor.implements(aten.pin_memory.default)
1429
+ def _pin_memory_composite(
1430
+ inp: TableTensor,
1431
+ device: torch.device | None = None,
1432
+ ) -> TableTensor:
1433
+ if _is_pinned(inp):
1434
+ return inp
1435
+ return _pin_memory(inp)
1436
+
1437
+
1438
+ @TableTensor.implements(aten.equal.default)
1439
+ def _equal(inp: TableTensor, other: Tensor) -> bool:
1440
+ if inp.__class__ is not other.__class__:
1441
+ return False
1442
+ if inp.size() != other.size():
1443
+ return False
1444
+ if inp.stypes != other.stypes:
1445
+ return False
1446
+
1447
+ for stype, block in _align_like(inp, other).items():
1448
+ if not block.equal(other.blocks[stype]):
1449
+ return False
1450
+
1451
+ return True
1452
+
1453
+
1454
+ @TableTensor.implements(aten.allclose.default)
1455
+ def _allclose(
1456
+ inp: TableTensor,
1457
+ other: Tensor,
1458
+ rtol: float = 1e-05,
1459
+ atol: float = 1e-08,
1460
+ equal_nan: bool = False,
1461
+ ) -> bool:
1462
+ if inp.__class__ is not other.__class__:
1463
+ return False
1464
+ if inp.size() != other.size():
1465
+ return False
1466
+ if inp.stypes != other.stypes:
1467
+ return False
1468
+
1469
+ for stype, block in _align_like(inp, other).items():
1470
+ if stype != Stype.numerical: # Tolerance only applies for numerical:
1471
+ if not block.equal(other.blocks[stype]):
1472
+ return False
1473
+ elif not block.allclose(other.blocks[stype], rtol, atol, equal_nan):
1474
+ return False
1475
+
1476
+ return True
1477
+
1478
+
1479
+ @TableTensor.implements(aten.view.default)
1480
+ @preserve_view_inference_mode
1481
+ def _view(inp: TableTensor, size: Sequence[int]) -> TableTensor:
1482
+ size = tuple(size)
1483
+ for i, dim_size in enumerate(size):
1484
+ if dim_size < -1:
1485
+ raise RuntimeError(
1486
+ f"Invalid shape dimension {dim_size} at index {i} of shape "
1487
+ f"{size}"
1488
+ )
1489
+
1490
+ if size.count(-1) > 1:
1491
+ raise RuntimeError("Only one dimension can be inferred")
1492
+
1493
+ if -1 in size:
1494
+ known = math.prod(dim_size for dim_size in size if dim_size != -1)
1495
+ if known == 0:
1496
+ raise RuntimeError(
1497
+ f"Cannot reshape tensor of {inp.numel()} elements into "
1498
+ f"shape {size} because the unspecified dimension size -1 can "
1499
+ f"be any value and is ambiguous"
1500
+ )
1501
+ if inp.numel() % known != 0:
1502
+ raise RuntimeError(
1503
+ f"Shape {size} is invalid for input of size {inp.numel()}"
1504
+ )
1505
+ dim = size.index(-1)
1506
+ size = (*size[:dim], inp.numel() // known, *size[dim + 1 :])
1507
+
1508
+ if len(size) == 0 or size[-1] != inp.size(-1):
1509
+ _columns = "column" if inp.size(-1) == 1 else "columns"
1510
+ raise RuntimeError(
1511
+ f"Can't reshape {inp.__class__.__name__!r} with "
1512
+ f"{inp.size(-1)} {_columns} into shape {size}"
1513
+ )
1514
+
1515
+ blocks = {
1516
+ stype: tensor.view((*size[:-1], tensor.size(-1)))
1517
+ for stype, tensor in inp.items()
1518
+ }
1519
+ return inp.__class__(
1520
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1521
+ **blocks,
1522
+ )
1523
+
1524
+
1525
+ @TableTensor.implements(aten._unsafe_view.default)
1526
+ @preserve_view_inference_mode
1527
+ def _unsafe_view(inp: TableTensor, size: Sequence[int]) -> TableTensor:
1528
+ return _view(inp, size)
1529
+
1530
+
1531
+ @TableTensor.implements(aten.reshape.default)
1532
+ def _reshape(inp: TableTensor, size: Sequence[int]) -> TableTensor:
1533
+ return cast(
1534
+ TableTensor,
1535
+ aten.reshape.default.decompose(inp, size),
1536
+ )
1537
+
1538
+
1539
+ @TableTensor.implements(aten.flatten.using_ints)
1540
+ def _flatten(
1541
+ inp: TableTensor,
1542
+ start_dim: int = 0,
1543
+ end_dim: int = -1,
1544
+ ) -> TableTensor:
1545
+ return cast(
1546
+ TableTensor,
1547
+ aten.flatten.using_ints.decompose(inp, start_dim, end_dim),
1548
+ )
1549
+
1550
+
1551
+ @TableTensor.implements(aten.squeeze.default)
1552
+ @preserve_view_inference_mode
1553
+ def _squeeze(inp: TableTensor) -> TableTensor:
1554
+ return _squeeze_dims(inp, range(inp.dim() - 1))
1555
+
1556
+
1557
+ @TableTensor.implements(aten.squeeze.dim)
1558
+ @preserve_view_inference_mode
1559
+ def _squeeze_dim(inp: TableTensor, dim: int) -> TableTensor:
1560
+ return _squeeze_dims(inp, (dim,))
1561
+
1562
+
1563
+ @TableTensor.implements(aten.squeeze.dims)
1564
+ @preserve_view_inference_mode
1565
+ def _squeeze_dims(inp: TableTensor, dim: Sequence[int]) -> TableTensor:
1566
+ dims = tuple(dim)
1567
+ blocks = {stype: tensor.squeeze(dims) for stype, tensor in inp.items()}
1568
+
1569
+ if inp.dim() - 1 in tuple(dim % inp.dim() for dim in dims):
1570
+ raise RuntimeError(
1571
+ f"Can't squeeze the column dimension of {inp.__class__.__name__!r}"
1572
+ )
1573
+
1574
+ return inp.__class__(
1575
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1576
+ **blocks,
1577
+ )
1578
+
1579
+
1580
+ @TableTensor.implements(aten.unsqueeze.default)
1581
+ @preserve_view_inference_mode
1582
+ def _unsqueeze(inp: TableTensor, dim: int) -> TableTensor:
1583
+ blocks = {stype: tensor.unsqueeze(dim) for stype, tensor in inp.items()}
1584
+
1585
+ if dim % (inp.dim() + 1) == inp.dim():
1586
+ raise RuntimeError(
1587
+ f"Can't unsqueeze after the column dimension of "
1588
+ f"{inp.__class__.__name__!r}"
1589
+ )
1590
+
1591
+ return inp.__class__(
1592
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1593
+ **blocks,
1594
+ )
1595
+
1596
+
1597
+ @TableTensor.implements(aten.expand.default)
1598
+ @preserve_view_inference_mode
1599
+ def _expand(
1600
+ inp: TableTensor,
1601
+ size: Sequence[int],
1602
+ *,
1603
+ implicit: bool = False,
1604
+ ) -> TableTensor:
1605
+ size = tuple(size)
1606
+ if len(size) == 0 or size[-1] not in (-1, inp.size(-1)):
1607
+ _columns = "column" if inp.size(-1) == 1 else "columns"
1608
+ raise RuntimeError(
1609
+ f"Can't expand {inp.__class__.__name__!r} with "
1610
+ f"{inp.size(-1)} {_columns} to shape {size}"
1611
+ )
1612
+
1613
+ blocks = {
1614
+ stype: aten.expand.default(
1615
+ tensor,
1616
+ (*size[:-1], tensor.size(-1)),
1617
+ implicit=implicit,
1618
+ )
1619
+ for stype, tensor in inp.items()
1620
+ }
1621
+ return inp.__class__(
1622
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1623
+ **blocks,
1624
+ )
1625
+
1626
+
1627
+ @TableTensor.implements(aten.transpose.int)
1628
+ @preserve_view_inference_mode
1629
+ def _transpose(inp: TableTensor, dim0: int, dim1: int) -> TableTensor:
1630
+ blocks = {
1631
+ stype: tensor.transpose(dim0, dim1) for stype, tensor in inp.items()
1632
+ }
1633
+
1634
+ dim0 %= inp.dim()
1635
+ dim1 %= inp.dim()
1636
+ if dim0 != dim1 and inp.dim() - 1 in (dim0, dim1):
1637
+ raise RuntimeError(
1638
+ f"Can't transpose the column dimension of "
1639
+ f"{inp.__class__.__name__!r}"
1640
+ )
1641
+
1642
+ return inp.__class__(
1643
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1644
+ **blocks,
1645
+ )
1646
+
1647
+
1648
+ @TableTensor.implements(aten.permute.default)
1649
+ @preserve_view_inference_mode
1650
+ def _permute(inp: TableTensor, dims: Sequence[int]) -> TableTensor:
1651
+ dims = tuple(dims)
1652
+ blocks = {stype: tensor.permute(dims) for stype, tensor in inp.items()}
1653
+
1654
+ dims = tuple(dim % inp.dim() for dim in dims)
1655
+ if dims[-1] != inp.dim() - 1:
1656
+ raise RuntimeError(
1657
+ f"Can't permute the column dimension of {inp.__class__.__name__!r}"
1658
+ )
1659
+
1660
+ return inp.__class__(
1661
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1662
+ **blocks,
1663
+ )
1664
+
1665
+
1666
+ @TableTensor.implements(aten.movedim.int)
1667
+ def _movedim_int(
1668
+ inp: TableTensor,
1669
+ source: int,
1670
+ destination: int,
1671
+ ) -> TableTensor:
1672
+ return cast(
1673
+ TableTensor,
1674
+ aten.movedim.int.decompose(inp, source, destination),
1675
+ )
1676
+
1677
+
1678
+ @TableTensor.implements(aten.movedim.intlist)
1679
+ def _movedim_intlist(
1680
+ inp: TableTensor,
1681
+ source: Sequence[int],
1682
+ destination: Sequence[int],
1683
+ ) -> TableTensor:
1684
+ return cast(
1685
+ TableTensor,
1686
+ aten.movedim.intlist.decompose(inp, source, destination),
1687
+ )
1688
+
1689
+
1690
+ @TableTensor.implements(aten.select.int)
1691
+ @preserve_view_inference_mode
1692
+ def _select(inp: TableTensor, dim: int, index: int) -> TableTensor:
1693
+ if _is_column_dim(inp, dim):
1694
+ raise RuntimeError(
1695
+ f"Can't select the column dimension of {inp.__class__.__name__!r}"
1696
+ )
1697
+
1698
+ blocks = {
1699
+ stype: tensor.select(dim, index) for stype, tensor in inp.items()
1700
+ }
1701
+
1702
+ return inp.__class__(
1703
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1704
+ **blocks,
1705
+ )
1706
+
1707
+
1708
+ @TableTensor.implements(aten.slice.Tensor)
1709
+ @preserve_view_inference_mode
1710
+ def _slice(
1711
+ inp: TableTensor,
1712
+ dim: int = 0,
1713
+ start: int | None = None,
1714
+ end: int | None = None,
1715
+ step: int = 1,
1716
+ ) -> TableTensor:
1717
+ if _is_column_dim(inp, dim):
1718
+ if (
1719
+ (start is None or start == 0 or start <= -inp.size(-1))
1720
+ and (end is None or end >= inp.size(dim))
1721
+ and step == 1
1722
+ ):
1723
+ return _alias(inp)
1724
+ raise RuntimeError(
1725
+ f"Can't slice the column dimension of '{inp.__class__.__name__}'"
1726
+ )
1727
+
1728
+ blocks = {
1729
+ stype: aten.slice.Tensor(tensor, dim, start, end, step)
1730
+ for stype, tensor in inp.items()
1731
+ }
1732
+
1733
+ return inp.__class__(
1734
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1735
+ **blocks,
1736
+ )
1737
+
1738
+
1739
+ @TableTensor.implements(aten.narrow.default)
1740
+ @preserve_view_inference_mode
1741
+ def _narrow(
1742
+ inp: TableTensor,
1743
+ dim: int,
1744
+ start: int,
1745
+ length: int,
1746
+ ) -> TableTensor:
1747
+ if _is_column_dim(inp, dim):
1748
+ if (start == 0 or start == -inp.size(-1)) and length == inp.size(-1):
1749
+ return _alias(inp)
1750
+ raise RuntimeError(
1751
+ f"Can't narrow the column dimension of '{inp.__class__.__name__}'"
1752
+ )
1753
+
1754
+ blocks = {
1755
+ stype: tensor.narrow(dim, start, length)
1756
+ for stype, tensor in inp.items()
1757
+ }
1758
+
1759
+ return inp.__class__(
1760
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1761
+ **blocks,
1762
+ )
1763
+
1764
+
1765
+ @TableTensor.implements(aten.unbind.int)
1766
+ @preserve_view_inference_mode
1767
+ def _unbind(inp: TableTensor, dim: int = 0) -> tuple[TableTensor, ...]:
1768
+ if _is_column_dim(inp, dim):
1769
+ return _split(inp, split_size=1, dim=dim)
1770
+
1771
+ tensors_dict: dict[Stype, tuple[Tensor, ...]] = {
1772
+ stype: tensor.unbind(dim) for stype, tensor in inp.items()
1773
+ }
1774
+
1775
+ stypes = tuple(tensors_dict.keys())
1776
+ return tuple(
1777
+ inp.__class__(
1778
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1779
+ **dict(zip(stypes, blocks)),
1780
+ )
1781
+ for blocks in zip(*tensors_dict.values())
1782
+ )
1783
+
1784
+
1785
+ @TableTensor.implements(aten.split.Tensor)
1786
+ @preserve_view_inference_mode
1787
+ def _split(
1788
+ inp: TableTensor,
1789
+ split_size: int,
1790
+ dim: int = 0,
1791
+ ) -> tuple[TableTensor, ...]:
1792
+ if _is_column_dim(inp, dim):
1793
+ if split_size != 1:
1794
+ raise RuntimeError(
1795
+ f"Can only split the column dimension of "
1796
+ f"{inp.__class__.__name__!r} with split size 1"
1797
+ )
1798
+ return tuple(
1799
+ inp.__class__(
1800
+ columns={stype: (name,)},
1801
+ **{stype: tensor.narrow(-1, i, 1)},
1802
+ )
1803
+ for stype, tensor in inp.items()
1804
+ for i, name in enumerate(inp._columns[stype])
1805
+ )
1806
+
1807
+ tensors_dict: dict[Stype, tuple[Tensor, ...]] = {
1808
+ stype: tensor.split(split_size, dim) for stype, tensor in inp.items()
1809
+ }
1810
+
1811
+ stypes = tuple(tensors_dict.keys())
1812
+ return tuple(
1813
+ inp.__class__(
1814
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1815
+ **dict(zip(stypes, blocks)),
1816
+ )
1817
+ for blocks in zip(*tensors_dict.values())
1818
+ )
1819
+
1820
+
1821
+ @TableTensor.implements(aten.split.sizes)
1822
+ @TableTensor.implements(aten.split.default)
1823
+ @TableTensor.implements(aten.split_with_sizes.default)
1824
+ @preserve_view_inference_mode
1825
+ def _split_with_sizes(
1826
+ inp: TableTensor,
1827
+ split_sizes: Sequence[int],
1828
+ dim: int = 0,
1829
+ ) -> tuple[TableTensor, ...]:
1830
+ if _is_column_dim(inp, dim):
1831
+ raise RuntimeError(
1832
+ f"Can't split the column dimension of {inp.__class__.__name__!r}"
1833
+ )
1834
+
1835
+ split_sizes = tuple(split_sizes)
1836
+ blocks_dict: dict[Stype, tuple[Tensor, ...]] = {
1837
+ stype: tensor.split(split_sizes, dim) for stype, tensor in inp.items()
1838
+ }
1839
+
1840
+ stypes = tuple(blocks_dict.keys())
1841
+ return tuple(
1842
+ inp.__class__(
1843
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1844
+ **dict(zip(stypes, blocks)),
1845
+ )
1846
+ for blocks in zip(*blocks_dict.values())
1847
+ )
1848
+
1849
+
1850
+ @TableTensor.implements(aten.index_select.default)
1851
+ def _index_select(
1852
+ inp: TableTensor,
1853
+ dim: int,
1854
+ index: Tensor,
1855
+ ) -> TableTensor:
1856
+ if _is_column_dim(inp, dim):
1857
+ raise RuntimeError(
1858
+ f"Can't index the column dimension of {inp.__class__.__name__!r}"
1859
+ )
1860
+
1861
+ blocks = {
1862
+ stype: tensor.index_select(dim, index) for stype, tensor in inp.items()
1863
+ }
1864
+
1865
+ return inp.__class__(
1866
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1867
+ **blocks,
1868
+ )
1869
+
1870
+
1871
+ @TableTensor.implements(aten.index.Tensor)
1872
+ def _index(
1873
+ inp: TableTensor,
1874
+ indices: Sequence[Tensor | None],
1875
+ ) -> TableTensor:
1876
+
1877
+ current_dim = 0
1878
+ for index in indices:
1879
+ if index is None:
1880
+ current_dim += 1
1881
+ continue
1882
+
1883
+ # Check whether we index the column dimension:
1884
+ num_indexed_dims = index.dim() if index.dtype == torch.bool else 1
1885
+ if current_dim <= inp.dim() - 1 < current_dim + num_indexed_dims:
1886
+ raise RuntimeError(
1887
+ f"Can't index the column dimension of "
1888
+ f"{inp.__class__.__name__!r}"
1889
+ )
1890
+ current_dim += num_indexed_dims
1891
+
1892
+ blocks = {
1893
+ stype: aten.index.Tensor(tensor, indices)
1894
+ for stype, tensor in inp.items()
1895
+ }
1896
+
1897
+ return inp.__class__(
1898
+ columns=cast(dict[StypeLike, tuple[str, ...]], inp._columns),
1899
+ **blocks,
1900
+ )
1901
+
1902
+
1903
+ @TableTensor.implements(aten.cat.default)
1904
+ @preserve_autograd_state
1905
+ def _cat(tensors: Sequence[Tensor], dim: int = 0) -> TableTensor:
1906
+ if len(tensors) == 0:
1907
+ raise ValueError("torch.cat(): expected a non-empty list of Tensors")
1908
+
1909
+ if not all(isinstance(tensor, TableTensor) for tensor in tensors):
1910
+ raise TypeError(
1911
+ f"Expected all tensors to be {TableTensor.__name__!r} instances"
1912
+ )
1913
+ tensors = cast(Sequence[TableTensor], tensors)
1914
+
1915
+ ref = tensors[0]
1916
+ if not _is_column_dim(ref, dim):
1917
+ tensors = (ref, *(_align_like(tensor, ref) for tensor in tensors[1:]))
1918
+
1919
+ blocks: dict[Stype, Tensor] = {}
1920
+ for stype, _ in ref.items():
1921
+ block_list = [tensor.blocks[stype] for tensor in tensors]
1922
+ block_list = [block for block in block_list if block.size(-1) > 0]
1923
+ if len(block_list) == 1:
1924
+ blocks[stype] = block_list[0]
1925
+ elif len(block_list) > 1:
1926
+ blocks[stype] = torch.cat(block_list, dim=dim)
1927
+
1928
+ size: Sequence[int] | None = None
1929
+ if not _is_column_dim(ref, dim):
1930
+ columns = ref._columns
1931
+ if len(blocks) == 0:
1932
+ size = list(tensors[0].size())
1933
+ for tensor in tensors[1:]:
1934
+ size[dim] += tensor.size(dim)
1935
+ else:
1936
+ columns = {
1937
+ stype: tuple(
1938
+ chain.from_iterable(t._columns[stype] for t in tensors)
1939
+ )
1940
+ for stype, _ in ref.items()
1941
+ }
1942
+ size = ref.size()
1943
+
1944
+ return ref.__class__(
1945
+ size=size[:-1] if size is not None else None,
1946
+ columns=cast(dict[StypeLike, tuple[str, ...]], columns),
1947
+ device=ref.device if len(blocks) == 0 else None,
1948
+ **blocks,
1949
+ )
1950
+
1951
+
1952
+ @TableTensor.implements(aten.stack.default)
1953
+ @preserve_autograd_state
1954
+ def _stack(tensors: Sequence[Tensor], dim: int = 0) -> TableTensor:
1955
+ if len(tensors) == 0:
1956
+ raise RuntimeError("stack expects a non-empty TensorList")
1957
+
1958
+ if not all(isinstance(tensor, TableTensor) for tensor in tensors):
1959
+ raise TypeError(
1960
+ f"Expected all tensors to be {TableTensor.__name__!r} instances"
1961
+ )
1962
+
1963
+ dim %= tensors[0].dim() + 1
1964
+ if dim >= tensors[0].dim():
1965
+ raise RuntimeError(
1966
+ f"Can't stack after the column dimension of "
1967
+ f"{tensors[0].__class__.__name__!r}"
1968
+ )
1969
+
1970
+ tensors = cast(Sequence[TableTensor], tensors)
1971
+
1972
+ ref = tensors[0]
1973
+ tensors = (ref, *(_align_like(tensor, ref) for tensor in tensors[1:]))
1974
+
1975
+ blocks = {
1976
+ stype: torch.stack([tensor.blocks[stype] for tensor in tensors], dim)
1977
+ for stype in ref._columns
1978
+ }
1979
+
1980
+ return ref.__class__(
1981
+ columns=cast(dict[StypeLike, tuple[str, ...]], ref._columns),
1982
+ **blocks,
1983
+ )
1984
+
1985
+
1986
+ # Helpers #####################################################################
1987
+
1988
+
1989
+ def _block_size_repr(size: Sequence[int]) -> str:
1990
+ if len(size) == 0:
1991
+ return "(*,)"
1992
+ if len(size) == 1:
1993
+ return f"({size[0]}, *)"
1994
+ return f"{str(tuple(size))[:-1]}, *)"
1995
+
1996
+
1997
+ def _is_column_dim(inp: Tensor, dim: int) -> bool:
1998
+ if dim < -inp.dim() or dim >= inp.dim():
1999
+ return False
2000
+ return dim % inp.dim() == inp.dim() - 1
2001
+
2002
+
2003
+ def _align_like(inp: TableTensor, ref: TableTensor) -> TableTensor:
2004
+ if inp._columns == ref._columns:
2005
+ return inp
2006
+
2007
+ if inp.stypes != ref.stypes:
2008
+ raise ValueError(
2009
+ "Expected tensors to have the same column names and stypes"
2010
+ )
2011
+
2012
+ blocks: dict[Stype, Tensor] = {}
2013
+ for stype, ref_columns in ref._columns.items():
2014
+ if len(ref_columns) > 0:
2015
+ column_to_index = {
2016
+ column: i for i, column in enumerate(inp._columns[stype])
2017
+ }
2018
+ indices = [column_to_index[column] for column in ref_columns]
2019
+ block = inp.blocks[stype]
2020
+ if indices == list(range(len(inp._columns[stype]))):
2021
+ blocks[stype] = block
2022
+ elif len(indices) == 1:
2023
+ blocks[stype] = block.narrow(-1, indices[0], 1)
2024
+ else:
2025
+ blocks[stype] = torch.cat(
2026
+ [block.narrow(-1, index, 1) for index in indices],
2027
+ dim=-1,
2028
+ )
2029
+ else:
2030
+ blocks[stype] = inp.blocks[stype]
2031
+
2032
+ return inp.__class__(
2033
+ columns=cast(Mapping[StypeLike, Sequence[str]], ref._columns),
2034
+ **blocks,
2035
+ )