structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from collections.abc import Mapping, Sequence
|
|
5
|
+
from typing import Literal
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
from torch import Tensor
|
|
9
|
+
|
|
10
|
+
from sdm import TableTensor
|
|
11
|
+
from sdm.relational import RelationalData, TaskLink
|
|
12
|
+
from sdm.relational.join import join_index
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class PyGLibRelationalSampler:
|
|
16
|
+
def __init__(
|
|
17
|
+
self,
|
|
18
|
+
data: RelationalData,
|
|
19
|
+
time_columns: Mapping[str, str],
|
|
20
|
+
) -> None:
|
|
21
|
+
|
|
22
|
+
if not data.is_cpu:
|
|
23
|
+
raise ValueError(
|
|
24
|
+
f"{self.__class__.__name__!r} requires input data on CPU "
|
|
25
|
+
f"(got '{data.device}')"
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
import pyg_lib # noqa
|
|
30
|
+
except ImportError as e:
|
|
31
|
+
torch_version = torch.__version__.split("+", maxsplit=1)[0]
|
|
32
|
+
if not all(part.isdigit() for part in torch_version.split(".")):
|
|
33
|
+
raise ImportError(
|
|
34
|
+
"No module named 'pyg_lib'. Pre-built pyg-lib wheels are "
|
|
35
|
+
"only published for stable PyTorch releases. Please "
|
|
36
|
+
"install a stable PyTorch release or build pyg-lib "
|
|
37
|
+
"from source (see 'https://github.com/pyg-team/pyg-lib' "
|
|
38
|
+
"for more information)"
|
|
39
|
+
) from e
|
|
40
|
+
if torch.version.cuda is None:
|
|
41
|
+
cuda_version = "cpu"
|
|
42
|
+
else:
|
|
43
|
+
cuda_version = f"cu{torch.version.cuda.replace('.', '')}"
|
|
44
|
+
raise ImportError(
|
|
45
|
+
f"No module named 'pyg_lib'. Please install it via "
|
|
46
|
+
f"'pip install pyg-lib -f https://data.pyg.org/whl/"
|
|
47
|
+
f"torch-{torch_version}+{cuda_version}.html' (see "
|
|
48
|
+
"'https://github.com/pyg-team/pyg-lib' for more information)"
|
|
49
|
+
) from e
|
|
50
|
+
|
|
51
|
+
self.data = data
|
|
52
|
+
self.time_columns = time_columns
|
|
53
|
+
|
|
54
|
+
self._row_dict: dict[tuple[str, str, str], Tensor] = {}
|
|
55
|
+
self._colptr_dict: dict[tuple[str, str, str], Tensor] = {}
|
|
56
|
+
self._time_dict: dict[str, Tensor] = {}
|
|
57
|
+
for table_name, time_column in self.time_columns.items():
|
|
58
|
+
time = self.data.tables[table_name][time_column].datetime
|
|
59
|
+
self._time_dict[table_name] = time.squeeze(-1).contiguous()
|
|
60
|
+
|
|
61
|
+
for i, (rel, edge_index) in enumerate(
|
|
62
|
+
zip(self.data.relationships, self.data.edge_indices())
|
|
63
|
+
):
|
|
64
|
+
edge_type = (rel.left_table, str(2 * i), rel.right_table)
|
|
65
|
+
self._row_dict[edge_type], self._colptr_dict[edge_type] = _to_csc(
|
|
66
|
+
edge_index=edge_index,
|
|
67
|
+
num_dst_nodes=self.data.tables[rel.right_table].size(0),
|
|
68
|
+
src_time=self._time_dict.get(rel.left_table),
|
|
69
|
+
)
|
|
70
|
+
edge_type = (rel.right_table, str(2 * i + 1), rel.left_table)
|
|
71
|
+
self._row_dict[edge_type], self._colptr_dict[edge_type] = _to_csc(
|
|
72
|
+
edge_index=edge_index.flip(0),
|
|
73
|
+
num_dst_nodes=self.data.tables[rel.left_table].size(0),
|
|
74
|
+
src_time=self._time_dict.get(rel.right_table),
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
def sample(
|
|
78
|
+
self,
|
|
79
|
+
task_table: TableTensor,
|
|
80
|
+
task_link: TaskLink,
|
|
81
|
+
num_neighbors: Sequence[int],
|
|
82
|
+
task_time_column: str | None = None,
|
|
83
|
+
temporal_strategy: Literal["last", "uniform"] = "last",
|
|
84
|
+
) -> dict[str, tuple[Tensor, Tensor]]:
|
|
85
|
+
|
|
86
|
+
if not task_table.is_cpu:
|
|
87
|
+
raise NotImplementedError(
|
|
88
|
+
f"{self.__class__.__name__!r} requires input data on CPU "
|
|
89
|
+
f"(got '{task_table.device}')"
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
# Resolve entity table node indices:
|
|
93
|
+
task_index, seed = join_index(
|
|
94
|
+
left_table=task_table,
|
|
95
|
+
right_table=self.data.tables[task_link.table],
|
|
96
|
+
left_keys=task_link.task_columns,
|
|
97
|
+
right_keys=task_link.table_columns,
|
|
98
|
+
device=task_table.device,
|
|
99
|
+
)
|
|
100
|
+
task_index, perm = task_index.sort()
|
|
101
|
+
seed = seed[perm]
|
|
102
|
+
|
|
103
|
+
expected = torch.arange(
|
|
104
|
+
task_table.size(-2),
|
|
105
|
+
dtype=task_index.dtype,
|
|
106
|
+
device=task_index.device,
|
|
107
|
+
)
|
|
108
|
+
if not task_index.equal(expected):
|
|
109
|
+
raise ValueError(
|
|
110
|
+
f"Expected each task row to match exactly one row in "
|
|
111
|
+
f"{task_link.table!r}"
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
if task_time_column is not None:
|
|
115
|
+
seed_time = task_table[task_time_column].datetime.squeeze(-1)
|
|
116
|
+
else:
|
|
117
|
+
fill_value = torch.iinfo(torch.int64).max
|
|
118
|
+
seed_time = torch.full_like(seed, fill_value)
|
|
119
|
+
|
|
120
|
+
# Perform subgraph sampling:
|
|
121
|
+
_, _, node_dict, *_ = torch.ops.pyg.hetero_neighbor_sample(
|
|
122
|
+
node_types=list(self.data.tables),
|
|
123
|
+
edge_types=list(self._colptr_dict),
|
|
124
|
+
rowptr_dict={
|
|
125
|
+
"__".join(edge_type): colptr
|
|
126
|
+
for edge_type, colptr in self._colptr_dict.items()
|
|
127
|
+
},
|
|
128
|
+
col_dict={
|
|
129
|
+
"__".join(edge_type): row
|
|
130
|
+
for edge_type, row in self._row_dict.items()
|
|
131
|
+
},
|
|
132
|
+
seed_dict={task_link.table: seed},
|
|
133
|
+
num_neighbors_dict={
|
|
134
|
+
"__".join(edge_type): list(num_neighbors)
|
|
135
|
+
for edge_type in self._row_dict
|
|
136
|
+
},
|
|
137
|
+
node_time_dict=self._time_dict,
|
|
138
|
+
edge_time_dict=None,
|
|
139
|
+
seed_time_dict={task_link.table: seed_time},
|
|
140
|
+
edge_weight_dict=None,
|
|
141
|
+
csc=True,
|
|
142
|
+
replace=False,
|
|
143
|
+
directed=True,
|
|
144
|
+
disjoint=True,
|
|
145
|
+
temporal_strategy=temporal_strategy,
|
|
146
|
+
return_edge_id=False,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
return {
|
|
150
|
+
table_name: tuple(node.t().contiguous())
|
|
151
|
+
for table_name, node in node_dict.items()
|
|
152
|
+
if node.numel() > 0
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _to_csc(
|
|
157
|
+
edge_index: Tensor,
|
|
158
|
+
num_dst_nodes: int,
|
|
159
|
+
src_time: Tensor | None = None,
|
|
160
|
+
) -> tuple[Tensor, Tensor]:
|
|
161
|
+
|
|
162
|
+
if src_time is None: # Sort primarily by destination node:
|
|
163
|
+
perm = edge_index[1].argsort()
|
|
164
|
+
else: # Sort secondarily by source timestamp:
|
|
165
|
+
perm = src_time[edge_index[0]].argsort()
|
|
166
|
+
edge_index = edge_index[:, perm]
|
|
167
|
+
perm = edge_index[1].argsort(stable=True)
|
|
168
|
+
edge_index = edge_index[:, perm]
|
|
169
|
+
|
|
170
|
+
row, col = edge_index
|
|
171
|
+
colptr = torch._convert_indices_from_coo_to_csr(
|
|
172
|
+
col, num_dst_nodes, out_int32=col.dtype != torch.int64
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
return row, colptr
|
sdm/relational/data.py
ADDED
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from collections.abc import Callable, Collection, Iterator, Mapping, Sequence
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from html import escape
|
|
9
|
+
from typing import TYPE_CHECKING, Any, Self, cast
|
|
10
|
+
|
|
11
|
+
import torch
|
|
12
|
+
from torch import Tensor
|
|
13
|
+
|
|
14
|
+
from sdm import Stype, TableTensor
|
|
15
|
+
from sdm.relational.join import LEFT_ROW_ID, RIGHT_ROW_ID, join_index
|
|
16
|
+
from sdm.tensor.mixin import DeviceMixin
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
import graphviz
|
|
20
|
+
|
|
21
|
+
from sdm.relational import RelationalSampler
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True, repr=False)
|
|
25
|
+
class Relationship:
|
|
26
|
+
r"""Join relationship between two tables.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
left_table: Name of the left table.
|
|
30
|
+
left_columns: Column names from the left table.
|
|
31
|
+
right_table: Name of the right table.
|
|
32
|
+
right_columns: Column names from the right table.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
left_table: str
|
|
36
|
+
left_columns: tuple[str, ...]
|
|
37
|
+
right_table: str
|
|
38
|
+
right_columns: tuple[str, ...]
|
|
39
|
+
|
|
40
|
+
def __post_init__(self) -> None:
|
|
41
|
+
if len(self.left_columns) != len(self.right_columns):
|
|
42
|
+
raise ValueError(
|
|
43
|
+
f"Expected 'left_columns' and 'right_columns' to have the "
|
|
44
|
+
f"same length (got {len(self.left_columns)} and "
|
|
45
|
+
f"{len(self.right_columns)})"
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
if len(self.left_columns) == 0:
|
|
49
|
+
raise ValueError(
|
|
50
|
+
"Expected 'left_columns' and 'right_columns' to be non-empty"
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
for column in (*self.left_columns, *self.right_columns):
|
|
54
|
+
for reserved in (LEFT_ROW_ID, RIGHT_ROW_ID):
|
|
55
|
+
if column == reserved:
|
|
56
|
+
raise ValueError(
|
|
57
|
+
f"Column name {column!r} is reserved for internal "
|
|
58
|
+
f"row indexing"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
@classmethod
|
|
62
|
+
def from_mapping(cls, mapping: Mapping[str, str | Sequence[str]]) -> Self:
|
|
63
|
+
r"""Create a :class:`Relationship` from a mapping.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
mapping: The mapping with ``"left_table"``, ``"left_columns"``,
|
|
67
|
+
``"right_table"``, ``"right_columns"`` entries.
|
|
68
|
+
"""
|
|
69
|
+
left_table = mapping["left_table"]
|
|
70
|
+
assert isinstance(left_table, str)
|
|
71
|
+
|
|
72
|
+
if "left_column" in mapping:
|
|
73
|
+
left_columns = mapping["left_column"]
|
|
74
|
+
else:
|
|
75
|
+
left_columns = mapping["left_columns"]
|
|
76
|
+
if isinstance(left_columns, str):
|
|
77
|
+
left_columns = (left_columns,)
|
|
78
|
+
|
|
79
|
+
right_table = mapping["right_table"]
|
|
80
|
+
assert isinstance(right_table, str)
|
|
81
|
+
|
|
82
|
+
if "right_column" in mapping:
|
|
83
|
+
right_columns = mapping["right_column"]
|
|
84
|
+
else:
|
|
85
|
+
right_columns = mapping["right_columns"]
|
|
86
|
+
if isinstance(right_columns, str):
|
|
87
|
+
right_columns = (right_columns,)
|
|
88
|
+
|
|
89
|
+
return cls(
|
|
90
|
+
left_table=left_table,
|
|
91
|
+
left_columns=tuple(left_columns),
|
|
92
|
+
right_table=right_table,
|
|
93
|
+
right_columns=tuple(right_columns),
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
def _left_columns_repr(self) -> str:
|
|
97
|
+
if len(self.left_columns) == 1:
|
|
98
|
+
return f"{self.left_table}.{self.left_columns[0]}"
|
|
99
|
+
return f"{self.left_table}.[{','.join(self.left_columns)}]"
|
|
100
|
+
|
|
101
|
+
def _right_columns_repr(self) -> str:
|
|
102
|
+
if len(self.right_columns) == 1:
|
|
103
|
+
return f"{self.right_table}.{self.right_columns[0]}"
|
|
104
|
+
return f"{self.right_table}.[{','.join(self.right_columns)}]"
|
|
105
|
+
|
|
106
|
+
def __repr__(self) -> str:
|
|
107
|
+
return f"{self._left_columns_repr()} <> {self._right_columns_repr()}"
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@dataclass(frozen=True, init=False, repr=False)
|
|
111
|
+
class RelationalData(DeviceMixin):
|
|
112
|
+
r"""Collection of named tables and join relationships.
|
|
113
|
+
|
|
114
|
+
.. testcode::
|
|
115
|
+
|
|
116
|
+
from sdm import RelationalData, TableTensor
|
|
117
|
+
|
|
118
|
+
data = RelationalData(
|
|
119
|
+
tables={
|
|
120
|
+
"users": TableTensor.from_columns(
|
|
121
|
+
{"user_id": [0, 1]},
|
|
122
|
+
stypes={"user_id": "id"},
|
|
123
|
+
),
|
|
124
|
+
"orders": TableTensor.from_columns(
|
|
125
|
+
{
|
|
126
|
+
"user_id": [0, 1],
|
|
127
|
+
"item_id": [10, 11],
|
|
128
|
+
},
|
|
129
|
+
stypes={
|
|
130
|
+
"user_id": "id",
|
|
131
|
+
"item_id": "id",
|
|
132
|
+
},
|
|
133
|
+
),
|
|
134
|
+
"items": TableTensor.from_columns(
|
|
135
|
+
{"item_id": [10, 11]},
|
|
136
|
+
stypes={"item_id": "id"},
|
|
137
|
+
),
|
|
138
|
+
},
|
|
139
|
+
relationships=[
|
|
140
|
+
# Foreign key from orders to users:
|
|
141
|
+
dict(left_table="orders", left_column="user_id", right_table="users", right_column="user_id"),
|
|
142
|
+
# Foreign key from orders to items:
|
|
143
|
+
dict(left_table="orders", left_column="item_id", right_table="items", right_column="item_id"),
|
|
144
|
+
],
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
Args:
|
|
148
|
+
tables: Tables keyed by table name.
|
|
149
|
+
relationships: Join relationships among ``tables``.
|
|
150
|
+
""" # noqa: E501
|
|
151
|
+
|
|
152
|
+
tables: Mapping[str, TableTensor]
|
|
153
|
+
relationships: tuple[Relationship, ...]
|
|
154
|
+
|
|
155
|
+
def __init__(
|
|
156
|
+
self,
|
|
157
|
+
tables: Mapping[str, TableTensor],
|
|
158
|
+
relationships: Collection[
|
|
159
|
+
Relationship | Mapping[str, str | Sequence[str]]
|
|
160
|
+
],
|
|
161
|
+
) -> None:
|
|
162
|
+
|
|
163
|
+
relationships = tuple(
|
|
164
|
+
relationship
|
|
165
|
+
if isinstance(relationship, Relationship)
|
|
166
|
+
else Relationship.from_mapping(relationship)
|
|
167
|
+
for relationship in relationships
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
object.__setattr__(self, "tables", tables)
|
|
171
|
+
object.__setattr__(self, "relationships", relationships)
|
|
172
|
+
self.__post_init__()
|
|
173
|
+
|
|
174
|
+
def __post_init__(self) -> None:
|
|
175
|
+
for table in self.tables.values():
|
|
176
|
+
if table.dim() != 2:
|
|
177
|
+
raise ValueError("Tables need to be two-dimensional")
|
|
178
|
+
|
|
179
|
+
for relationship in self.relationships:
|
|
180
|
+
for table, columns in (
|
|
181
|
+
(relationship.left_table, relationship.left_columns),
|
|
182
|
+
(relationship.right_table, relationship.right_columns),
|
|
183
|
+
):
|
|
184
|
+
if table not in self.tables:
|
|
185
|
+
raise ValueError(
|
|
186
|
+
f"Expected {table!r} to be registered as a table"
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
for column in columns:
|
|
190
|
+
stype = self.tables[table].stype(column)
|
|
191
|
+
if stype != Stype.id:
|
|
192
|
+
raise ValueError(
|
|
193
|
+
f"Expected column {column!r} in table {table!r} "
|
|
194
|
+
f"to have semantic type {str(Stype.id)!r} "
|
|
195
|
+
f"(got {str(stype)!r})"
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
def _tensors(self) -> Iterator[Tensor]:
|
|
199
|
+
yield from self.tables.values()
|
|
200
|
+
|
|
201
|
+
def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self:
|
|
202
|
+
return self.__class__(
|
|
203
|
+
tables={
|
|
204
|
+
name: cast(TableTensor, fn(table))
|
|
205
|
+
for name, table in self.tables.items()
|
|
206
|
+
},
|
|
207
|
+
relationships=self.relationships,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
def edge_indices(
|
|
211
|
+
self,
|
|
212
|
+
dtype: torch.dtype | None = None,
|
|
213
|
+
device: torch.device | str | None = None,
|
|
214
|
+
) -> tuple[Tensor, ...]:
|
|
215
|
+
r"""Materialize heterogeneous graph edges for table relationships.
|
|
216
|
+
|
|
217
|
+
Args:
|
|
218
|
+
dtype: The dtype.
|
|
219
|
+
device: The device.
|
|
220
|
+
|
|
221
|
+
Returns:
|
|
222
|
+
The edge indices for each relationship in order.
|
|
223
|
+
Each edge index has shape ``[2, num_edges]`` and stores left table
|
|
224
|
+
indices in the first row and right table indices in the second row.
|
|
225
|
+
"""
|
|
226
|
+
edge_indices: list[Tensor] = []
|
|
227
|
+
for rel in self.relationships:
|
|
228
|
+
src, dst = join_index(
|
|
229
|
+
left_table=self.tables[rel.left_table],
|
|
230
|
+
right_table=self.tables[rel.right_table],
|
|
231
|
+
left_keys=rel.left_columns,
|
|
232
|
+
right_keys=rel.right_columns,
|
|
233
|
+
how="inner",
|
|
234
|
+
dtype=dtype,
|
|
235
|
+
device=device,
|
|
236
|
+
)
|
|
237
|
+
edge_indices.append(torch.stack([src, dst], dim=0))
|
|
238
|
+
|
|
239
|
+
return tuple(edge_indices)
|
|
240
|
+
|
|
241
|
+
def sampler(
|
|
242
|
+
self,
|
|
243
|
+
time_columns: Mapping[str, str] | None = None,
|
|
244
|
+
) -> RelationalSampler:
|
|
245
|
+
r"""Create a device-appropriate sampler over this relational data.
|
|
246
|
+
|
|
247
|
+
.. code-block:: python
|
|
248
|
+
|
|
249
|
+
import sdm
|
|
250
|
+
|
|
251
|
+
data = sdm.RelationalData(
|
|
252
|
+
tables={
|
|
253
|
+
"users": sdm.TableTensor.from_columns(
|
|
254
|
+
{"user_id": [0, 1]},
|
|
255
|
+
stypes={"user_id": "id"},
|
|
256
|
+
),
|
|
257
|
+
"orders": sdm.TableTensor.from_columns(
|
|
258
|
+
{
|
|
259
|
+
"user_id": [0, 1],
|
|
260
|
+
"item_id": [10, 11],
|
|
261
|
+
"order_date": ["2026-01-01", "2026-01-02"],
|
|
262
|
+
},
|
|
263
|
+
stypes={
|
|
264
|
+
"user_id": "id",
|
|
265
|
+
"item_id": "id",
|
|
266
|
+
"order_date": "datetime",
|
|
267
|
+
},
|
|
268
|
+
),
|
|
269
|
+
"items": sdm.TableTensor.from_columns(
|
|
270
|
+
{"item_id": [10, 11]},
|
|
271
|
+
stypes={"item_id": "id"},
|
|
272
|
+
),
|
|
273
|
+
},
|
|
274
|
+
relationships=[
|
|
275
|
+
# Foreign key from orders to users:
|
|
276
|
+
dict(left_table="orders", left_column="user_id", right_table="users", right_column="user_id"),
|
|
277
|
+
# Foreign key from orders to items:
|
|
278
|
+
dict(left_table="orders", left_column="item_id", right_table="items", right_column="item_id"),
|
|
279
|
+
],
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
sampler = data.sampler(time_columns={"orders": "order_date"})
|
|
283
|
+
|
|
284
|
+
Args:
|
|
285
|
+
time_columns: Mapping from table name to the datetime column used
|
|
286
|
+
for temporal sampling. A row in a time-aware table can only be
|
|
287
|
+
sampled if its timestamp does not exceed the query timestamp.
|
|
288
|
+
""" # noqa: E501
|
|
289
|
+
from sdm.relational import RelationalSampler # noqa: PLC0415
|
|
290
|
+
|
|
291
|
+
return RelationalSampler(data=self, time_columns=time_columns)
|
|
292
|
+
|
|
293
|
+
def to_graphviz(
|
|
294
|
+
self,
|
|
295
|
+
*,
|
|
296
|
+
hide_columns: bool = False,
|
|
297
|
+
**kwargs: Any,
|
|
298
|
+
) -> graphviz.Graph:
|
|
299
|
+
r"""Return a graph visualization of the relational schema.
|
|
300
|
+
|
|
301
|
+
Args:
|
|
302
|
+
hide_columns: Whether to hide column name descriptions.
|
|
303
|
+
**kwargs: Additional keyword arguments passed to
|
|
304
|
+
:class:`graphviz.Graph`.
|
|
305
|
+
"""
|
|
306
|
+
import graphviz
|
|
307
|
+
|
|
308
|
+
def left_align(keys: list[str]) -> str:
|
|
309
|
+
if len(keys) == 0:
|
|
310
|
+
return ""
|
|
311
|
+
return "\\l".join(keys) + "\\l"
|
|
312
|
+
|
|
313
|
+
graph = graphviz.Graph(**kwargs)
|
|
314
|
+
|
|
315
|
+
for table_name, table in self.tables.items():
|
|
316
|
+
if hide_columns:
|
|
317
|
+
label = f"{{{table_name}}}"
|
|
318
|
+
else:
|
|
319
|
+
columns = [
|
|
320
|
+
f"{column}: {stype}"
|
|
321
|
+
for stype, columns in table._columns.items()
|
|
322
|
+
for column in columns
|
|
323
|
+
]
|
|
324
|
+
label = f"{{{table_name}|{left_align(columns)}}}"
|
|
325
|
+
graph.node(table_name, shape="record", label=label)
|
|
326
|
+
|
|
327
|
+
for rel in self.relationships:
|
|
328
|
+
label = "\\n".join(
|
|
329
|
+
f" {left_column} <> {right_column} "
|
|
330
|
+
for left_column, right_column in zip(
|
|
331
|
+
rel.left_columns, rel.right_columns
|
|
332
|
+
)
|
|
333
|
+
)
|
|
334
|
+
graph.edge(
|
|
335
|
+
rel.left_table,
|
|
336
|
+
rel.right_table,
|
|
337
|
+
label=label,
|
|
338
|
+
fontsize="11pt",
|
|
339
|
+
)
|
|
340
|
+
|
|
341
|
+
return graph
|
|
342
|
+
|
|
343
|
+
def __repr__(self) -> str:
|
|
344
|
+
out = f"{self.__class__.__name__}(\n"
|
|
345
|
+
if len(self.tables) > 0:
|
|
346
|
+
out += " tables={\n"
|
|
347
|
+
out += "".join(
|
|
348
|
+
f" {name}: {table.__repr__(indent=4)[4:]},\n"
|
|
349
|
+
for name, table in self.tables.items()
|
|
350
|
+
)
|
|
351
|
+
out += " },\n"
|
|
352
|
+
else:
|
|
353
|
+
out += " tables={},\n"
|
|
354
|
+
if len(self.relationships) > 0:
|
|
355
|
+
out += " relationships=[\n"
|
|
356
|
+
out += "".join(f" {rel},\n" for rel in self.relationships)
|
|
357
|
+
out += " ],\n"
|
|
358
|
+
else:
|
|
359
|
+
out += " relationships=[],\n"
|
|
360
|
+
out += ")"
|
|
361
|
+
return out
|
|
362
|
+
|
|
363
|
+
def _repr_html_(self) -> str:
|
|
364
|
+
import pandas as pd
|
|
365
|
+
|
|
366
|
+
rows = [
|
|
367
|
+
[
|
|
368
|
+
name,
|
|
369
|
+
table.size(-2),
|
|
370
|
+
table.size(-1),
|
|
371
|
+
", ".join(
|
|
372
|
+
str(stype)
|
|
373
|
+
for stype, tensor in table.items()
|
|
374
|
+
if tensor.size(-1) > 0
|
|
375
|
+
),
|
|
376
|
+
]
|
|
377
|
+
for name, table in self.tables.items()
|
|
378
|
+
]
|
|
379
|
+
df = pd.DataFrame(
|
|
380
|
+
rows,
|
|
381
|
+
columns=pd.Index(["Table", "Rows", "Columns", "Stypes"]),
|
|
382
|
+
)
|
|
383
|
+
|
|
384
|
+
ul = "".join(
|
|
385
|
+
f"<li>"
|
|
386
|
+
f"<code>{escape(rel._left_columns_repr())}</code>"
|
|
387
|
+
f" ↔️ "
|
|
388
|
+
f"<code>{escape(rel._right_columns_repr())}</code>"
|
|
389
|
+
f"</li>"
|
|
390
|
+
for rel in self.relationships
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
return df.to_html(index=False, escape=True) + f"<ul>{ul}</ul>"
|