structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/relational/task.py
ADDED
|
@@ -0,0 +1,465 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from collections.abc import (
|
|
7
|
+
Callable,
|
|
8
|
+
Collection,
|
|
9
|
+
Iterable,
|
|
10
|
+
Iterator,
|
|
11
|
+
Mapping,
|
|
12
|
+
Sequence,
|
|
13
|
+
)
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from html import escape
|
|
16
|
+
from typing import TYPE_CHECKING, Any, Generic, Self, TypeVar, cast
|
|
17
|
+
|
|
18
|
+
from torch import Tensor
|
|
19
|
+
|
|
20
|
+
from sdm import EnsembleTable, TableTensor
|
|
21
|
+
from sdm.relational import RelationalData, Relationship
|
|
22
|
+
from sdm.relational.join import LEFT_ROW_ID, RIGHT_ROW_ID
|
|
23
|
+
from sdm.tensor.mixin import DeviceMixin
|
|
24
|
+
from sdm.tensor.table import TableSchema
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
import graphviz
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
T = TypeVar("T", bound=TableTensor | EnsembleTable)
|
|
31
|
+
|
|
32
|
+
TASK_TABLE = "__task_table__"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True, repr=False)
|
|
36
|
+
class TaskLink:
|
|
37
|
+
r"""Link between task rows to a table in relational data.
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
task_columns: Column names in the task table.
|
|
41
|
+
table: Name of the table referenced by task rows.
|
|
42
|
+
table_columns: Column names in ``table``.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
task_columns: tuple[str, ...]
|
|
46
|
+
table: str
|
|
47
|
+
table_columns: tuple[str, ...]
|
|
48
|
+
|
|
49
|
+
def __post_init__(self) -> None:
|
|
50
|
+
if len(self.task_columns) != len(self.table_columns):
|
|
51
|
+
raise ValueError(
|
|
52
|
+
f"Expected 'task_columns' and 'table_columns' to have the "
|
|
53
|
+
f"same length (got {len(self.task_columns)} and "
|
|
54
|
+
f"{len(self.table_columns)})"
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
if len(self.task_columns) == 0:
|
|
58
|
+
raise ValueError(
|
|
59
|
+
"Expected 'task_columns' and 'table_columns' to be non-empty"
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
for column in (*self.task_columns, *self.table_columns):
|
|
63
|
+
for reserved in (LEFT_ROW_ID, RIGHT_ROW_ID):
|
|
64
|
+
if column == reserved:
|
|
65
|
+
raise ValueError(
|
|
66
|
+
f"Column name {column!r} is reserved for internal "
|
|
67
|
+
f"row indexing"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
@classmethod
|
|
71
|
+
def from_mapping(cls, mapping: Mapping[str, str | Sequence[str]]) -> Self:
|
|
72
|
+
r"""Create a :class:`TaskLink` from a mapping.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
mapping: The mapping with ``"task_columns"``, ``"table"``, and
|
|
76
|
+
``"table_columns"`` entries.
|
|
77
|
+
"""
|
|
78
|
+
if "task_column" in mapping:
|
|
79
|
+
task_columns = mapping["task_column"]
|
|
80
|
+
else:
|
|
81
|
+
task_columns = mapping["task_columns"]
|
|
82
|
+
if isinstance(task_columns, str):
|
|
83
|
+
task_columns = (task_columns,)
|
|
84
|
+
|
|
85
|
+
table = mapping["table"]
|
|
86
|
+
assert isinstance(table, str)
|
|
87
|
+
|
|
88
|
+
if "table_column" in mapping:
|
|
89
|
+
table_columns = mapping["table_column"]
|
|
90
|
+
else:
|
|
91
|
+
table_columns = mapping["table_columns"]
|
|
92
|
+
if isinstance(table_columns, str):
|
|
93
|
+
table_columns = (table_columns,)
|
|
94
|
+
|
|
95
|
+
return cls(
|
|
96
|
+
task_columns=tuple(task_columns),
|
|
97
|
+
table=table,
|
|
98
|
+
table_columns=tuple(table_columns),
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
def _task_columns_repr(self) -> str:
|
|
102
|
+
if len(self.task_columns) == 1:
|
|
103
|
+
return self.task_columns[0]
|
|
104
|
+
return f"[{','.join(self.task_columns)}]"
|
|
105
|
+
|
|
106
|
+
def _table_columns_repr(self) -> str:
|
|
107
|
+
if len(self.table_columns) == 1:
|
|
108
|
+
return f"{self.table}.{self.table_columns[0]}"
|
|
109
|
+
return f"{self.table}.[{','.join(self.table_columns)}]"
|
|
110
|
+
|
|
111
|
+
def __repr__(self) -> str:
|
|
112
|
+
return f"{self._task_columns_repr()} > {self._table_columns_repr()}"
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@dataclass(frozen=True)
|
|
116
|
+
class RelatedTablesSchema:
|
|
117
|
+
r"""The schema of :class:`RelatedTables`.
|
|
118
|
+
|
|
119
|
+
Args:
|
|
120
|
+
tables: Table schema keyed by table name.
|
|
121
|
+
relationships: Join relationships among ``tables``.
|
|
122
|
+
task_links: Links from task columns to related ``tables``.
|
|
123
|
+
"""
|
|
124
|
+
|
|
125
|
+
tables: Mapping[str, TableSchema]
|
|
126
|
+
relationships: tuple[Relationship, ...]
|
|
127
|
+
task_links: tuple[TaskLink, ...]
|
|
128
|
+
|
|
129
|
+
def is_subset_of(self, other: RelatedTablesSchema) -> bool:
|
|
130
|
+
r"""Whether this schema is an induced subset of ``other``."""
|
|
131
|
+
for table_name, schema in self.tables.items():
|
|
132
|
+
if schema != other.tables.get(table_name):
|
|
133
|
+
return False
|
|
134
|
+
|
|
135
|
+
other_relationships = {
|
|
136
|
+
relationship
|
|
137
|
+
for relationship in other.relationships
|
|
138
|
+
if relationship.left_table in self.tables
|
|
139
|
+
and relationship.right_table in self.tables
|
|
140
|
+
}
|
|
141
|
+
if set(self.relationships) != other_relationships:
|
|
142
|
+
return False
|
|
143
|
+
|
|
144
|
+
other_task_links = {
|
|
145
|
+
task_link
|
|
146
|
+
for task_link in other.task_links
|
|
147
|
+
if task_link.table in self.tables
|
|
148
|
+
}
|
|
149
|
+
return set(self.task_links) == other_task_links
|
|
150
|
+
|
|
151
|
+
def __hash__(self) -> int:
|
|
152
|
+
return hash(
|
|
153
|
+
(
|
|
154
|
+
frozenset(self.tables.items()),
|
|
155
|
+
self.relationships,
|
|
156
|
+
self.task_links,
|
|
157
|
+
)
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@dataclass(frozen=True, init=False, repr=False)
|
|
162
|
+
class RelatedTables(DeviceMixin, Generic[T]):
|
|
163
|
+
r"""Task-specific related tables attached to model inputs.
|
|
164
|
+
|
|
165
|
+
:class:`RelatedTables` store the relational context provided to a model
|
|
166
|
+
for a particular task table.
|
|
167
|
+
It may contain a sampled subset of a larger :class:`RelationalData`.
|
|
168
|
+
The ``task_links`` describe how rows in the model input match to rows in
|
|
169
|
+
the related tables.
|
|
170
|
+
|
|
171
|
+
.. testcode::
|
|
172
|
+
|
|
173
|
+
from sdm import RelatedTables, TableTensor
|
|
174
|
+
|
|
175
|
+
data = RelatedTables(
|
|
176
|
+
tables={
|
|
177
|
+
"users": TableTensor.from_columns(
|
|
178
|
+
{"user_id": [0, 1]},
|
|
179
|
+
stypes={"user_id": "id"},
|
|
180
|
+
),
|
|
181
|
+
"orders": TableTensor.from_columns(
|
|
182
|
+
{
|
|
183
|
+
"user_id": [0, 1],
|
|
184
|
+
"item_id": [10, 11],
|
|
185
|
+
},
|
|
186
|
+
stypes={
|
|
187
|
+
"user_id": "id",
|
|
188
|
+
"item_id": "id",
|
|
189
|
+
},
|
|
190
|
+
),
|
|
191
|
+
"items": TableTensor.from_columns(
|
|
192
|
+
{"item_id": [10, 11]},
|
|
193
|
+
stypes={"item_id": "id"},
|
|
194
|
+
),
|
|
195
|
+
},
|
|
196
|
+
relationships=[
|
|
197
|
+
# Foreign key from orders to users:
|
|
198
|
+
dict(left_table="orders", left_column="user_id", right_table="users", right_column="user_id"),
|
|
199
|
+
# Foreign key from orders to items:
|
|
200
|
+
dict(left_table="orders", left_column="item_id", right_table="items", right_column="item_id"),
|
|
201
|
+
],
|
|
202
|
+
task_links=[
|
|
203
|
+
# Foreign key in the task table to users:
|
|
204
|
+
dict(task_column="ENTITY", table="users", table_column="user_id")
|
|
205
|
+
],
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
Args:
|
|
209
|
+
tables: Related tables keyed by table name.
|
|
210
|
+
relationships: Join relationships among ``tables``.
|
|
211
|
+
task_links: Links from task columns to related ``tables``.
|
|
212
|
+
""" # noqa: E501
|
|
213
|
+
|
|
214
|
+
tables: Mapping[str, T]
|
|
215
|
+
relationships: tuple[Relationship, ...]
|
|
216
|
+
task_links: tuple[TaskLink, ...]
|
|
217
|
+
|
|
218
|
+
def __init__(
|
|
219
|
+
self,
|
|
220
|
+
tables: Mapping[str, T],
|
|
221
|
+
relationships: Collection[
|
|
222
|
+
Relationship | Mapping[str, str | Sequence[str]]
|
|
223
|
+
],
|
|
224
|
+
task_links: Collection[TaskLink | Mapping[str, str | Sequence[str]]],
|
|
225
|
+
) -> None:
|
|
226
|
+
|
|
227
|
+
relationships = tuple(
|
|
228
|
+
relationship
|
|
229
|
+
if isinstance(relationship, Relationship)
|
|
230
|
+
else Relationship.from_mapping(relationship)
|
|
231
|
+
for relationship in relationships
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
task_links = tuple(
|
|
235
|
+
task_link
|
|
236
|
+
if isinstance(task_link, TaskLink)
|
|
237
|
+
else TaskLink.from_mapping(task_link)
|
|
238
|
+
for task_link in task_links
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
object.__setattr__(self, "tables", tables)
|
|
242
|
+
object.__setattr__(self, "relationships", relationships)
|
|
243
|
+
object.__setattr__(self, "task_links", task_links)
|
|
244
|
+
|
|
245
|
+
def _tensors(self) -> Iterator[Tensor]:
|
|
246
|
+
for table in self.tables.values():
|
|
247
|
+
if isinstance(table, DeviceMixin):
|
|
248
|
+
yield from table._tensors()
|
|
249
|
+
continue
|
|
250
|
+
yield table
|
|
251
|
+
|
|
252
|
+
def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self:
|
|
253
|
+
return self.__class__(
|
|
254
|
+
tables={
|
|
255
|
+
name: table._apply_tensor(fn)
|
|
256
|
+
if isinstance(table, DeviceMixin)
|
|
257
|
+
else cast(TableTensor, fn(table))
|
|
258
|
+
for name, table in self.tables.items()
|
|
259
|
+
},
|
|
260
|
+
relationships=self.relationships,
|
|
261
|
+
task_links=self.task_links,
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
@property
|
|
265
|
+
def schema(self) -> RelatedTablesSchema:
|
|
266
|
+
r"""The schema of this related context."""
|
|
267
|
+
table_schemas = {}
|
|
268
|
+
for name, table in self.tables.items():
|
|
269
|
+
if isinstance(table, EnsembleTable):
|
|
270
|
+
schemas = {group.schema for group in table._iter_groups()}
|
|
271
|
+
if len(schemas) != 1:
|
|
272
|
+
raise ValueError(
|
|
273
|
+
"'schema' requires each 'EnsembleTable' to have a "
|
|
274
|
+
"unique table schema"
|
|
275
|
+
)
|
|
276
|
+
table_schemas[name] = next(iter(schemas))
|
|
277
|
+
continue
|
|
278
|
+
table_schemas[name] = table.schema
|
|
279
|
+
|
|
280
|
+
return RelatedTablesSchema(
|
|
281
|
+
tables=table_schemas,
|
|
282
|
+
relationships=self.relationships,
|
|
283
|
+
task_links=self.task_links,
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
def is_same_schema(self, other: RelatedTables) -> bool:
|
|
287
|
+
r"""Whether ``other`` has the same schema layout.
|
|
288
|
+
|
|
289
|
+
Args:
|
|
290
|
+
other: The object to compare against.
|
|
291
|
+
"""
|
|
292
|
+
return self.schema == other.schema
|
|
293
|
+
|
|
294
|
+
def select_tables(self, tables: Iterable[str]) -> Self:
|
|
295
|
+
r"""Return related tables containing only ``tables``.
|
|
296
|
+
|
|
297
|
+
Args:
|
|
298
|
+
tables: The table names to select.
|
|
299
|
+
"""
|
|
300
|
+
tables = set(tables)
|
|
301
|
+
|
|
302
|
+
return self.__class__(
|
|
303
|
+
tables={
|
|
304
|
+
table_name: table
|
|
305
|
+
for table_name, table in self.tables.items()
|
|
306
|
+
if table_name in tables
|
|
307
|
+
},
|
|
308
|
+
relationships=tuple(
|
|
309
|
+
relationship
|
|
310
|
+
for relationship in self.relationships
|
|
311
|
+
if relationship.left_table in tables
|
|
312
|
+
and relationship.right_table in tables
|
|
313
|
+
),
|
|
314
|
+
task_links=tuple(
|
|
315
|
+
task_link
|
|
316
|
+
for task_link in self.task_links
|
|
317
|
+
if task_link.table in tables
|
|
318
|
+
),
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
def replace_tables(self, tables: Mapping[str, T]) -> Self:
|
|
322
|
+
r"""Return related tables with replaced table data.
|
|
323
|
+
|
|
324
|
+
Args:
|
|
325
|
+
tables: Related tables keyed by table name.
|
|
326
|
+
"""
|
|
327
|
+
if tables.keys() != self.tables.keys():
|
|
328
|
+
raise ValueError("Expected 'tables' to match existing table names")
|
|
329
|
+
|
|
330
|
+
return self.__class__(
|
|
331
|
+
tables=tables,
|
|
332
|
+
relationships=self.relationships,
|
|
333
|
+
task_links=self.task_links,
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
def to_graphviz(
|
|
337
|
+
self,
|
|
338
|
+
*,
|
|
339
|
+
hide_columns: bool = False,
|
|
340
|
+
**kwargs: Any,
|
|
341
|
+
) -> graphviz.Graph:
|
|
342
|
+
r"""Return a task visualization of the relational schema.
|
|
343
|
+
|
|
344
|
+
Args:
|
|
345
|
+
hide_columns: Whether to hide column name descriptions.
|
|
346
|
+
**kwargs: Additional keyword arguments passed to
|
|
347
|
+
:class:`graphviz.Graph`.
|
|
348
|
+
"""
|
|
349
|
+
tables = {}
|
|
350
|
+
for name, table in self.tables.items():
|
|
351
|
+
if isinstance(table, EnsembleTable):
|
|
352
|
+
if len({group.schema for group in table._iter_groups()}) != 1:
|
|
353
|
+
raise ValueError(
|
|
354
|
+
"'to_graphviz' requires each 'EnsembleTable' to have "
|
|
355
|
+
"a unique table schema"
|
|
356
|
+
)
|
|
357
|
+
tables[name] = table[0]
|
|
358
|
+
continue
|
|
359
|
+
tables[name] = table
|
|
360
|
+
|
|
361
|
+
graph = RelationalData(
|
|
362
|
+
tables=tables,
|
|
363
|
+
relationships=self.relationships,
|
|
364
|
+
).to_graphviz(hide_columns=hide_columns, **kwargs)
|
|
365
|
+
|
|
366
|
+
graph.node(TASK_TABLE, label="", shape="point")
|
|
367
|
+
|
|
368
|
+
for link in self.task_links:
|
|
369
|
+
label = "\\n".join(
|
|
370
|
+
f" {task_column} > {table_column} "
|
|
371
|
+
for task_column, table_column in zip(
|
|
372
|
+
link.task_columns, link.table_columns
|
|
373
|
+
)
|
|
374
|
+
)
|
|
375
|
+
graph.edge(
|
|
376
|
+
TASK_TABLE,
|
|
377
|
+
link.table,
|
|
378
|
+
label=label,
|
|
379
|
+
fontsize="11pt",
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
return graph
|
|
383
|
+
|
|
384
|
+
def __repr__(self) -> str:
|
|
385
|
+
out = f"{self.__class__.__name__}(\n"
|
|
386
|
+
if len(self.tables) > 0:
|
|
387
|
+
out += " tables={\n"
|
|
388
|
+
out += "".join(
|
|
389
|
+
f" {key}: {value.__repr__(indent=4)[4:]},\n" # type: ignore
|
|
390
|
+
for key, value in self.tables.items()
|
|
391
|
+
)
|
|
392
|
+
out += " },\n"
|
|
393
|
+
else:
|
|
394
|
+
out += " tables={},\n"
|
|
395
|
+
if len(self.relationships) > 0:
|
|
396
|
+
out += " relationships=[\n"
|
|
397
|
+
out += "".join(f" {rel},\n" for rel in self.relationships)
|
|
398
|
+
out += " ],\n"
|
|
399
|
+
else:
|
|
400
|
+
out += " relationships=[],\n"
|
|
401
|
+
if len(self.task_links) > 0:
|
|
402
|
+
out += " task_links=[\n"
|
|
403
|
+
out += "".join(f" {link},\n" for link in self.task_links)
|
|
404
|
+
out += " ],\n"
|
|
405
|
+
else:
|
|
406
|
+
out += " task_links=[],\n"
|
|
407
|
+
out += ")"
|
|
408
|
+
return out
|
|
409
|
+
|
|
410
|
+
def _repr_html_(self) -> str:
|
|
411
|
+
import pandas as pd
|
|
412
|
+
|
|
413
|
+
rows = []
|
|
414
|
+
for name, table in self.tables.items():
|
|
415
|
+
row: list[Any] = [name]
|
|
416
|
+
if isinstance(table, EnsembleTable):
|
|
417
|
+
num_rows = {group.size(-2) for group in table._iter_groups()}
|
|
418
|
+
row.append(
|
|
419
|
+
next(iter(num_rows))
|
|
420
|
+
if len(num_rows) == 1
|
|
421
|
+
else f"{min(num_rows)} - {max(num_rows)}"
|
|
422
|
+
)
|
|
423
|
+
num_cols = {group.size(-1) for group in table._iter_groups()}
|
|
424
|
+
row.append(
|
|
425
|
+
next(iter(num_cols))
|
|
426
|
+
if len(num_cols) == 1
|
|
427
|
+
else f"{min(num_cols)} - {max(num_cols)}"
|
|
428
|
+
)
|
|
429
|
+
stypes = {
|
|
430
|
+
stype
|
|
431
|
+
for g in table._iter_groups()
|
|
432
|
+
for stype in g.active_stypes
|
|
433
|
+
}
|
|
434
|
+
row.append(", ".join(stypes))
|
|
435
|
+
else:
|
|
436
|
+
row += [
|
|
437
|
+
table.size(-2),
|
|
438
|
+
table.size(-1),
|
|
439
|
+
", ".join(table.active_stypes),
|
|
440
|
+
]
|
|
441
|
+
rows.append(row)
|
|
442
|
+
|
|
443
|
+
df = pd.DataFrame(
|
|
444
|
+
rows,
|
|
445
|
+
columns=pd.Index(["Table", "Rows", "Columns", "Stypes"]),
|
|
446
|
+
)
|
|
447
|
+
|
|
448
|
+
ul = "".join(
|
|
449
|
+
f"<li>"
|
|
450
|
+
f"<code>{escape(link._task_columns_repr())}</code>"
|
|
451
|
+
f" ➡️ "
|
|
452
|
+
f"<code>{escape(link._table_columns_repr())}</code>"
|
|
453
|
+
f"</li>"
|
|
454
|
+
for link in self.task_links
|
|
455
|
+
)
|
|
456
|
+
ul += "".join(
|
|
457
|
+
f"<li>"
|
|
458
|
+
f"<code>{escape(rel._left_columns_repr())}</code>"
|
|
459
|
+
f" ↔️ "
|
|
460
|
+
f"<code>{escape(rel._right_columns_repr())}</code>"
|
|
461
|
+
f"</li>"
|
|
462
|
+
for rel in self.relationships
|
|
463
|
+
)
|
|
464
|
+
|
|
465
|
+
return df.to_html(index=False, escape=True) + f"<ul>{ul}</ul>"
|