structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/relational/join.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import importlib.util
|
|
5
|
+
import math
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
from typing import Literal
|
|
8
|
+
|
|
9
|
+
import pyarrow as pa
|
|
10
|
+
import pyarrow.compute as pc
|
|
11
|
+
import torch
|
|
12
|
+
from torch import Tensor
|
|
13
|
+
|
|
14
|
+
from sdm import TableTensor
|
|
15
|
+
from sdm._warnings import warn_once
|
|
16
|
+
from sdm.tensor.io import arrow_as_tensor, to_cudf
|
|
17
|
+
|
|
18
|
+
PREFIX = "sdm_internal"
|
|
19
|
+
LEFT_ROW_ID = f"__{PREFIX}_left_row_id__"
|
|
20
|
+
RIGHT_ROW_ID = f"__{PREFIX}_right_row_id__"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def join_index(
|
|
24
|
+
left_table: TableTensor,
|
|
25
|
+
right_table: TableTensor,
|
|
26
|
+
left_keys: Sequence[str],
|
|
27
|
+
right_keys: Sequence[str],
|
|
28
|
+
how: Literal["inner"] = "inner",
|
|
29
|
+
dtype: torch.dtype | None = None,
|
|
30
|
+
device: torch.device | str | None = None,
|
|
31
|
+
) -> tuple[Tensor, Tensor]:
|
|
32
|
+
r"""Return row-index pairs for matching rows in two tables.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
left_table: The left table.
|
|
36
|
+
right_table: The right table.
|
|
37
|
+
left_keys: Column names from ``left_table`` used as join keys.
|
|
38
|
+
right_keys: Column names from ``right_table`` used as join keys.
|
|
39
|
+
how: The join type.
|
|
40
|
+
dtype: The dtype.
|
|
41
|
+
device: The device.
|
|
42
|
+
|
|
43
|
+
Returns:
|
|
44
|
+
``(left_index, right_index)`` pair with one entry per matched row.
|
|
45
|
+
"""
|
|
46
|
+
assert how == "inner"
|
|
47
|
+
|
|
48
|
+
if left_table.device != right_table.device:
|
|
49
|
+
raise RuntimeError(
|
|
50
|
+
"Expected 'left_table' and 'right_table' to be on the same device "
|
|
51
|
+
f"(got '{left_table.device}' and '{right_table.device}')"
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
dtype = dtype or torch.long
|
|
55
|
+
device = device or left_table.device
|
|
56
|
+
|
|
57
|
+
left_table = left_table[list(left_keys)]
|
|
58
|
+
right_table = right_table[list(right_keys)]
|
|
59
|
+
left_rows = math.prod(left_table.size()[:-1])
|
|
60
|
+
right_rows = math.prod(right_table.size()[:-1])
|
|
61
|
+
|
|
62
|
+
if max(left_rows, right_rows) - 1 > torch.iinfo(dtype).max:
|
|
63
|
+
raise ValueError(
|
|
64
|
+
f"Creating row indices up to {max(left_rows, right_rows) - 1:,} "
|
|
65
|
+
f"in '{dtype}' would overflow"
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
backend: Literal["arrow", "cudf"] = "arrow"
|
|
69
|
+
if left_table.is_cuda:
|
|
70
|
+
if importlib.util.find_spec("cudf") is not None:
|
|
71
|
+
backend = "cudf"
|
|
72
|
+
else:
|
|
73
|
+
warn_once(
|
|
74
|
+
key="missing-cudf-join",
|
|
75
|
+
message=(
|
|
76
|
+
"Falling back to a CPU-based join because cuDF is not "
|
|
77
|
+
"installed. Install cuDF to enable faster CUDA-based "
|
|
78
|
+
"joins without device synchronization."
|
|
79
|
+
),
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
if backend == "arrow":
|
|
83
|
+
left = left_table.to_arrow().append_column(
|
|
84
|
+
LEFT_ROW_ID,
|
|
85
|
+
pa.array(torch.arange(left_rows, dtype=dtype).numpy()),
|
|
86
|
+
)
|
|
87
|
+
right = right_table.to_arrow().append_column(
|
|
88
|
+
RIGHT_ROW_ID,
|
|
89
|
+
pa.array(torch.arange(right_rows, dtype=dtype).numpy()),
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
left = dropna(left, subset=left_keys)
|
|
93
|
+
right = dropna(right, subset=right_keys)
|
|
94
|
+
|
|
95
|
+
right_schema = right.schema
|
|
96
|
+
for left_key, right_key in zip(left_keys, right_keys):
|
|
97
|
+
left_type = left[left_key].type
|
|
98
|
+
if left_type != right[right_key].type:
|
|
99
|
+
index = right.schema.get_field_index(right_key)
|
|
100
|
+
field = right.schema.field(index).with_type(left_type)
|
|
101
|
+
right_schema = right_schema.set(index, field)
|
|
102
|
+
if right_schema != right.schema:
|
|
103
|
+
right = right.cast(right_schema, safe=True)
|
|
104
|
+
|
|
105
|
+
joined = left.join(
|
|
106
|
+
right,
|
|
107
|
+
keys=left_keys,
|
|
108
|
+
right_keys=right_keys,
|
|
109
|
+
join_type=how,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
return (
|
|
113
|
+
arrow_as_tensor(joined[LEFT_ROW_ID], device=device),
|
|
114
|
+
arrow_as_tensor(joined[RIGHT_ROW_ID], device=device),
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
assert backend == "cudf"
|
|
118
|
+
with torch.cuda.device(left_table.device):
|
|
119
|
+
left = left_table.to_cudf()
|
|
120
|
+
left[LEFT_ROW_ID] = to_cudf(
|
|
121
|
+
torch.arange(left_rows, dtype=dtype, device=left_table.device)
|
|
122
|
+
)
|
|
123
|
+
right = right_table.to_cudf()
|
|
124
|
+
right[RIGHT_ROW_ID] = to_cudf(
|
|
125
|
+
torch.arange(right_rows, dtype=dtype, device=right_table.device)
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
joined = left.dropna(subset=list(left_keys)).merge(
|
|
129
|
+
right.dropna(subset=list(right_keys)),
|
|
130
|
+
left_on=left_keys,
|
|
131
|
+
right_on=right_keys,
|
|
132
|
+
how=how,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
return (
|
|
136
|
+
torch.as_tensor(joined[LEFT_ROW_ID]).to(device),
|
|
137
|
+
torch.as_tensor(joined[RIGHT_ROW_ID]).to(device),
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def dropna(
|
|
142
|
+
table: pa.Table,
|
|
143
|
+
subset: Sequence[str] | None = None,
|
|
144
|
+
) -> pa.Table:
|
|
145
|
+
r"""Drop rows with null or NaN values in the given columns."""
|
|
146
|
+
columns = table.column_names if subset is None else subset
|
|
147
|
+
|
|
148
|
+
mask = None
|
|
149
|
+
for name in columns:
|
|
150
|
+
column = table[name]
|
|
151
|
+
|
|
152
|
+
_mask = None
|
|
153
|
+
if column.null_count > 0:
|
|
154
|
+
_mask = column.is_valid()
|
|
155
|
+
|
|
156
|
+
if pa.types.is_floating(column.type):
|
|
157
|
+
__mask = pc.call_function(
|
|
158
|
+
"invert", [pc.call_function("is_nan", [column])]
|
|
159
|
+
)
|
|
160
|
+
if _mask is not None:
|
|
161
|
+
_mask = pc.call_function("and_kleene", [_mask, __mask])
|
|
162
|
+
else:
|
|
163
|
+
_mask = __mask
|
|
164
|
+
|
|
165
|
+
if _mask is None:
|
|
166
|
+
continue
|
|
167
|
+
|
|
168
|
+
if mask is not None:
|
|
169
|
+
mask = pc.call_function("and_kleene", [mask, _mask])
|
|
170
|
+
else:
|
|
171
|
+
mask = _mask
|
|
172
|
+
|
|
173
|
+
return table if mask is None else table.filter(mask)
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from collections.abc import Callable, Iterator, Mapping, Sequence
|
|
5
|
+
from typing import Generic, Literal, NamedTuple, Self, TypeVar, cast
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
from torch import Tensor
|
|
9
|
+
|
|
10
|
+
from sdm import ColumnarTensor, EnsembleTable, Stype, TableTensor
|
|
11
|
+
from sdm.relational import (
|
|
12
|
+
RelatedTables,
|
|
13
|
+
RelationalData,
|
|
14
|
+
Relationship,
|
|
15
|
+
TaskLink,
|
|
16
|
+
)
|
|
17
|
+
from sdm.relational.backend import (
|
|
18
|
+
CuGraphRelationalSampler,
|
|
19
|
+
PyGLibRelationalSampler,
|
|
20
|
+
)
|
|
21
|
+
from sdm.tensor.mixin import DeviceMixin
|
|
22
|
+
|
|
23
|
+
T = TypeVar("T", bound=TableTensor | EnsembleTable)
|
|
24
|
+
|
|
25
|
+
EXAMPLE_ID = "__example__"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class _RelationalSamplerOutput(NamedTuple, Generic[T]):
|
|
29
|
+
task_table: TableTensor
|
|
30
|
+
related_tables: RelatedTables[T]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class RelationalSamplerOutput(
|
|
34
|
+
_RelationalSamplerOutput[T], DeviceMixin, Generic[T]
|
|
35
|
+
):
|
|
36
|
+
r"""Relational sampler output.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
task_table: The task table.
|
|
40
|
+
related_tables: The related tables for the task table.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
def _tensors(self) -> Iterator[Tensor]:
|
|
44
|
+
yield self.task_table
|
|
45
|
+
yield from self.related_tables._tensors()
|
|
46
|
+
|
|
47
|
+
def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self:
|
|
48
|
+
return self.__class__(
|
|
49
|
+
task_table=cast(TableTensor, fn(self.task_table)),
|
|
50
|
+
related_tables=self.related_tables._apply_tensor(fn),
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class RelationalSampler:
|
|
55
|
+
r"""Subgraph sampler over relational data.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
data: The collection of named tables and their relationships.
|
|
59
|
+
time_columns: Mapping from table name to the datetime column used for
|
|
60
|
+
temporal sampling. A row in a time-aware table can only be sampled
|
|
61
|
+
if its timestamp does not exceed the query timestamp.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
def __init__(
|
|
65
|
+
self,
|
|
66
|
+
data: RelationalData,
|
|
67
|
+
time_columns: Mapping[str, str] | None = None,
|
|
68
|
+
) -> None:
|
|
69
|
+
self.data = data
|
|
70
|
+
self.time_columns = time_columns or {}
|
|
71
|
+
|
|
72
|
+
for table_name, column_name in self.time_columns.items():
|
|
73
|
+
stype = data.tables[table_name].stype(column_name)
|
|
74
|
+
if stype != Stype.datetime:
|
|
75
|
+
raise ValueError(
|
|
76
|
+
f"Expected '{column_name}' in table '{table_name}' to "
|
|
77
|
+
f"have semantic type {str(Stype.datetime)!r} "
|
|
78
|
+
f"(got {str(stype)!r})"
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
if self.data.is_cuda:
|
|
82
|
+
self._sampler = CuGraphRelationalSampler(data, self.time_columns)
|
|
83
|
+
else:
|
|
84
|
+
self._sampler = PyGLibRelationalSampler(data, self.time_columns)
|
|
85
|
+
|
|
86
|
+
def __call__(
|
|
87
|
+
self,
|
|
88
|
+
task_table: TableTensor,
|
|
89
|
+
task_link: TaskLink | Mapping[str, str | Sequence[str]],
|
|
90
|
+
num_neighbors: Sequence[int],
|
|
91
|
+
task_time_column: str | None = None,
|
|
92
|
+
temporal_strategy: Literal["last", "uniform"] = "last",
|
|
93
|
+
) -> RelationalSamplerOutput:
|
|
94
|
+
r"""Alias of :meth:`sample`."""
|
|
95
|
+
return self.sample(
|
|
96
|
+
task_table=task_table,
|
|
97
|
+
task_link=task_link,
|
|
98
|
+
num_neighbors=num_neighbors,
|
|
99
|
+
task_time_column=task_time_column,
|
|
100
|
+
temporal_strategy=temporal_strategy,
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
def sample(
|
|
104
|
+
self,
|
|
105
|
+
task_table: TableTensor,
|
|
106
|
+
task_link: TaskLink | Mapping[str, str | Sequence[str]],
|
|
107
|
+
num_neighbors: Sequence[int],
|
|
108
|
+
task_time_column: str | None = None,
|
|
109
|
+
temporal_strategy: Literal["last", "uniform"] = "last",
|
|
110
|
+
) -> RelationalSamplerOutput:
|
|
111
|
+
r"""Sample :class:`RelatedTables` for task rows.
|
|
112
|
+
|
|
113
|
+
Args:
|
|
114
|
+
num_neighbors: Number of neighbors to sample per hop.
|
|
115
|
+
task_table: Task table whose rows define the sampling queries.
|
|
116
|
+
task_link: Link from ``task_table`` rows to a table in the
|
|
117
|
+
relational data.
|
|
118
|
+
task_time_column: Datetime column in ``task_table`` used as the
|
|
119
|
+
query timestamp for temporal sampling.
|
|
120
|
+
temporal_strategy: How temporal neighbors are selected. ``"last"``
|
|
121
|
+
selects the most recent neighbors before each seed timestamp.
|
|
122
|
+
``"uniform"`` samples uniformly from neighbors before each seed
|
|
123
|
+
timestamp.
|
|
124
|
+
"""
|
|
125
|
+
if not isinstance(task_link, TaskLink):
|
|
126
|
+
task_link = TaskLink.from_mapping(task_link)
|
|
127
|
+
|
|
128
|
+
for table, columns in (
|
|
129
|
+
(task_table, task_link.task_columns),
|
|
130
|
+
(self.data.tables[task_link.table], task_link.table_columns),
|
|
131
|
+
):
|
|
132
|
+
for column in columns:
|
|
133
|
+
stype = table.stype(column)
|
|
134
|
+
if stype != Stype.id:
|
|
135
|
+
raise ValueError(
|
|
136
|
+
f"Expected column '{column}' to have semantic type "
|
|
137
|
+
f"{str(Stype.id)!r} (got {str(stype)!r})"
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
if task_time_column is not None:
|
|
141
|
+
stype = task_table.stype(task_time_column)
|
|
142
|
+
if stype != Stype.datetime:
|
|
143
|
+
raise ValueError(
|
|
144
|
+
f"Expected task time column to have semantic type "
|
|
145
|
+
f"{str(Stype.datetime)!r} (got {str(stype)!r})"
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
shared = False
|
|
149
|
+
if task_table.dim() not in (2, 3):
|
|
150
|
+
raise ValueError(
|
|
151
|
+
f"Task table needs to be either 2D or 3D "
|
|
152
|
+
f"(got {task_table.dim()})"
|
|
153
|
+
)
|
|
154
|
+
if task_table.dim() == 3:
|
|
155
|
+
num_members, num_rows = task_table.size()[:2]
|
|
156
|
+
shared = all(
|
|
157
|
+
all(column.stride(0) == 0 for column in block.unbind(-1))
|
|
158
|
+
if isinstance(block, ColumnarTensor)
|
|
159
|
+
else block.stride(0) == 0
|
|
160
|
+
for _, block in task_table.items()
|
|
161
|
+
if block.size(-1) > 0
|
|
162
|
+
)
|
|
163
|
+
if shared:
|
|
164
|
+
task_table = task_table[0]
|
|
165
|
+
else:
|
|
166
|
+
task_table = cast(TableTensor, task_table.flatten(0, 1))
|
|
167
|
+
else:
|
|
168
|
+
num_members, num_rows = None, task_table.size(0)
|
|
169
|
+
|
|
170
|
+
nodes = self._sampler.sample(
|
|
171
|
+
task_table=task_table,
|
|
172
|
+
task_link=task_link,
|
|
173
|
+
num_neighbors=num_neighbors,
|
|
174
|
+
task_time_column=task_time_column,
|
|
175
|
+
temporal_strategy=temporal_strategy,
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
tables: dict[str, TableTensor | EnsembleTable] = {}
|
|
179
|
+
for table_name, (example, index) in nodes.items():
|
|
180
|
+
table = torch.cat(
|
|
181
|
+
[
|
|
182
|
+
self.data.tables[table_name][index],
|
|
183
|
+
TableTensor(
|
|
184
|
+
columns={"id": (EXAMPLE_ID,)},
|
|
185
|
+
id=ColumnarTensor(
|
|
186
|
+
(
|
|
187
|
+
example
|
|
188
|
+
if num_members is None or shared
|
|
189
|
+
else example % num_rows,
|
|
190
|
+
)
|
|
191
|
+
),
|
|
192
|
+
),
|
|
193
|
+
],
|
|
194
|
+
dim=-1,
|
|
195
|
+
)
|
|
196
|
+
table = cast(TableTensor, table)
|
|
197
|
+
|
|
198
|
+
if num_members is None:
|
|
199
|
+
tables[table_name] = table
|
|
200
|
+
continue
|
|
201
|
+
|
|
202
|
+
if shared:
|
|
203
|
+
tables[table_name] = EnsembleTable.from_tables(
|
|
204
|
+
tables=[table for _ in range(num_members)],
|
|
205
|
+
member_table_ids=range(num_members),
|
|
206
|
+
)
|
|
207
|
+
continue
|
|
208
|
+
|
|
209
|
+
member = example // num_rows
|
|
210
|
+
tables[table_name] = EnsembleTable.from_tables(
|
|
211
|
+
tables=[table[member == i] for i in range(num_members)],
|
|
212
|
+
member_table_ids=range(num_members),
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
# Build composite keys for disjoint linkage across examples:
|
|
216
|
+
relationships = tuple(
|
|
217
|
+
Relationship(
|
|
218
|
+
left_table=rel.left_table,
|
|
219
|
+
left_columns=(*rel.left_columns, EXAMPLE_ID),
|
|
220
|
+
right_table=rel.right_table,
|
|
221
|
+
right_columns=(*rel.right_columns, EXAMPLE_ID),
|
|
222
|
+
)
|
|
223
|
+
for rel in self.data.relationships
|
|
224
|
+
if rel.left_table in tables and rel.right_table in tables
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
task_link = TaskLink(
|
|
228
|
+
task_columns=(*task_link.task_columns, EXAMPLE_ID),
|
|
229
|
+
table=task_link.table,
|
|
230
|
+
table_columns=(*task_link.table_columns, EXAMPLE_ID),
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
arange = torch.arange(task_table.size(0), device=task_table.device)
|
|
234
|
+
if num_members is not None:
|
|
235
|
+
arange = arange % num_rows
|
|
236
|
+
task_table: Tensor = torch.cat(
|
|
237
|
+
[
|
|
238
|
+
task_table,
|
|
239
|
+
TableTensor(
|
|
240
|
+
columns={"id": (EXAMPLE_ID,)},
|
|
241
|
+
id=ColumnarTensor((arange,)),
|
|
242
|
+
),
|
|
243
|
+
],
|
|
244
|
+
dim=-1,
|
|
245
|
+
)
|
|
246
|
+
if num_members is not None:
|
|
247
|
+
if shared:
|
|
248
|
+
task_table = task_table.expand(num_members, *task_table.size())
|
|
249
|
+
else:
|
|
250
|
+
task_table = task_table.view(num_members, num_rows, -1)
|
|
251
|
+
|
|
252
|
+
return RelationalSamplerOutput(
|
|
253
|
+
task_table=cast(TableTensor, task_table),
|
|
254
|
+
related_tables=RelatedTables(
|
|
255
|
+
tables=tables,
|
|
256
|
+
relationships=relationships,
|
|
257
|
+
task_links=(task_link,),
|
|
258
|
+
),
|
|
259
|
+
)
|