structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from collections.abc import Mapping, Sequence
|
|
5
|
+
from typing import Any, Literal, cast
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import torch
|
|
9
|
+
from torch import Tensor
|
|
10
|
+
|
|
11
|
+
from sdm import NaT, TableTensor
|
|
12
|
+
from sdm._warnings import warn_once
|
|
13
|
+
from sdm.relational import RelationalData, TaskLink
|
|
14
|
+
from sdm.relational.join import join_index
|
|
15
|
+
|
|
16
|
+
_INTEGER_DTYPES = {
|
|
17
|
+
torch.uint8,
|
|
18
|
+
torch.int8,
|
|
19
|
+
torch.int16,
|
|
20
|
+
torch.int32,
|
|
21
|
+
torch.int64,
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class CuGraphRelationalSampler:
|
|
26
|
+
def __init__(
|
|
27
|
+
self,
|
|
28
|
+
data: RelationalData,
|
|
29
|
+
time_columns: Mapping[str, str],
|
|
30
|
+
) -> None:
|
|
31
|
+
|
|
32
|
+
if not data.is_cuda:
|
|
33
|
+
raise ValueError(
|
|
34
|
+
f"{self.__class__.__name__!r} requires input data on a CUDA "
|
|
35
|
+
f"device (got '{data.device}')"
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
try:
|
|
39
|
+
import cupy as cp
|
|
40
|
+
import pylibcugraph # noqa: PLC0415
|
|
41
|
+
except ImportError as e:
|
|
42
|
+
raise ImportError(
|
|
43
|
+
f"{self.__class__.__name__!r} requires 'cupy' and "
|
|
44
|
+
f"'pylibcugraph'"
|
|
45
|
+
) from e
|
|
46
|
+
|
|
47
|
+
self.data = data
|
|
48
|
+
self.time_columns = time_columns
|
|
49
|
+
|
|
50
|
+
self._cp = cp
|
|
51
|
+
self._pylibcugraph = pylibcugraph
|
|
52
|
+
self._table_names = tuple(self.data.tables)
|
|
53
|
+
self._table_ids = {
|
|
54
|
+
table_name: i for i, table_name in enumerate(self._table_names)
|
|
55
|
+
}
|
|
56
|
+
self._numeric_seed_lookups: dict[
|
|
57
|
+
tuple[str, str], tuple[Tensor, int, Tensor, Tensor]
|
|
58
|
+
] = {}
|
|
59
|
+
|
|
60
|
+
offsets = [0]
|
|
61
|
+
for table in self.data.tables.values():
|
|
62
|
+
offsets.append(offsets[-1] + table.size(0))
|
|
63
|
+
self._vertex_offsets = torch.tensor(
|
|
64
|
+
offsets,
|
|
65
|
+
dtype=torch.int64,
|
|
66
|
+
device=self.data.device,
|
|
67
|
+
)
|
|
68
|
+
self._num_vertices = offsets[-1]
|
|
69
|
+
self._num_edge_types = 2 * len(self.data.relationships)
|
|
70
|
+
self._num_edges = 0
|
|
71
|
+
self._outgoing_edge_types = {
|
|
72
|
+
table_name: [] for table_name in self._table_names
|
|
73
|
+
}
|
|
74
|
+
self._neighbor_tables = {
|
|
75
|
+
table_name: set() for table_name in self._table_names
|
|
76
|
+
}
|
|
77
|
+
for i, relationship in enumerate(self.data.relationships):
|
|
78
|
+
self._outgoing_edge_types[relationship.right_table].append(2 * i)
|
|
79
|
+
self._outgoing_edge_types[relationship.left_table].append(
|
|
80
|
+
2 * i + 1
|
|
81
|
+
)
|
|
82
|
+
self._neighbor_tables[relationship.right_table].add(
|
|
83
|
+
relationship.left_table
|
|
84
|
+
)
|
|
85
|
+
self._neighbor_tables[relationship.left_table].add(
|
|
86
|
+
relationship.right_table
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
with torch.cuda.device(self.data.device):
|
|
90
|
+
self._build_graph()
|
|
91
|
+
|
|
92
|
+
def sample(
|
|
93
|
+
self,
|
|
94
|
+
task_table: TableTensor,
|
|
95
|
+
task_link: TaskLink,
|
|
96
|
+
num_neighbors: Sequence[int],
|
|
97
|
+
task_time_column: str | None = None,
|
|
98
|
+
temporal_strategy: Literal["last", "uniform"] = "last",
|
|
99
|
+
) -> dict[str, tuple[Tensor, Tensor]]:
|
|
100
|
+
|
|
101
|
+
if task_table.device != self.data.device:
|
|
102
|
+
raise ValueError(
|
|
103
|
+
f"Expected task and relational tables on the same CUDA device "
|
|
104
|
+
f"(got '{task_table.device}' and '{self.data.device}')"
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
with torch.cuda.device(self.data.device):
|
|
108
|
+
seed = self._resolve_seed(
|
|
109
|
+
task_table=task_table,
|
|
110
|
+
task_link=task_link,
|
|
111
|
+
)
|
|
112
|
+
if task_time_column is None:
|
|
113
|
+
seed_time = torch.full_like(
|
|
114
|
+
seed,
|
|
115
|
+
torch.iinfo(torch.int64).max,
|
|
116
|
+
)
|
|
117
|
+
else:
|
|
118
|
+
seed_time = task_table[task_time_column].datetime.squeeze(-1)
|
|
119
|
+
|
|
120
|
+
if self._num_edges == 0 or len(num_neighbors) == 0:
|
|
121
|
+
return self._seed_nodes(seed=seed, task_link=task_link)
|
|
122
|
+
|
|
123
|
+
if len(self.time_columns) == 0:
|
|
124
|
+
return self._sample_non_temporal(
|
|
125
|
+
seed=seed,
|
|
126
|
+
num_neighbors=num_neighbors,
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
if temporal_strategy == "last":
|
|
130
|
+
warn_once(
|
|
131
|
+
key="cugraph-last-strategy",
|
|
132
|
+
message=(
|
|
133
|
+
"cugraph-based sampling does not yet support "
|
|
134
|
+
"temporal_strategy='last'. Falling back to 'uniform' "
|
|
135
|
+
"temporal sampling."
|
|
136
|
+
),
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
return self._sample_temporal(
|
|
140
|
+
seed=seed,
|
|
141
|
+
seed_time=seed_time,
|
|
142
|
+
num_neighbors=num_neighbors,
|
|
143
|
+
seed_table=task_link.table,
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
def _build_graph(self) -> None:
|
|
147
|
+
cp = self._cp
|
|
148
|
+
pylibcugraph = self._pylibcugraph
|
|
149
|
+
edge_indices = self.data.edge_indices(dtype=torch.int64)
|
|
150
|
+
|
|
151
|
+
srcs: list[Tensor] = []
|
|
152
|
+
dsts: list[Tensor] = []
|
|
153
|
+
edge_types: list[Tensor] = []
|
|
154
|
+
edge_times: list[Tensor] = []
|
|
155
|
+
|
|
156
|
+
times = {
|
|
157
|
+
table_name: self.data.tables[table_name][
|
|
158
|
+
column_name
|
|
159
|
+
].datetime.squeeze(-1)
|
|
160
|
+
for table_name, column_name in self.time_columns.items()
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
for i, (relationship, edge_index) in enumerate(
|
|
164
|
+
zip(self.data.relationships, edge_indices)
|
|
165
|
+
):
|
|
166
|
+
left_offset = self._table_offset(relationship.left_table)
|
|
167
|
+
right_offset = self._table_offset(relationship.right_table)
|
|
168
|
+
left, right = edge_index
|
|
169
|
+
|
|
170
|
+
# cuGraph traverses outgoing edges. Reverse each PyG direction so
|
|
171
|
+
# a right-table seed discovers its left-table relational rows.
|
|
172
|
+
srcs.extend((right + right_offset, left + left_offset))
|
|
173
|
+
dsts.extend((left + left_offset, right + right_offset))
|
|
174
|
+
edge_types.extend(
|
|
175
|
+
(
|
|
176
|
+
torch.full_like(left, 2 * i, dtype=torch.int32),
|
|
177
|
+
torch.full_like(left, 2 * i + 1, dtype=torch.int32),
|
|
178
|
+
)
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
for table_name, index in (
|
|
182
|
+
(relationship.left_table, left),
|
|
183
|
+
(relationship.right_table, right),
|
|
184
|
+
):
|
|
185
|
+
if table_name in times:
|
|
186
|
+
edge_times.append(times[table_name][index])
|
|
187
|
+
else:
|
|
188
|
+
edge_times.append(torch.full_like(index, NaT))
|
|
189
|
+
|
|
190
|
+
if self._num_edge_types == 0:
|
|
191
|
+
self._resource_handle = None
|
|
192
|
+
self._graph = None
|
|
193
|
+
return
|
|
194
|
+
|
|
195
|
+
src = torch.cat(srcs)
|
|
196
|
+
dst = torch.cat(dsts)
|
|
197
|
+
self._num_edges = src.numel()
|
|
198
|
+
edge_type = torch.cat(edge_types)
|
|
199
|
+
edge_time = torch.cat(edge_times) if self.time_columns else None
|
|
200
|
+
self._has_outgoing = torch.zeros(
|
|
201
|
+
self._num_vertices,
|
|
202
|
+
dtype=torch.bool,
|
|
203
|
+
device=self.data.device,
|
|
204
|
+
)
|
|
205
|
+
self._has_outgoing[src] = True
|
|
206
|
+
|
|
207
|
+
self._resource_handle = pylibcugraph.ResourceHandle()
|
|
208
|
+
properties = pylibcugraph.GraphProperties(
|
|
209
|
+
is_multigraph=True,
|
|
210
|
+
is_symmetric=False,
|
|
211
|
+
)
|
|
212
|
+
self._graph = pylibcugraph.SGGraph(
|
|
213
|
+
self._resource_handle,
|
|
214
|
+
properties,
|
|
215
|
+
cp.from_dlpack(src),
|
|
216
|
+
cp.from_dlpack(dst),
|
|
217
|
+
vertices_array=cp.arange(self._num_vertices, dtype=cp.int64),
|
|
218
|
+
edge_id_array=cp.arange(src.numel(), dtype=cp.int64),
|
|
219
|
+
edge_type_array=cp.from_dlpack(edge_type),
|
|
220
|
+
edge_start_time_array=(
|
|
221
|
+
cp.from_dlpack(edge_time) if edge_time is not None else None
|
|
222
|
+
),
|
|
223
|
+
store_transposed=False,
|
|
224
|
+
renumber=False,
|
|
225
|
+
do_expensive_check=False,
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
def _resolve_seed(
|
|
229
|
+
self,
|
|
230
|
+
task_table: TableTensor,
|
|
231
|
+
task_link: TaskLink,
|
|
232
|
+
) -> Tensor:
|
|
233
|
+
local_seed = self._resolve_numeric_seed(
|
|
234
|
+
task_table=task_table,
|
|
235
|
+
task_link=task_link,
|
|
236
|
+
)
|
|
237
|
+
if local_seed is None:
|
|
238
|
+
local_seed = self._resolve_seed_join(
|
|
239
|
+
task_table=task_table,
|
|
240
|
+
task_link=task_link,
|
|
241
|
+
)
|
|
242
|
+
return local_seed + self._table_offset(task_link.table)
|
|
243
|
+
|
|
244
|
+
def _resolve_numeric_seed(
|
|
245
|
+
self,
|
|
246
|
+
task_table: TableTensor,
|
|
247
|
+
task_link: TaskLink,
|
|
248
|
+
) -> Tensor | None:
|
|
249
|
+
if len(task_link.task_columns) != 1:
|
|
250
|
+
return None
|
|
251
|
+
|
|
252
|
+
task_value = task_table[task_link.task_columns[0]].id[..., 0]
|
|
253
|
+
table_value = self.data.tables[task_link.table][
|
|
254
|
+
task_link.table_columns[0]
|
|
255
|
+
].id[..., 0]
|
|
256
|
+
|
|
257
|
+
# ColumnarTensor stores numeric IDs as plain tensors. String and
|
|
258
|
+
# composite IDs continue through the general cuDF join below.
|
|
259
|
+
if (
|
|
260
|
+
task_value.__class__ is not Tensor
|
|
261
|
+
or table_value.__class__ is not Tensor
|
|
262
|
+
or task_value.dtype != table_value.dtype
|
|
263
|
+
or task_value.dtype not in _INTEGER_DTYPES
|
|
264
|
+
):
|
|
265
|
+
return None
|
|
266
|
+
|
|
267
|
+
key = (task_link.table, task_link.table_columns[0])
|
|
268
|
+
lookup = self._numeric_seed_lookups.get(key)
|
|
269
|
+
if lookup is None or not self._lookup_matches_source(
|
|
270
|
+
lookup,
|
|
271
|
+
table_value,
|
|
272
|
+
):
|
|
273
|
+
value, row = table_value.sort()
|
|
274
|
+
lookup = (table_value, table_value._version, value, row)
|
|
275
|
+
self._numeric_seed_lookups[key] = lookup
|
|
276
|
+
_, _, value, row = lookup
|
|
277
|
+
|
|
278
|
+
task_value = task_value.contiguous()
|
|
279
|
+
lower = torch.searchsorted(value, task_value)
|
|
280
|
+
upper = torch.searchsorted(value, task_value, right=True)
|
|
281
|
+
if not torch.all((upper - lower) == 1):
|
|
282
|
+
raise ValueError(
|
|
283
|
+
f"Expected each task row to match exactly one row in "
|
|
284
|
+
f"'{task_link.table}'"
|
|
285
|
+
)
|
|
286
|
+
return row[lower]
|
|
287
|
+
|
|
288
|
+
@staticmethod
|
|
289
|
+
def _lookup_matches_source(
|
|
290
|
+
lookup: tuple[Tensor, int, Tensor, Tensor],
|
|
291
|
+
source: Tensor,
|
|
292
|
+
) -> bool:
|
|
293
|
+
cached, version, _, _ = lookup
|
|
294
|
+
return cast(
|
|
295
|
+
bool,
|
|
296
|
+
cached._version == version
|
|
297
|
+
and cached.dtype == source.dtype
|
|
298
|
+
and cached.device == source.device
|
|
299
|
+
and cached.layout == source.layout
|
|
300
|
+
and cached.data_ptr() == source.data_ptr()
|
|
301
|
+
and cached.size() == source.size()
|
|
302
|
+
and cached.stride() == source.stride()
|
|
303
|
+
and cached.storage_offset() == source.storage_offset(),
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
def _resolve_seed_join(
|
|
307
|
+
self,
|
|
308
|
+
task_table: TableTensor,
|
|
309
|
+
task_link: TaskLink,
|
|
310
|
+
) -> Tensor:
|
|
311
|
+
task_index, seed = join_index(
|
|
312
|
+
left_table=task_table,
|
|
313
|
+
right_table=self.data.tables[task_link.table],
|
|
314
|
+
left_keys=task_link.task_columns,
|
|
315
|
+
right_keys=task_link.table_columns,
|
|
316
|
+
dtype=torch.int64,
|
|
317
|
+
device=task_table.device,
|
|
318
|
+
)
|
|
319
|
+
task_index, perm = task_index.sort()
|
|
320
|
+
seed = seed[perm]
|
|
321
|
+
|
|
322
|
+
expected = torch.arange(
|
|
323
|
+
task_table.size(0),
|
|
324
|
+
dtype=task_index.dtype,
|
|
325
|
+
device=task_index.device,
|
|
326
|
+
)
|
|
327
|
+
if not task_index.equal(expected):
|
|
328
|
+
raise ValueError(
|
|
329
|
+
f"Expected each task row to match exactly one row in "
|
|
330
|
+
f"'{task_link.table}'"
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
return seed
|
|
334
|
+
|
|
335
|
+
def _sample_non_temporal(
|
|
336
|
+
self,
|
|
337
|
+
seed: Tensor,
|
|
338
|
+
num_neighbors: Sequence[int],
|
|
339
|
+
) -> dict[str, tuple[Tensor, Tensor]]:
|
|
340
|
+
cp = self._cp
|
|
341
|
+
example = torch.arange(seed.numel(), device=seed.device)
|
|
342
|
+
seed_keys = example * self._num_vertices + seed
|
|
343
|
+
if num_neighbors[0] == 0:
|
|
344
|
+
return self._nodes_from_keys(seed_keys)
|
|
345
|
+
|
|
346
|
+
active = self._has_outgoing[seed]
|
|
347
|
+
active_seed = seed[active]
|
|
348
|
+
active_example = example[active]
|
|
349
|
+
if active_seed.numel() == 0:
|
|
350
|
+
return self._nodes_from_keys(seed_keys)
|
|
351
|
+
|
|
352
|
+
fanout = np.repeat(
|
|
353
|
+
np.asarray(num_neighbors, dtype=np.int32),
|
|
354
|
+
self._num_edge_types,
|
|
355
|
+
)
|
|
356
|
+
label_offsets = cp.arange(active_seed.numel() + 1, dtype=cp.int64)
|
|
357
|
+
result = self._pylibcugraph.heterogeneous_uniform_neighbor_sample(
|
|
358
|
+
self._resource_handle,
|
|
359
|
+
self._graph,
|
|
360
|
+
cp.from_dlpack(active_seed),
|
|
361
|
+
label_offsets,
|
|
362
|
+
cp.from_dlpack(self._vertex_offsets),
|
|
363
|
+
fanout,
|
|
364
|
+
num_edge_types=self._num_edge_types,
|
|
365
|
+
with_replacement=False,
|
|
366
|
+
do_expensive_check=False,
|
|
367
|
+
prior_sources_behavior="exclude",
|
|
368
|
+
deduplicate_sources=True,
|
|
369
|
+
disjoint_sampling=False,
|
|
370
|
+
return_hops=True,
|
|
371
|
+
renumber=True,
|
|
372
|
+
retain_seeds=True,
|
|
373
|
+
compression="COO",
|
|
374
|
+
compress_per_hop=False,
|
|
375
|
+
random_state=self._next_random_state(),
|
|
376
|
+
)
|
|
377
|
+
node = self._as_tensor(result["renumber_map"])
|
|
378
|
+
offsets = self._as_tensor(result["renumber_map_offsets"]).long()
|
|
379
|
+
position = torch.arange(node.numel(), device=node.device)
|
|
380
|
+
segment = torch.bucketize(position, offsets[1:], right=True)
|
|
381
|
+
num_tables = len(self._table_names)
|
|
382
|
+
sampled_example = active_example[
|
|
383
|
+
segment.div(num_tables, rounding_mode="floor")
|
|
384
|
+
]
|
|
385
|
+
sampled_keys = sampled_example * self._num_vertices + node
|
|
386
|
+
return self._nodes_from_keys(
|
|
387
|
+
torch.cat((seed_keys, sampled_keys)).unique(sorted=True)
|
|
388
|
+
)
|
|
389
|
+
|
|
390
|
+
def _sample_temporal(
|
|
391
|
+
self,
|
|
392
|
+
seed: Tensor,
|
|
393
|
+
seed_time: Tensor,
|
|
394
|
+
num_neighbors: Sequence[int],
|
|
395
|
+
seed_table: str,
|
|
396
|
+
) -> dict[str, tuple[Tensor, Tensor]]:
|
|
397
|
+
total = self._num_vertices
|
|
398
|
+
example = torch.arange(seed.numel(), device=seed.device)
|
|
399
|
+
visited = (example * total + seed).unique(sorted=True)
|
|
400
|
+
frontier = visited
|
|
401
|
+
possible_source_tables = {seed_table}
|
|
402
|
+
|
|
403
|
+
for count in num_neighbors:
|
|
404
|
+
if frontier.numel() == 0:
|
|
405
|
+
break
|
|
406
|
+
frontier_example = frontier.div(total, rounding_mode="floor")
|
|
407
|
+
frontier_node = frontier.remainder(total)
|
|
408
|
+
sampled = self._sample_temporal_hop(
|
|
409
|
+
frontier_node=frontier_node,
|
|
410
|
+
frontier_example=frontier_example,
|
|
411
|
+
seed_time=seed_time,
|
|
412
|
+
count=count,
|
|
413
|
+
possible_source_tables=possible_source_tables,
|
|
414
|
+
)
|
|
415
|
+
if sampled.numel() == 0:
|
|
416
|
+
break
|
|
417
|
+
|
|
418
|
+
sampled = sampled.unique(sorted=True)
|
|
419
|
+
position = torch.searchsorted(visited, sampled)
|
|
420
|
+
clamped = position.clamp_max(max(visited.numel() - 1, 0))
|
|
421
|
+
is_visited = (position < visited.numel()) & (
|
|
422
|
+
visited[clamped] == sampled
|
|
423
|
+
)
|
|
424
|
+
frontier = sampled[~is_visited]
|
|
425
|
+
visited = torch.cat((visited, frontier)).sort().values
|
|
426
|
+
possible_source_tables = {
|
|
427
|
+
neighbor
|
|
428
|
+
for table_name in possible_source_tables
|
|
429
|
+
for neighbor in self._neighbor_tables[table_name]
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
return self._nodes_from_keys(visited)
|
|
433
|
+
|
|
434
|
+
def _nodes_from_keys(
|
|
435
|
+
self,
|
|
436
|
+
keys: Tensor,
|
|
437
|
+
) -> dict[str, tuple[Tensor, Tensor]]:
|
|
438
|
+
example = keys.div(self._num_vertices, rounding_mode="floor")
|
|
439
|
+
node = keys.remainder(self._num_vertices)
|
|
440
|
+
table_id = torch.bucketize(
|
|
441
|
+
node,
|
|
442
|
+
self._vertex_offsets[1:],
|
|
443
|
+
right=True,
|
|
444
|
+
)
|
|
445
|
+
nodes: dict[str, tuple[Tensor, Tensor]] = {}
|
|
446
|
+
for i, table_name in enumerate(self._table_names):
|
|
447
|
+
mask = table_id == i
|
|
448
|
+
nodes[table_name] = (
|
|
449
|
+
example[mask],
|
|
450
|
+
node[mask] - self._vertex_offsets[i],
|
|
451
|
+
)
|
|
452
|
+
return {
|
|
453
|
+
table_name: (example, node)
|
|
454
|
+
for table_name, (example, node) in nodes.items()
|
|
455
|
+
if node.numel() > 0
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
def _sample_temporal_hop(
|
|
459
|
+
self,
|
|
460
|
+
frontier_node: Tensor,
|
|
461
|
+
frontier_example: Tensor,
|
|
462
|
+
seed_time: Tensor,
|
|
463
|
+
count: int,
|
|
464
|
+
possible_source_tables: set[str],
|
|
465
|
+
) -> Tensor:
|
|
466
|
+
cp = self._cp
|
|
467
|
+
num_examples = seed_time.numel()
|
|
468
|
+
counts = torch.bincount(
|
|
469
|
+
frontier_example,
|
|
470
|
+
minlength=num_examples,
|
|
471
|
+
)
|
|
472
|
+
label_offsets = torch.cat(
|
|
473
|
+
(
|
|
474
|
+
torch.zeros(1, dtype=torch.int64, device=self.data.device),
|
|
475
|
+
counts.cumsum(0),
|
|
476
|
+
)
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
if count == 0:
|
|
480
|
+
return torch.empty(0, dtype=torch.int64, device=self.data.device)
|
|
481
|
+
# cuGraph interprets fanout independently for every edge type. Avoid
|
|
482
|
+
# scheduling types whose source table cannot occur in this hop.
|
|
483
|
+
fanout = np.zeros(self._num_edge_types, dtype=np.int32)
|
|
484
|
+
for table_name in possible_source_tables:
|
|
485
|
+
fanout[self._outgoing_edge_types[table_name]] = count
|
|
486
|
+
|
|
487
|
+
result = (
|
|
488
|
+
self._pylibcugraph.heterogeneous_uniform_temporal_neighbor_sample(
|
|
489
|
+
self._resource_handle,
|
|
490
|
+
self._graph,
|
|
491
|
+
"edge_start_time",
|
|
492
|
+
cp.from_dlpack(frontier_node),
|
|
493
|
+
cp.from_dlpack(seed_time[frontier_example]),
|
|
494
|
+
cp.from_dlpack(label_offsets),
|
|
495
|
+
cp.from_dlpack(self._vertex_offsets),
|
|
496
|
+
fanout,
|
|
497
|
+
num_edge_types=self._num_edge_types,
|
|
498
|
+
with_replacement=False,
|
|
499
|
+
do_expensive_check=False,
|
|
500
|
+
prior_sources_behavior=None,
|
|
501
|
+
deduplicate_sources=True,
|
|
502
|
+
disjoint_sampling=False,
|
|
503
|
+
return_hops=False,
|
|
504
|
+
renumber=False,
|
|
505
|
+
retain_seeds=False,
|
|
506
|
+
compression="COO",
|
|
507
|
+
compress_per_hop=False,
|
|
508
|
+
random_state=self._next_random_state(),
|
|
509
|
+
temporal_sampling_comparison="monotonically_decreasing",
|
|
510
|
+
)
|
|
511
|
+
)
|
|
512
|
+
minor = self._as_tensor(result["minors"])
|
|
513
|
+
if minor.numel() == 0:
|
|
514
|
+
return minor
|
|
515
|
+
batch = self._as_tensor(result["batch_id"]).long()
|
|
516
|
+
|
|
517
|
+
return batch * self._num_vertices + minor
|
|
518
|
+
|
|
519
|
+
def _seed_nodes(
|
|
520
|
+
self,
|
|
521
|
+
seed: Tensor,
|
|
522
|
+
task_link: TaskLink,
|
|
523
|
+
) -> dict[str, tuple[Tensor, Tensor]]:
|
|
524
|
+
local_seed = seed - self._table_offset(task_link.table)
|
|
525
|
+
return {
|
|
526
|
+
task_link.table: (
|
|
527
|
+
torch.arange(seed.numel(), device=seed.device),
|
|
528
|
+
local_seed,
|
|
529
|
+
)
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
def _table_offset(self, table_name: str) -> Tensor:
|
|
533
|
+
return self._vertex_offsets[self._table_ids[table_name]]
|
|
534
|
+
|
|
535
|
+
def _next_random_state(self) -> int:
|
|
536
|
+
return int(
|
|
537
|
+
torch.randint(
|
|
538
|
+
low=0,
|
|
539
|
+
high=torch.iinfo(torch.int32).max,
|
|
540
|
+
size=(),
|
|
541
|
+
)
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
@staticmethod
|
|
545
|
+
def _as_tensor(array: Any) -> Tensor:
|
|
546
|
+
return torch.from_dlpack(array)
|