structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Self
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
from torch import Tensor
|
|
9
|
+
|
|
10
|
+
from sdm import RelatedTables, TableTensor
|
|
11
|
+
from sdm.models.kumo.relational.graph import HomogeneousGraph
|
|
12
|
+
from sdm.relational.join import join_index
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class TaskGraph: # noqa: D101
|
|
17
|
+
x: TableTensor
|
|
18
|
+
related_tables: RelatedTables[TableTensor]
|
|
19
|
+
graph: HomogeneousGraph
|
|
20
|
+
readout_table: str
|
|
21
|
+
readout_index: Tensor # Entity-table rows ordered by task row.
|
|
22
|
+
task_row_by_table: dict[str, Tensor] # Per table task-row assignment.
|
|
23
|
+
num_hops: int
|
|
24
|
+
|
|
25
|
+
@classmethod
|
|
26
|
+
def from_input( # noqa: D102
|
|
27
|
+
cls,
|
|
28
|
+
x: TableTensor,
|
|
29
|
+
related_tables: RelatedTables[TableTensor],
|
|
30
|
+
num_hops: int | None = None,
|
|
31
|
+
) -> Self:
|
|
32
|
+
|
|
33
|
+
if x.dim() != 2:
|
|
34
|
+
raise ValueError("Tables need to be two-dimensional")
|
|
35
|
+
|
|
36
|
+
for table in related_tables.tables.values():
|
|
37
|
+
if table.dim() != 2:
|
|
38
|
+
raise ValueError("Tables need to be two-dimensional")
|
|
39
|
+
|
|
40
|
+
if len(related_tables.task_links) != 1:
|
|
41
|
+
raise ValueError(
|
|
42
|
+
f"'KumoRelational' expects exactly one task link to an "
|
|
43
|
+
f"entity table "
|
|
44
|
+
f"(got {len(related_tables.task_links)})"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
graph = HomogeneousGraph.from_tables(
|
|
48
|
+
tables=related_tables.tables,
|
|
49
|
+
relationships=related_tables.relationships,
|
|
50
|
+
)
|
|
51
|
+
readout_table = related_tables.task_links[0].table
|
|
52
|
+
readout_offset = graph.start_node_offsets[readout_table]
|
|
53
|
+
|
|
54
|
+
# Map each task row to exactly one entity-table row:
|
|
55
|
+
task_index, readout_index = join_index(
|
|
56
|
+
left_table=x,
|
|
57
|
+
right_table=related_tables.tables[readout_table],
|
|
58
|
+
left_keys=related_tables.task_links[0].task_columns,
|
|
59
|
+
right_keys=related_tables.task_links[0].table_columns,
|
|
60
|
+
how="inner",
|
|
61
|
+
)
|
|
62
|
+
task_index, perm = task_index.sort()
|
|
63
|
+
readout_index = readout_index[perm]
|
|
64
|
+
global_readout_index = readout_index + readout_offset
|
|
65
|
+
|
|
66
|
+
arange = torch.arange(
|
|
67
|
+
x.size(-2),
|
|
68
|
+
dtype=task_index.dtype,
|
|
69
|
+
device=task_index.device,
|
|
70
|
+
)
|
|
71
|
+
if (
|
|
72
|
+
not task_index.equal(arange)
|
|
73
|
+
or readout_index.unique().numel() != readout_index.numel()
|
|
74
|
+
):
|
|
75
|
+
raise ValueError(
|
|
76
|
+
f"Expected each task row to match exactly one distinct row in "
|
|
77
|
+
f"{readout_table!r}"
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
task_row = readout_index.new_full((graph.num_nodes,), fill_value=-1)
|
|
81
|
+
task_row[global_readout_index] = task_index
|
|
82
|
+
frontier = torch.zeros_like(task_row, dtype=torch.bool)
|
|
83
|
+
frontier[global_readout_index] = True
|
|
84
|
+
|
|
85
|
+
# Propagate task assignment along graph edges.
|
|
86
|
+
# NOTE This assumes neighborhoods do not overlap.
|
|
87
|
+
propagated_hops = 0
|
|
88
|
+
while True:
|
|
89
|
+
if num_hops is not None and propagated_hops >= num_hops:
|
|
90
|
+
break
|
|
91
|
+
mask = frontier[graph.row] & (task_row[graph.col] < 0)
|
|
92
|
+
row = graph.row[mask]
|
|
93
|
+
if row.numel() == 0:
|
|
94
|
+
break
|
|
95
|
+
col = graph.col[mask]
|
|
96
|
+
|
|
97
|
+
task_row[col] = task_row[row]
|
|
98
|
+
frontier.fill_(False)
|
|
99
|
+
frontier[col] = True
|
|
100
|
+
propagated_hops += 1
|
|
101
|
+
|
|
102
|
+
return cls(
|
|
103
|
+
x=x,
|
|
104
|
+
related_tables=related_tables,
|
|
105
|
+
graph=graph,
|
|
106
|
+
readout_table=readout_table,
|
|
107
|
+
readout_index=readout_index,
|
|
108
|
+
task_row_by_table={
|
|
109
|
+
table_name: task_row[graph.node_slice(table_name)]
|
|
110
|
+
for table_name in related_tables.tables
|
|
111
|
+
},
|
|
112
|
+
num_hops=propagated_hops if num_hops is None else num_hops,
|
|
113
|
+
)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""KumoTabular."""
|
|
5
|
+
|
|
6
|
+
from sdm.models.kumo.tabular.model import KumoTabular
|
|
7
|
+
from sdm.models.kumo.tabular.batching import (
|
|
8
|
+
estimate_fit_batch_size,
|
|
9
|
+
estimate_predict_batch_size,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"KumoTabular",
|
|
15
|
+
"estimate_fit_batch_size",
|
|
16
|
+
"estimate_predict_batch_size",
|
|
17
|
+
]
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import math
|
|
5
|
+
from typing import cast
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
|
|
9
|
+
from sdm import TableTensor, Task
|
|
10
|
+
from sdm.models.kumo.tabular.block import KumoTabularTransformerBlock
|
|
11
|
+
from sdm.models.kumo.tabular.model import KumoTabular, _KumoTabular
|
|
12
|
+
from sdm.nn import InducedTransformerBlock
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def estimate_fit_batch_size(
|
|
16
|
+
model: KumoTabular,
|
|
17
|
+
x: TableTensor,
|
|
18
|
+
y: TableTensor,
|
|
19
|
+
*,
|
|
20
|
+
memory_budget: int,
|
|
21
|
+
num_estimators: int = 1,
|
|
22
|
+
dtype: torch.dtype = torch.float16,
|
|
23
|
+
) -> int:
|
|
24
|
+
"""Estimate how many ensemble members to fit together.
|
|
25
|
+
|
|
26
|
+
For default-recipe inference. Uses half the budget; the estimate does
|
|
27
|
+
not guarantee that it fits in memory.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
model: Kumo Tabular model with the target task initialized.
|
|
31
|
+
x: Unprocessed context features of shape ``[R, C]``.
|
|
32
|
+
y: Context targets of shape ``[R, 1]``. Categorical metadata defines
|
|
33
|
+
all classes; numerical targets select regression.
|
|
34
|
+
memory_budget: Available bytes excluding weights and inputs.
|
|
35
|
+
num_estimators: Total number of ensemble members.
|
|
36
|
+
dtype: Neural execution or autocast dtype.
|
|
37
|
+
|
|
38
|
+
Returns:
|
|
39
|
+
At least one member, capped at ``num_estimators``. Pass this as
|
|
40
|
+
``estimator_batch_size`` to :meth:`~sdm.models.KumoTabular.fit`.
|
|
41
|
+
"""
|
|
42
|
+
num_rows, num_columns = x.shape[-2:]
|
|
43
|
+
num_classes = (
|
|
44
|
+
len(y.categorical.categories[0]) if y.categorical.size(-1) else 0
|
|
45
|
+
)
|
|
46
|
+
workspace, cache, member = _row_bytes(
|
|
47
|
+
model=model,
|
|
48
|
+
num_columns=num_columns,
|
|
49
|
+
num_classes=num_classes,
|
|
50
|
+
dtype=dtype,
|
|
51
|
+
)
|
|
52
|
+
capacity = (memory_budget // 2) // max(
|
|
53
|
+
num_rows * (workspace + cache) + member, 1
|
|
54
|
+
)
|
|
55
|
+
return max(1, min(num_estimators, capacity))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def estimate_predict_batch_size(
|
|
59
|
+
model: KumoTabular,
|
|
60
|
+
x: TableTensor,
|
|
61
|
+
y: TableTensor,
|
|
62
|
+
*,
|
|
63
|
+
memory_budget: int,
|
|
64
|
+
num_estimators: int = 1,
|
|
65
|
+
estimator_batch_size: int = 1,
|
|
66
|
+
dtype: torch.dtype = torch.float16,
|
|
67
|
+
) -> int:
|
|
68
|
+
"""Estimate how many query rows to predict together.
|
|
69
|
+
|
|
70
|
+
For default-recipe inference. Uses half the budget; the estimate does
|
|
71
|
+
not guarantee that it fits in memory.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
model: Kumo Tabular model with the target task initialized.
|
|
75
|
+
x: Unprocessed context features of shape ``[R, C]``.
|
|
76
|
+
y: Context targets of shape ``[R, 1]``. Categorical metadata defines
|
|
77
|
+
all classes; numerical targets select regression.
|
|
78
|
+
memory_budget: Available bytes excluding weights, inputs, and context
|
|
79
|
+
caches, including overlapping transfers.
|
|
80
|
+
num_estimators: Total number of ensemble members used for fitting.
|
|
81
|
+
estimator_batch_size: Ensemble members run together, as set during fit.
|
|
82
|
+
dtype: Neural execution or autocast dtype.
|
|
83
|
+
|
|
84
|
+
Returns:
|
|
85
|
+
At least one query row per prediction call.
|
|
86
|
+
"""
|
|
87
|
+
num_columns = x.size(-1)
|
|
88
|
+
num_classes = (
|
|
89
|
+
len(y.categorical.categories[0]) if y.categorical.size(-1) else 0
|
|
90
|
+
)
|
|
91
|
+
workspace, _, _ = _row_bytes(
|
|
92
|
+
model=model,
|
|
93
|
+
num_columns=num_columns,
|
|
94
|
+
num_classes=num_classes,
|
|
95
|
+
dtype=dtype,
|
|
96
|
+
)
|
|
97
|
+
# Preprocessing and output reduction retain all ensemble members.
|
|
98
|
+
row_bytes = workspace * estimator_batch_size
|
|
99
|
+
row_bytes += (
|
|
100
|
+
num_estimators * 8 * (8 * num_columns + 4 * (num_classes or 999))
|
|
101
|
+
)
|
|
102
|
+
return max(1, (memory_budget // 2) // max(row_bytes, 1))
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _row_bytes(
|
|
106
|
+
model: KumoTabular,
|
|
107
|
+
num_columns: int,
|
|
108
|
+
num_classes: int,
|
|
109
|
+
dtype: torch.dtype,
|
|
110
|
+
) -> tuple[int, int, int]:
|
|
111
|
+
task = Task.classification if num_classes else Task.regression
|
|
112
|
+
network = cast(_KumoTabular, model.models[task])
|
|
113
|
+
row = network.row_embedding
|
|
114
|
+
icl = network.icl_block
|
|
115
|
+
layer = cast(KumoTabularTransformerBlock, icl.layers[0])
|
|
116
|
+
tasks = 1
|
|
117
|
+
if num_classes and num_classes > model.ecoc.max_classes:
|
|
118
|
+
tasks = max(
|
|
119
|
+
math.ceil(num_classes / (model.ecoc.max_classes - 1)),
|
|
120
|
+
4 * math.ceil(math.log(num_classes, model.ecoc.max_classes)),
|
|
121
|
+
)
|
|
122
|
+
# The default recipe adds count columns and keeps at most 500 features.
|
|
123
|
+
columns = min(2 * num_columns, 500)
|
|
124
|
+
element_size = dtype.itemsize
|
|
125
|
+
workspace = tasks * (
|
|
126
|
+
element_size * 4 * (columns + row.readout_token.size(0)) * row.channels
|
|
127
|
+
+ layer.peak_bytes_per_example(
|
|
128
|
+
element_size=element_size, query_length=1
|
|
129
|
+
)
|
|
130
|
+
)
|
|
131
|
+
# Fit projects all heads before retaining the smaller query KV heads.
|
|
132
|
+
cache = tasks * element_size * 2 * layer.attn.q_dim * len(icl.layers)
|
|
133
|
+
# Fit records column-block caches per estimator, not per row.
|
|
134
|
+
inducing = sum(
|
|
135
|
+
cast(InducedTransformerBlock, block).inducing_points.size(0)
|
|
136
|
+
for block in row.col_blocks
|
|
137
|
+
)
|
|
138
|
+
member = tasks * element_size * 2 * columns * inducing * row.channels
|
|
139
|
+
return workspace, cache, member
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
# ruff: noqa: D101
|
|
5
|
+
|
|
6
|
+
from typing import Any, cast
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
from torch import Tensor
|
|
10
|
+
from torch.nn import GELU, Linear, Sequential
|
|
11
|
+
|
|
12
|
+
from sdm.nn import QueryScaling, RMSNorm, RotaryEmbedding, TransformerBlock
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class _RoPERMSNorm(Sequential):
|
|
16
|
+
def forward(self, input: Tensor) -> Tensor: # noqa: A002
|
|
17
|
+
rope, norm = self
|
|
18
|
+
return norm(input, rope=rope)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class KumoTabularTransformerBlock(TransformerBlock):
|
|
22
|
+
def __init__(
|
|
23
|
+
self,
|
|
24
|
+
channels: int,
|
|
25
|
+
num_heads: int,
|
|
26
|
+
query_scaling: QueryScaling | None,
|
|
27
|
+
rope: RotaryEmbedding | None = None,
|
|
28
|
+
device: torch.device | str | None = None,
|
|
29
|
+
dtype: torch.dtype | None = None,
|
|
30
|
+
) -> None:
|
|
31
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
32
|
+
|
|
33
|
+
query_transforms: list[torch.nn.Module] = []
|
|
34
|
+
key_transforms: list[torch.nn.Module] = []
|
|
35
|
+
if rope is not None:
|
|
36
|
+
query_transforms.append(rope)
|
|
37
|
+
key_transforms.append(rope)
|
|
38
|
+
query_transforms.append(
|
|
39
|
+
RMSNorm(
|
|
40
|
+
channels // num_heads,
|
|
41
|
+
eps=1e-6,
|
|
42
|
+
elementwise_affine=False,
|
|
43
|
+
**factory_kwargs,
|
|
44
|
+
)
|
|
45
|
+
)
|
|
46
|
+
key_transforms.append(
|
|
47
|
+
RMSNorm(
|
|
48
|
+
channels // num_heads,
|
|
49
|
+
eps=1e-6,
|
|
50
|
+
elementwise_affine=False,
|
|
51
|
+
**factory_kwargs,
|
|
52
|
+
)
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
mlp = Sequential(
|
|
56
|
+
RMSNorm(channels, **factory_kwargs),
|
|
57
|
+
Linear(channels, 2 * channels, **factory_kwargs),
|
|
58
|
+
GELU(),
|
|
59
|
+
Linear(2 * channels, channels, **factory_kwargs),
|
|
60
|
+
)
|
|
61
|
+
torch.nn.init.zeros_(cast(Linear, mlp[-1]).weight)
|
|
62
|
+
torch.nn.init.zeros_(cast(Linear, mlp[-1]).bias)
|
|
63
|
+
|
|
64
|
+
transform = Sequential if rope is None else _RoPERMSNorm
|
|
65
|
+
super().__init__(
|
|
66
|
+
channels=channels,
|
|
67
|
+
num_query_heads=num_heads,
|
|
68
|
+
mlp=mlp,
|
|
69
|
+
query_norm=RMSNorm(channels, **factory_kwargs),
|
|
70
|
+
key_value_norm=RMSNorm(channels, **factory_kwargs),
|
|
71
|
+
query_transform=transform(*query_transforms),
|
|
72
|
+
key_transform=transform(*key_transforms),
|
|
73
|
+
query_scaling=query_scaling,
|
|
74
|
+
**factory_kwargs,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
def peak_bytes_per_example(
|
|
78
|
+
self,
|
|
79
|
+
element_size: int,
|
|
80
|
+
query_length: int,
|
|
81
|
+
key_value_length: int | None = None,
|
|
82
|
+
) -> int:
|
|
83
|
+
r""":meta private:""" # noqa: D415
|
|
84
|
+
length = max(query_length, key_value_length or 0)
|
|
85
|
+
factor = 15 if element_size <= 2 else 8
|
|
86
|
+
return factor * length * element_size * self.attn.q_dim
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
if __name__ == "__main__":
|
|
90
|
+
from sdm.nn import LogScale
|
|
91
|
+
from sdm.testing.memory import benchmark_transformer_block_memory_peak
|
|
92
|
+
|
|
93
|
+
benchmark_transformer_block_memory_peak(
|
|
94
|
+
block=lambda channels, num_heads: KumoTabularTransformerBlock(
|
|
95
|
+
channels=channels,
|
|
96
|
+
num_heads=num_heads,
|
|
97
|
+
query_scaling=LogScale(num_heads=num_heads),
|
|
98
|
+
),
|
|
99
|
+
channels_and_heads=[(256, 4), (512, 4)],
|
|
100
|
+
)
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
# ruff: noqa: D101, D102
|
|
5
|
+
|
|
6
|
+
from typing import Literal, cast
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
from torch import Tensor
|
|
10
|
+
from torch.nn import Linear
|
|
11
|
+
|
|
12
|
+
from sdm.cache import Cache
|
|
13
|
+
from sdm.models.tabfm.cell_embedding import CellEmbedding as TabFMCellEmbedding
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class CellEmbedding(TabFMCellEmbedding):
|
|
17
|
+
def __init__(
|
|
18
|
+
self,
|
|
19
|
+
channels: int,
|
|
20
|
+
group_size: int,
|
|
21
|
+
num_frequencies: int,
|
|
22
|
+
device: torch.device | str | None = None,
|
|
23
|
+
dtype: torch.dtype | None = None,
|
|
24
|
+
) -> None:
|
|
25
|
+
super().__init__(
|
|
26
|
+
channels=channels,
|
|
27
|
+
group_size=group_size,
|
|
28
|
+
num_frequencies=num_frequencies,
|
|
29
|
+
device=device,
|
|
30
|
+
dtype=dtype,
|
|
31
|
+
)
|
|
32
|
+
self.nan_lin = Linear(
|
|
33
|
+
group_size,
|
|
34
|
+
channels,
|
|
35
|
+
bias=False,
|
|
36
|
+
device=device,
|
|
37
|
+
dtype=dtype,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
def forward( # type: ignore
|
|
41
|
+
self,
|
|
42
|
+
x: Tensor, # [..., R, C]
|
|
43
|
+
categorical_mask: Tensor, # [..., C]
|
|
44
|
+
*,
|
|
45
|
+
train_size: int | None = None,
|
|
46
|
+
cache: Cache | None = None,
|
|
47
|
+
batch_size_limit: int | Literal["auto"] | None = None,
|
|
48
|
+
out: Tensor | None = None,
|
|
49
|
+
) -> Tensor: # [..., R, C, D]
|
|
50
|
+
missing = x.isnan()
|
|
51
|
+
|
|
52
|
+
if cache is not None and cache.is_replaying:
|
|
53
|
+
impute_mean = cast(Tensor, cache["cell_impute_mean"])
|
|
54
|
+
else:
|
|
55
|
+
assert train_size is not None
|
|
56
|
+
impute_mean = x[..., :train_size, :].nanmean(dim=-2, keepdim=True)
|
|
57
|
+
impute_mean = impute_mean.masked_fill(impute_mean.isnan(), 0.0)
|
|
58
|
+
|
|
59
|
+
if cache is not None and cache.is_recording:
|
|
60
|
+
cache["cell_impute_mean"] = impute_mean
|
|
61
|
+
|
|
62
|
+
x = torch.where(missing, impute_mean, x)
|
|
63
|
+
|
|
64
|
+
return super().forward(
|
|
65
|
+
x,
|
|
66
|
+
categorical_mask,
|
|
67
|
+
batch_size_limit=batch_size_limit,
|
|
68
|
+
out=out,
|
|
69
|
+
missing=missing,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
def _forward( # type: ignore
|
|
73
|
+
self,
|
|
74
|
+
x: Tensor, # [..., R, C, G]
|
|
75
|
+
freq: Tensor, # [..., 1, C, G, F]
|
|
76
|
+
weight: Tensor, # [..., 1, C, G, D, 2F]
|
|
77
|
+
bias: Tensor, # [..., 1, C, D]
|
|
78
|
+
*,
|
|
79
|
+
missing: Tensor, # [..., R, C, G]
|
|
80
|
+
out: Tensor | None = None,
|
|
81
|
+
) -> Tensor:
|
|
82
|
+
|
|
83
|
+
out = super()._forward(x, freq, weight, bias, out=out)
|
|
84
|
+
out += self.nan_lin(missing.to(out.dtype))
|
|
85
|
+
return out
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
# ruff: noqa: D101, D102
|
|
5
|
+
|
|
6
|
+
from typing import Any, cast
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
from torch import Tensor
|
|
10
|
+
from torch.nn import GELU, Embedding, Linear, ModuleList, Sequential
|
|
11
|
+
|
|
12
|
+
from sdm.cache import Cache, KVCacheEntry
|
|
13
|
+
from sdm.models.kumo.tabular.block import KumoTabularTransformerBlock
|
|
14
|
+
from sdm.nn import LogScale, RMSNorm
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ICLBlock(torch.nn.Module):
|
|
18
|
+
def __init__(
|
|
19
|
+
self,
|
|
20
|
+
num_classes: int,
|
|
21
|
+
out_channels: int,
|
|
22
|
+
channels: int,
|
|
23
|
+
num_layers: int,
|
|
24
|
+
num_heads: int,
|
|
25
|
+
num_key_value_heads_for_query: int | None = None,
|
|
26
|
+
device: torch.device | str | None = None,
|
|
27
|
+
dtype: torch.dtype | None = None,
|
|
28
|
+
) -> None:
|
|
29
|
+
super().__init__()
|
|
30
|
+
factory_kwargs: dict[str, Any] = {"device": device, "dtype": dtype}
|
|
31
|
+
self.kv_heads = num_key_value_heads_for_query
|
|
32
|
+
|
|
33
|
+
self.y_emb: torch.nn.Module | None = None
|
|
34
|
+
self.y_lin: torch.nn.Module | None = None
|
|
35
|
+
if num_classes > 0:
|
|
36
|
+
self.y_emb = Embedding(num_classes, channels, **factory_kwargs)
|
|
37
|
+
else:
|
|
38
|
+
self.y_lin = Linear(1, channels, bias=False, **factory_kwargs)
|
|
39
|
+
|
|
40
|
+
self.layers = ModuleList(
|
|
41
|
+
KumoTabularTransformerBlock(
|
|
42
|
+
channels=channels,
|
|
43
|
+
num_heads=num_heads,
|
|
44
|
+
query_scaling=LogScale(
|
|
45
|
+
num_heads=num_heads,
|
|
46
|
+
**factory_kwargs,
|
|
47
|
+
),
|
|
48
|
+
**factory_kwargs,
|
|
49
|
+
)
|
|
50
|
+
for _ in range(num_layers)
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
self.norm = RMSNorm(channels, **factory_kwargs)
|
|
54
|
+
self.head = Sequential(
|
|
55
|
+
Linear(channels, 2 * channels, **factory_kwargs),
|
|
56
|
+
GELU(),
|
|
57
|
+
Linear(2 * channels, out_channels, **factory_kwargs),
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
def forward(
|
|
61
|
+
self,
|
|
62
|
+
x: Tensor, # [..., R, D]
|
|
63
|
+
y: Tensor, # [..., R_train]
|
|
64
|
+
*,
|
|
65
|
+
cache: Cache | None = None,
|
|
66
|
+
) -> Tensor: # [..., R_test, out_channels]
|
|
67
|
+
R_train = y.size(-1)
|
|
68
|
+
|
|
69
|
+
if y.numel() > 0:
|
|
70
|
+
if self.y_emb is not None:
|
|
71
|
+
y_emb = self.y_emb(y) # [..., R_train, D]
|
|
72
|
+
else:
|
|
73
|
+
assert self.y_lin is not None
|
|
74
|
+
y_emb = self.y_lin(y.unsqueeze(-1)) # [..., R_train, D]
|
|
75
|
+
|
|
76
|
+
x[..., :R_train, :] += y_emb.to(x.dtype)
|
|
77
|
+
del y_emb
|
|
78
|
+
|
|
79
|
+
for i, layer in enumerate(self.layers):
|
|
80
|
+
cache_key = f"icl_block.layer{i}"
|
|
81
|
+
last_layer = i == len(self.layers) - 1
|
|
82
|
+
|
|
83
|
+
if self.kv_heads is None or cache is not None:
|
|
84
|
+
result = layer(
|
|
85
|
+
query=x[..., R_train:, :] if last_layer else x,
|
|
86
|
+
key_value=(
|
|
87
|
+
cast(KVCacheEntry, cache[cache_key])
|
|
88
|
+
if cache is not None and cache.is_replaying
|
|
89
|
+
else x[..., :R_train, :]
|
|
90
|
+
),
|
|
91
|
+
return_key_value=cache is not None and cache.is_recording,
|
|
92
|
+
out=None
|
|
93
|
+
if torch.is_grad_enabled()
|
|
94
|
+
else x[..., R_train:, :]
|
|
95
|
+
if last_layer
|
|
96
|
+
else x,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
if cache is not None and cache.is_recording:
|
|
100
|
+
x, (key, value) = result
|
|
101
|
+
if self.kv_heads is not None:
|
|
102
|
+
key = key[..., : self.kv_heads, :].contiguous()
|
|
103
|
+
value = value[..., : self.kv_heads, :].contiguous()
|
|
104
|
+
cache[cache_key] = KVCacheEntry(key, value)
|
|
105
|
+
del key, value
|
|
106
|
+
else:
|
|
107
|
+
x = result
|
|
108
|
+
del result
|
|
109
|
+
continue
|
|
110
|
+
|
|
111
|
+
x_context, (key, value) = layer(
|
|
112
|
+
query=x[..., :0, :] if last_layer else x[..., :R_train, :],
|
|
113
|
+
key_value=x[..., :R_train, :],
|
|
114
|
+
return_key_value=True,
|
|
115
|
+
out=None
|
|
116
|
+
if torch.is_grad_enabled()
|
|
117
|
+
else x[..., :0, :]
|
|
118
|
+
if last_layer
|
|
119
|
+
else x[..., :R_train, :],
|
|
120
|
+
)
|
|
121
|
+
key_value = KVCacheEntry(
|
|
122
|
+
key=key[..., : self.kv_heads, :].contiguous(),
|
|
123
|
+
value=value[..., : self.kv_heads, :].contiguous(),
|
|
124
|
+
)
|
|
125
|
+
del key, value
|
|
126
|
+
x_query = layer(
|
|
127
|
+
query=x[..., R_train:, :],
|
|
128
|
+
key_value=key_value,
|
|
129
|
+
out=None if torch.is_grad_enabled() else x[..., R_train:, :],
|
|
130
|
+
)
|
|
131
|
+
if last_layer:
|
|
132
|
+
x = x_query
|
|
133
|
+
elif torch.is_grad_enabled():
|
|
134
|
+
x = torch.cat([x_context, x_query], -2)
|
|
135
|
+
|
|
136
|
+
return self.head(self.norm(x))
|