structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/__init__.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Structured data modeling primitives."""
|
|
5
|
+
|
|
6
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
7
|
+
|
|
8
|
+
from sdm._constants import NaT
|
|
9
|
+
from sdm.stype import Stype, StypeLike, infer_stypes
|
|
10
|
+
from sdm.task import Task, TaskLike
|
|
11
|
+
from sdm.tensor import (
|
|
12
|
+
VarLenTensor,
|
|
13
|
+
StringTensor,
|
|
14
|
+
NullableTensor,
|
|
15
|
+
CategoricalTensor,
|
|
16
|
+
ColumnarTensor,
|
|
17
|
+
TableTensor,
|
|
18
|
+
)
|
|
19
|
+
from sdm.ensemble import EnsembleData, EnsembleTable
|
|
20
|
+
from sdm.relational import (
|
|
21
|
+
Relationship,
|
|
22
|
+
RelationalData,
|
|
23
|
+
TaskLink,
|
|
24
|
+
RelatedTables,
|
|
25
|
+
)
|
|
26
|
+
from sdm.processing import Recipe
|
|
27
|
+
from sdm import models, evaluation, explain
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
__version__ = version("structured-data-models")
|
|
31
|
+
except PackageNotFoundError:
|
|
32
|
+
__version__ = "0+unknown"
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"NaT",
|
|
36
|
+
"Stype",
|
|
37
|
+
"StypeLike",
|
|
38
|
+
"Task",
|
|
39
|
+
"TaskLike",
|
|
40
|
+
"infer_stypes",
|
|
41
|
+
"VarLenTensor",
|
|
42
|
+
"StringTensor",
|
|
43
|
+
"NullableTensor",
|
|
44
|
+
"CategoricalTensor",
|
|
45
|
+
"ColumnarTensor",
|
|
46
|
+
"TableTensor",
|
|
47
|
+
"EnsembleData",
|
|
48
|
+
"EnsembleTable",
|
|
49
|
+
"Relationship",
|
|
50
|
+
"RelationalData",
|
|
51
|
+
"TaskLink",
|
|
52
|
+
"RelatedTables",
|
|
53
|
+
"Recipe",
|
|
54
|
+
"models",
|
|
55
|
+
"evaluation",
|
|
56
|
+
"explain",
|
|
57
|
+
"__version__",
|
|
58
|
+
]
|
sdm/_constants.py
ADDED
sdm/_inference.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import contextlib
|
|
5
|
+
from collections.abc import Iterator
|
|
6
|
+
from typing import Literal
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@contextlib.contextmanager
|
|
12
|
+
def inference_mode(
|
|
13
|
+
mode: Literal["inference", "no_grad", "grad", "none"] = "inference",
|
|
14
|
+
) -> Iterator[None]:
|
|
15
|
+
r"""Context manager to adjust PyTorch inference and autograd states.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
mode: The desired mode. ``"inference"`` runs under
|
|
19
|
+
:func:`torch.inference_mode` (falling back to :func:`torch.no_grad`
|
|
20
|
+
inside compiled regions), ``"no_grad"`` runs under
|
|
21
|
+
:func:`torch.no_grad`, ``"grad"`` runs under
|
|
22
|
+
:func:`torch.enable_grad`, and ``"none"`` preserves the ambient
|
|
23
|
+
PyTorch context.
|
|
24
|
+
"""
|
|
25
|
+
if mode == "inference":
|
|
26
|
+
# `torch.inference_mode` is not supported inside a compiled region:
|
|
27
|
+
# https://github.com/pytorch/pytorch/issues/180823
|
|
28
|
+
context = (
|
|
29
|
+
torch.no_grad()
|
|
30
|
+
if torch.compiler.is_compiling()
|
|
31
|
+
else torch.inference_mode()
|
|
32
|
+
)
|
|
33
|
+
elif mode == "no_grad":
|
|
34
|
+
context = torch.no_grad()
|
|
35
|
+
elif mode == "grad":
|
|
36
|
+
context = torch.enable_grad()
|
|
37
|
+
else:
|
|
38
|
+
assert mode == "none"
|
|
39
|
+
context = contextlib.nullcontext()
|
|
40
|
+
|
|
41
|
+
with context:
|
|
42
|
+
yield
|
sdm/_kernels/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from sdm._kernels.segment_multi_reduce import segment_multi_reduce
|
|
5
|
+
from sdm._kernels.rmsnorm_cast import rmsnorm_cast
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"segment_multi_reduce",
|
|
9
|
+
"rmsnorm_cast",
|
|
10
|
+
]
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
from typing import TypeAlias
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
import torch.nn.functional as F
|
|
9
|
+
from torch import Tensor
|
|
10
|
+
|
|
11
|
+
_RMSNormCast: TypeAlias = Callable[
|
|
12
|
+
[Tensor, Tensor | None, float, tuple[Tensor, Tensor] | None], Tensor
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
_triton_rmsnorm_cast: _RMSNormCast | None = None
|
|
16
|
+
try:
|
|
17
|
+
from sdm._kernels.triton.rmsnorm_cast import (
|
|
18
|
+
rmsnorm_cast as _triton_rmsnorm_cast_impl,
|
|
19
|
+
)
|
|
20
|
+
except ImportError:
|
|
21
|
+
pass
|
|
22
|
+
else:
|
|
23
|
+
_triton_rmsnorm_cast = _triton_rmsnorm_cast_impl
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def rmsnorm_cast(
|
|
27
|
+
x: Tensor,
|
|
28
|
+
weight: Tensor | None,
|
|
29
|
+
eps: float,
|
|
30
|
+
rope: tuple[Tensor, Tensor] | None = None,
|
|
31
|
+
) -> Tensor:
|
|
32
|
+
"""Apply RMSNorm over the last dimension and cast to the autocast dtype.
|
|
33
|
+
|
|
34
|
+
The output uses the configured autocast dtype for the input device,
|
|
35
|
+
even when autocast is disabled.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
x: Input tensor with shape ``[..., C]``, where ``C`` is the number
|
|
39
|
+
of channels.
|
|
40
|
+
weight: Scale tensor with shape ``[C]``, or ``None`` for no scaling.
|
|
41
|
+
eps: Constant added to the mean square before normalization.
|
|
42
|
+
rope: Optional cosine and sine tables with shape ``[S, C // 2]``
|
|
43
|
+
for split-half rotary embeddings. The input then has shape
|
|
44
|
+
``[..., S, H, C]``, where ``S`` is the sequence length and ``H``
|
|
45
|
+
is the number of heads. Rotation rounds in the input dtype.
|
|
46
|
+
"""
|
|
47
|
+
if (
|
|
48
|
+
_triton_rmsnorm_cast is not None
|
|
49
|
+
and x.is_cuda
|
|
50
|
+
and x.dtype in {torch.float16, torch.bfloat16, torch.float32}
|
|
51
|
+
and x.size(-1) in {32, 64, 128, 256, 512}
|
|
52
|
+
and (
|
|
53
|
+
weight is None
|
|
54
|
+
or (
|
|
55
|
+
weight.dtype in {torch.float16, torch.bfloat16, torch.float32}
|
|
56
|
+
and weight.device == x.device
|
|
57
|
+
and weight.shape == x.shape[-1:]
|
|
58
|
+
)
|
|
59
|
+
)
|
|
60
|
+
and (
|
|
61
|
+
rope is None
|
|
62
|
+
or (
|
|
63
|
+
x.dim() >= 3
|
|
64
|
+
and all(
|
|
65
|
+
table.shape == (x.size(-3), x.size(-1) // 2)
|
|
66
|
+
and table.dtype == x.dtype
|
|
67
|
+
and table.device == x.device
|
|
68
|
+
for table in rope
|
|
69
|
+
)
|
|
70
|
+
)
|
|
71
|
+
)
|
|
72
|
+
and x.numel() > 0
|
|
73
|
+
and not (
|
|
74
|
+
(
|
|
75
|
+
x.requires_grad
|
|
76
|
+
or (weight is not None and weight.requires_grad)
|
|
77
|
+
or (
|
|
78
|
+
rope is not None
|
|
79
|
+
and any(table.requires_grad for table in rope)
|
|
80
|
+
)
|
|
81
|
+
)
|
|
82
|
+
and torch.is_grad_enabled()
|
|
83
|
+
)
|
|
84
|
+
):
|
|
85
|
+
return _triton_rmsnorm_cast(x, weight, eps, rope)
|
|
86
|
+
if rope is not None:
|
|
87
|
+
cos, sin = (table.unsqueeze(-2) for table in rope)
|
|
88
|
+
x1, x2 = x.chunk(2, dim=-1)
|
|
89
|
+
x = torch.cat((x1 * cos - x2 * sin, x2 * cos + x1 * sin), dim=-1)
|
|
90
|
+
return F.rms_norm(x, (x.size(-1),), weight, eps).to(
|
|
91
|
+
torch.get_autocast_dtype(x.device.type)
|
|
92
|
+
)
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
from typing import TypeAlias
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
from torch import Tensor
|
|
9
|
+
|
|
10
|
+
_SegmentMultiReduce: TypeAlias = Callable[
|
|
11
|
+
[Tensor, Tensor, Tensor, Tensor, Tensor | None],
|
|
12
|
+
Tensor,
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
_triton_segment_multi_reduce: _SegmentMultiReduce | None = None
|
|
16
|
+
try:
|
|
17
|
+
from sdm._kernels.triton.segment_multi_reduce import (
|
|
18
|
+
segment_multi_reduce as _triton_segment_multi_reduce_impl,
|
|
19
|
+
)
|
|
20
|
+
except ImportError:
|
|
21
|
+
pass
|
|
22
|
+
else:
|
|
23
|
+
_triton_segment_multi_reduce = _triton_segment_multi_reduce_impl
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _eager_segment_multi_reduce(
|
|
27
|
+
src: Tensor,
|
|
28
|
+
index: Tensor,
|
|
29
|
+
edge_attr: Tensor,
|
|
30
|
+
offsets: Tensor,
|
|
31
|
+
edge_type: Tensor | None = None,
|
|
32
|
+
) -> Tensor:
|
|
33
|
+
if edge_type is not None:
|
|
34
|
+
edge_attr = edge_attr[edge_type]
|
|
35
|
+
values = src[index]
|
|
36
|
+
if src.dtype in {torch.float16, torch.bfloat16}:
|
|
37
|
+
values = values.float() + edge_attr.float()
|
|
38
|
+
else:
|
|
39
|
+
values = values + edge_attr
|
|
40
|
+
total = torch.segment_reduce(
|
|
41
|
+
values,
|
|
42
|
+
offsets=offsets,
|
|
43
|
+
reduce="sum",
|
|
44
|
+
unsafe=True,
|
|
45
|
+
initial=0,
|
|
46
|
+
)
|
|
47
|
+
mean = total / offsets.diff().clamp(min=1).unsqueeze(1)
|
|
48
|
+
variance = (
|
|
49
|
+
torch.segment_reduce(
|
|
50
|
+
values.square(),
|
|
51
|
+
offsets=offsets,
|
|
52
|
+
reduce="mean",
|
|
53
|
+
unsafe=True,
|
|
54
|
+
initial=0,
|
|
55
|
+
)
|
|
56
|
+
- mean.square()
|
|
57
|
+
)
|
|
58
|
+
std = torch.where(
|
|
59
|
+
variance <= 1e-5,
|
|
60
|
+
0.0,
|
|
61
|
+
variance.clamp(min=1e-5).sqrt(),
|
|
62
|
+
)
|
|
63
|
+
minimum = torch.segment_reduce(
|
|
64
|
+
values,
|
|
65
|
+
offsets=offsets,
|
|
66
|
+
reduce="min",
|
|
67
|
+
unsafe=True,
|
|
68
|
+
)
|
|
69
|
+
maximum = torch.segment_reduce(
|
|
70
|
+
values,
|
|
71
|
+
offsets=offsets,
|
|
72
|
+
reduce="max",
|
|
73
|
+
unsafe=True,
|
|
74
|
+
)
|
|
75
|
+
minimum = torch.where(minimum.isinf(), 0.0, minimum)
|
|
76
|
+
maximum = torch.where(maximum.isinf(), 0.0, maximum)
|
|
77
|
+
return torch.stack(
|
|
78
|
+
(
|
|
79
|
+
total,
|
|
80
|
+
mean,
|
|
81
|
+
std,
|
|
82
|
+
minimum,
|
|
83
|
+
maximum,
|
|
84
|
+
),
|
|
85
|
+
dim=1,
|
|
86
|
+
).to(src.dtype)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def segment_multi_reduce(
|
|
90
|
+
src: Tensor,
|
|
91
|
+
index: Tensor,
|
|
92
|
+
edge_attr: Tensor,
|
|
93
|
+
offsets: Tensor,
|
|
94
|
+
edge_type: Tensor | None = None,
|
|
95
|
+
) -> Tensor:
|
|
96
|
+
edge_attr_matches = (
|
|
97
|
+
edge_attr.shape == (index.numel(), src.size(1))
|
|
98
|
+
if edge_type is None
|
|
99
|
+
else (
|
|
100
|
+
edge_attr.dim() == 2
|
|
101
|
+
and edge_attr.size(1) == src.size(1)
|
|
102
|
+
and edge_type.numel() == index.numel()
|
|
103
|
+
)
|
|
104
|
+
)
|
|
105
|
+
if (
|
|
106
|
+
_triton_segment_multi_reduce is not None
|
|
107
|
+
and not (
|
|
108
|
+
(src.requires_grad or edge_attr.requires_grad)
|
|
109
|
+
and torch.is_grad_enabled()
|
|
110
|
+
)
|
|
111
|
+
and src.is_cuda
|
|
112
|
+
and src.dtype in {torch.float16, torch.bfloat16, torch.float32}
|
|
113
|
+
and edge_attr.dtype == src.dtype
|
|
114
|
+
and (
|
|
115
|
+
edge_type is None
|
|
116
|
+
or (
|
|
117
|
+
edge_type.device == src.device
|
|
118
|
+
and edge_type.dtype in {torch.int32, torch.int64}
|
|
119
|
+
)
|
|
120
|
+
)
|
|
121
|
+
and index.device == src.device
|
|
122
|
+
and edge_attr.device == src.device
|
|
123
|
+
and offsets.device == src.device
|
|
124
|
+
and index.dtype in {torch.int32, torch.int64}
|
|
125
|
+
and offsets.dtype in {torch.int32, torch.int64}
|
|
126
|
+
and edge_attr_matches
|
|
127
|
+
):
|
|
128
|
+
return _triton_segment_multi_reduce(
|
|
129
|
+
src,
|
|
130
|
+
index,
|
|
131
|
+
edge_attr,
|
|
132
|
+
offsets,
|
|
133
|
+
edge_type,
|
|
134
|
+
)
|
|
135
|
+
return _eager_segment_multi_reduce(
|
|
136
|
+
src,
|
|
137
|
+
index,
|
|
138
|
+
edge_attr,
|
|
139
|
+
offsets,
|
|
140
|
+
edge_type,
|
|
141
|
+
)
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from typing import Any, cast
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
import triton
|
|
8
|
+
import triton.language as tl
|
|
9
|
+
from torch import Tensor
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@triton.jit
|
|
13
|
+
def _load_input(
|
|
14
|
+
x_ptr,
|
|
15
|
+
cos_ptr,
|
|
16
|
+
sin_ptr,
|
|
17
|
+
row,
|
|
18
|
+
c,
|
|
19
|
+
mask,
|
|
20
|
+
channels: tl.constexpr,
|
|
21
|
+
seq_len,
|
|
22
|
+
num_heads,
|
|
23
|
+
) -> tl.tensor:
|
|
24
|
+
offsets = row[:, None] * channels
|
|
25
|
+
value = tl.load(x_ptr + offsets + c, mask=mask, other=0).to(tl.float32)
|
|
26
|
+
if cos_ptr is not None:
|
|
27
|
+
first = c < channels // 2
|
|
28
|
+
partner = tl.where(first, c + channels // 2, c - channels // 2)
|
|
29
|
+
other = tl.load(x_ptr + offsets + partner, mask=mask, other=0).to(
|
|
30
|
+
tl.float32
|
|
31
|
+
)
|
|
32
|
+
table = (row // num_heads % seq_len)[:, None] * (channels // 2)
|
|
33
|
+
table += c % (channels // 2)
|
|
34
|
+
cos = tl.load(cos_ptr + table, mask=mask, other=0).to(tl.float32)
|
|
35
|
+
sin = tl.load(sin_ptr + table, mask=mask, other=0).to(tl.float32)
|
|
36
|
+
# Eager RoPE rounds both products and their sum in the input dtype.
|
|
37
|
+
dtype = x_ptr.dtype.element_ty
|
|
38
|
+
a = (value * cos).to(dtype).to(tl.float32)
|
|
39
|
+
b = (other * sin).to(dtype).to(tl.float32)
|
|
40
|
+
value = tl.where(first, a - b, a + b).to(dtype).to(tl.float32)
|
|
41
|
+
return value
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@triton.jit(do_not_specialize=["rows", "seq_len", "num_heads"])
|
|
45
|
+
def _rmsnorm_cast_kernel(
|
|
46
|
+
x_ptr,
|
|
47
|
+
weight_ptr,
|
|
48
|
+
cos_ptr,
|
|
49
|
+
sin_ptr,
|
|
50
|
+
out_ptr,
|
|
51
|
+
rows,
|
|
52
|
+
seq_len,
|
|
53
|
+
num_heads,
|
|
54
|
+
channels: tl.constexpr,
|
|
55
|
+
eps: tl.constexpr,
|
|
56
|
+
block_rows: tl.constexpr,
|
|
57
|
+
) -> None:
|
|
58
|
+
row = tl.program_id(0).to(tl.int64) * block_rows + tl.arange(0, block_rows)
|
|
59
|
+
offsets = row[:, None] * channels
|
|
60
|
+
mask = row[:, None] < rows
|
|
61
|
+
groups = tl.arange(0, channels // 4)[None, :]
|
|
62
|
+
total = tl.full((block_rows, channels // 4), 0, tl.float32)
|
|
63
|
+
for i in tl.static_range(4):
|
|
64
|
+
v = _load_input( # ty: ignore[invalid-argument-type]
|
|
65
|
+
x_ptr,
|
|
66
|
+
cos_ptr,
|
|
67
|
+
sin_ptr,
|
|
68
|
+
row,
|
|
69
|
+
groups * 4 + i,
|
|
70
|
+
mask,
|
|
71
|
+
channels,
|
|
72
|
+
seq_len,
|
|
73
|
+
num_heads,
|
|
74
|
+
)
|
|
75
|
+
total = tl.fma(v, v, total)
|
|
76
|
+
if channels == 32:
|
|
77
|
+
# Preserve CUDA's shuffle order when Triton packs adjacent groups.
|
|
78
|
+
for shift in tl.static_range(2, -1, -1):
|
|
79
|
+
indices = tl.broadcast_to(groups ^ (1 << shift), (block_rows, 8))
|
|
80
|
+
total += tl.gather(total, indices, axis=1)
|
|
81
|
+
total = tl.where(groups == 0, total, 0)
|
|
82
|
+
variance = tl.sum(total, 1) # ty: ignore[invalid-argument-type]
|
|
83
|
+
c = tl.arange(0, channels)[None, :]
|
|
84
|
+
value = _load_input( # ty: ignore[invalid-argument-type]
|
|
85
|
+
x_ptr,
|
|
86
|
+
cos_ptr,
|
|
87
|
+
sin_ptr,
|
|
88
|
+
row,
|
|
89
|
+
c,
|
|
90
|
+
mask,
|
|
91
|
+
channels,
|
|
92
|
+
seq_len,
|
|
93
|
+
num_heads,
|
|
94
|
+
)
|
|
95
|
+
value *= tl.rsqrt(variance[:, None] / channels + eps)
|
|
96
|
+
if weight_ptr is not None:
|
|
97
|
+
value *= tl.load(weight_ptr + c).to(tl.float32)
|
|
98
|
+
tl.store(out_ptr + offsets + c, value, mask=mask)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def rmsnorm_cast(
|
|
102
|
+
x: Tensor,
|
|
103
|
+
weight: Tensor | None,
|
|
104
|
+
eps: float,
|
|
105
|
+
rope: tuple[Tensor, Tensor] | None = None,
|
|
106
|
+
) -> Tensor:
|
|
107
|
+
x = x.contiguous()
|
|
108
|
+
if weight is not None:
|
|
109
|
+
weight = weight.contiguous()
|
|
110
|
+
cos, sin = (
|
|
111
|
+
(None, None)
|
|
112
|
+
if rope is None
|
|
113
|
+
else (table.contiguous() for table in rope)
|
|
114
|
+
)
|
|
115
|
+
channels = x.size(-1)
|
|
116
|
+
rows = x.numel() // channels
|
|
117
|
+
block_rows = 8
|
|
118
|
+
grid = ((rows + block_rows - 1) // block_rows,)
|
|
119
|
+
out = torch.empty_like(x, dtype=torch.get_autocast_dtype("cuda"))
|
|
120
|
+
with torch.cuda.device(x.device):
|
|
121
|
+
cast(Any, _rmsnorm_cast_kernel)[grid](
|
|
122
|
+
x,
|
|
123
|
+
weight,
|
|
124
|
+
cos,
|
|
125
|
+
sin,
|
|
126
|
+
out,
|
|
127
|
+
rows,
|
|
128
|
+
x.size(-3) if rope is not None else 1,
|
|
129
|
+
x.size(-2) if rope is not None else 1,
|
|
130
|
+
channels,
|
|
131
|
+
eps,
|
|
132
|
+
block_rows,
|
|
133
|
+
num_warps=4,
|
|
134
|
+
enable_fp_fusion=False,
|
|
135
|
+
)
|
|
136
|
+
return out
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from typing import Any, cast
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
import triton
|
|
8
|
+
import triton.language as tl
|
|
9
|
+
from torch import Tensor
|
|
10
|
+
|
|
11
|
+
_NUM_STATISTICS = 5
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@triton.jit
|
|
15
|
+
def _segment_multi_reduce_kernel(
|
|
16
|
+
src_ptr,
|
|
17
|
+
index_ptr,
|
|
18
|
+
edge_attr_ptr,
|
|
19
|
+
edge_type_ptr,
|
|
20
|
+
offsets_ptr,
|
|
21
|
+
output_ptr,
|
|
22
|
+
num_channels: tl.constexpr,
|
|
23
|
+
num_statistics: tl.constexpr,
|
|
24
|
+
block_channels: tl.constexpr,
|
|
25
|
+
has_edge_type: tl.constexpr,
|
|
26
|
+
) -> None:
|
|
27
|
+
segment = tl.program_id(0)
|
|
28
|
+
channels = tl.program_id(1) * block_channels + tl.arange(0, block_channels)
|
|
29
|
+
channel_mask = channels < num_channels
|
|
30
|
+
start = tl.load(offsets_ptr + segment)
|
|
31
|
+
end = tl.load(offsets_ptr + segment + 1)
|
|
32
|
+
|
|
33
|
+
total = tl.zeros((block_channels,), tl.float32) # ty: ignore[invalid-argument-type]
|
|
34
|
+
square_total = tl.zeros((block_channels,), tl.float32) # ty: ignore[invalid-argument-type]
|
|
35
|
+
minimum = tl.full((block_channels,), float("inf"), tl.float32)
|
|
36
|
+
maximum = tl.full((block_channels,), -float("inf"), tl.float32)
|
|
37
|
+
has_nan = tl.zeros((block_channels,), tl.int1) # ty: ignore[invalid-argument-type]
|
|
38
|
+
|
|
39
|
+
edge = start
|
|
40
|
+
while edge < end:
|
|
41
|
+
row = tl.load(index_ptr + edge)
|
|
42
|
+
value = tl.load(
|
|
43
|
+
src_ptr + row.to(tl.int64) * num_channels + channels,
|
|
44
|
+
mask=channel_mask,
|
|
45
|
+
other=0.0,
|
|
46
|
+
).to(tl.float32)
|
|
47
|
+
if has_edge_type:
|
|
48
|
+
edge_attr_row = tl.load(edge_type_ptr + edge)
|
|
49
|
+
else:
|
|
50
|
+
edge_attr_row = edge
|
|
51
|
+
value += tl.load(
|
|
52
|
+
edge_attr_ptr
|
|
53
|
+
+ edge_attr_row.to(tl.int64) * num_channels
|
|
54
|
+
+ channels,
|
|
55
|
+
mask=channel_mask,
|
|
56
|
+
other=0.0,
|
|
57
|
+
).to(tl.float32)
|
|
58
|
+
total += value
|
|
59
|
+
square_total += value * value
|
|
60
|
+
minimum = tl.minimum(minimum, value)
|
|
61
|
+
maximum = tl.maximum(maximum, value)
|
|
62
|
+
has_nan |= value != value
|
|
63
|
+
edge += 1
|
|
64
|
+
|
|
65
|
+
count = end - start
|
|
66
|
+
divisor = tl.maximum(count, 1)
|
|
67
|
+
mean = total / divisor
|
|
68
|
+
variance = square_total / divisor - mean * mean
|
|
69
|
+
clamped_variance = tl.where(variance < 1e-5, 1e-5, variance)
|
|
70
|
+
std = tl.where(
|
|
71
|
+
variance <= 1e-5,
|
|
72
|
+
0.0,
|
|
73
|
+
tl.sqrt(clamped_variance),
|
|
74
|
+
)
|
|
75
|
+
minimum_is_inf = (minimum == float("inf")) | (minimum == -float("inf"))
|
|
76
|
+
maximum_is_inf = (maximum == float("inf")) | (maximum == -float("inf"))
|
|
77
|
+
minimum = tl.where(minimum_is_inf, 0.0, minimum)
|
|
78
|
+
maximum = tl.where(maximum_is_inf, 0.0, maximum)
|
|
79
|
+
minimum = tl.where(has_nan, float("nan"), minimum)
|
|
80
|
+
maximum = tl.where(has_nan, float("nan"), maximum)
|
|
81
|
+
|
|
82
|
+
output_offsets = (
|
|
83
|
+
segment.to(tl.int64) * num_statistics * num_channels + channels
|
|
84
|
+
)
|
|
85
|
+
tl.store(output_ptr + output_offsets, total, mask=channel_mask)
|
|
86
|
+
tl.store(
|
|
87
|
+
output_ptr + output_offsets + num_channels,
|
|
88
|
+
mean,
|
|
89
|
+
mask=channel_mask,
|
|
90
|
+
)
|
|
91
|
+
tl.store(
|
|
92
|
+
output_ptr + output_offsets + 2 * num_channels,
|
|
93
|
+
std,
|
|
94
|
+
mask=channel_mask,
|
|
95
|
+
)
|
|
96
|
+
tl.store(
|
|
97
|
+
output_ptr + output_offsets + 3 * num_channels,
|
|
98
|
+
minimum,
|
|
99
|
+
mask=channel_mask,
|
|
100
|
+
)
|
|
101
|
+
tl.store(
|
|
102
|
+
output_ptr + output_offsets + 4 * num_channels,
|
|
103
|
+
maximum,
|
|
104
|
+
mask=channel_mask,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def segment_multi_reduce(
|
|
109
|
+
src: Tensor,
|
|
110
|
+
index: Tensor,
|
|
111
|
+
edge_attr: Tensor,
|
|
112
|
+
offsets: Tensor,
|
|
113
|
+
edge_type: Tensor | None = None,
|
|
114
|
+
) -> Tensor:
|
|
115
|
+
src = src.contiguous()
|
|
116
|
+
index = index.contiguous()
|
|
117
|
+
edge_attr = edge_attr.contiguous()
|
|
118
|
+
offsets = offsets.contiguous()
|
|
119
|
+
if edge_type is not None:
|
|
120
|
+
edge_type = edge_type.contiguous()
|
|
121
|
+
if edge_type is None and edge_attr.shape != (
|
|
122
|
+
index.numel(),
|
|
123
|
+
src.size(1),
|
|
124
|
+
):
|
|
125
|
+
raise ValueError(
|
|
126
|
+
"edge_attr must have shape (index.numel(), src.size(1))"
|
|
127
|
+
)
|
|
128
|
+
if edge_type is not None and (
|
|
129
|
+
edge_attr.dim() != 2
|
|
130
|
+
or edge_attr.size(1) != src.size(1)
|
|
131
|
+
or edge_type.numel() != index.numel()
|
|
132
|
+
):
|
|
133
|
+
raise ValueError(
|
|
134
|
+
"edge_type must match index and edge_attr must have "
|
|
135
|
+
"src.size(1) columns"
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
shape = (offsets.numel() - 1, _NUM_STATISTICS, src.size(1))
|
|
139
|
+
output = src.new_empty(shape)
|
|
140
|
+
if shape[0] == 0 or shape[2] == 0:
|
|
141
|
+
return output
|
|
142
|
+
|
|
143
|
+
block_channels = min(
|
|
144
|
+
triton.next_power_of_2(shape[2]), # ty: ignore[invalid-argument-type]
|
|
145
|
+
512,
|
|
146
|
+
)
|
|
147
|
+
if block_channels >= 256:
|
|
148
|
+
num_warps = 8
|
|
149
|
+
elif block_channels >= 128:
|
|
150
|
+
num_warps = 4
|
|
151
|
+
else:
|
|
152
|
+
num_warps = 1
|
|
153
|
+
grid = (
|
|
154
|
+
shape[0],
|
|
155
|
+
triton.cdiv( # ty: ignore[invalid-argument-type]
|
|
156
|
+
shape[2], block_channels
|
|
157
|
+
),
|
|
158
|
+
)
|
|
159
|
+
with torch.cuda.device(src.device):
|
|
160
|
+
cast(Any, _segment_multi_reduce_kernel)[grid](
|
|
161
|
+
src,
|
|
162
|
+
index,
|
|
163
|
+
edge_attr,
|
|
164
|
+
edge_type if edge_type is not None else index,
|
|
165
|
+
offsets,
|
|
166
|
+
output,
|
|
167
|
+
num_channels=shape[2],
|
|
168
|
+
num_statistics=shape[1],
|
|
169
|
+
block_channels=block_channels,
|
|
170
|
+
num_warps=num_warps,
|
|
171
|
+
has_edge_type=edge_type is not None,
|
|
172
|
+
)
|
|
173
|
+
return output
|
sdm/_memory.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def chunk_memory_limit(device: torch.device) -> int:
|
|
10
|
+
r"""Bytes one chunk of a chunked operation may occupy on a CUDA device.
|
|
11
|
+
|
|
12
|
+
The limit is the ``SDM_CHUNK_MEMORY_FRACTION`` (default ``0.05``) share of
|
|
13
|
+
the device memory available to this process.
|
|
14
|
+
"""
|
|
15
|
+
return int(
|
|
16
|
+
torch.cuda.get_device_properties(device).total_memory
|
|
17
|
+
* torch.cuda.get_per_process_memory_fraction(device)
|
|
18
|
+
* float(os.getenv("SDM_CHUNK_MEMORY_FRACTION", "0.05"))
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def split_size(num_items: int, item_bytes: int, device: torch.device) -> int:
|
|
23
|
+
r"""Split size of balanced chunks of items of ``item_bytes`` bytes each.
|
|
24
|
+
|
|
25
|
+
On CUDA devices, chunks fit :func:`chunk_memory_limit`, unless a single
|
|
26
|
+
item exceeds the limit. Autograd keeps the memory of every chunk, so one
|
|
27
|
+
chunk holds all items elsewhere or while gradients are enabled.
|
|
28
|
+
"""
|
|
29
|
+
if device.type != "cuda" or torch.is_grad_enabled() or item_bytes == 0:
|
|
30
|
+
return max(num_items, 1)
|
|
31
|
+
limit = max(chunk_memory_limit(device), 1)
|
|
32
|
+
capacity = max(limit // item_bytes, 1)
|
|
33
|
+
num_chunks = max(-(-num_items // capacity), 1)
|
|
34
|
+
return max(-(-num_items // num_chunks), 1)
|