structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/__init__.py ADDED
@@ -0,0 +1,58 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Structured data modeling primitives."""
5
+
6
+ from importlib.metadata import PackageNotFoundError, version
7
+
8
+ from sdm._constants import NaT
9
+ from sdm.stype import Stype, StypeLike, infer_stypes
10
+ from sdm.task import Task, TaskLike
11
+ from sdm.tensor import (
12
+ VarLenTensor,
13
+ StringTensor,
14
+ NullableTensor,
15
+ CategoricalTensor,
16
+ ColumnarTensor,
17
+ TableTensor,
18
+ )
19
+ from sdm.ensemble import EnsembleData, EnsembleTable
20
+ from sdm.relational import (
21
+ Relationship,
22
+ RelationalData,
23
+ TaskLink,
24
+ RelatedTables,
25
+ )
26
+ from sdm.processing import Recipe
27
+ from sdm import models, evaluation, explain
28
+
29
+ try:
30
+ __version__ = version("structured-data-models")
31
+ except PackageNotFoundError:
32
+ __version__ = "0+unknown"
33
+
34
+ __all__ = [
35
+ "NaT",
36
+ "Stype",
37
+ "StypeLike",
38
+ "Task",
39
+ "TaskLike",
40
+ "infer_stypes",
41
+ "VarLenTensor",
42
+ "StringTensor",
43
+ "NullableTensor",
44
+ "CategoricalTensor",
45
+ "ColumnarTensor",
46
+ "TableTensor",
47
+ "EnsembleData",
48
+ "EnsembleTable",
49
+ "Relationship",
50
+ "RelationalData",
51
+ "TaskLink",
52
+ "RelatedTables",
53
+ "Recipe",
54
+ "models",
55
+ "evaluation",
56
+ "explain",
57
+ "__version__",
58
+ ]
sdm/_constants.py ADDED
@@ -0,0 +1,6 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from typing import Final
5
+
6
+ NaT: Final[int] = -9223372036854775808 # torch.iinfo(torch.int64).min
sdm/_inference.py ADDED
@@ -0,0 +1,42 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import contextlib
5
+ from collections.abc import Iterator
6
+ from typing import Literal
7
+
8
+ import torch
9
+
10
+
11
+ @contextlib.contextmanager
12
+ def inference_mode(
13
+ mode: Literal["inference", "no_grad", "grad", "none"] = "inference",
14
+ ) -> Iterator[None]:
15
+ r"""Context manager to adjust PyTorch inference and autograd states.
16
+
17
+ Args:
18
+ mode: The desired mode. ``"inference"`` runs under
19
+ :func:`torch.inference_mode` (falling back to :func:`torch.no_grad`
20
+ inside compiled regions), ``"no_grad"`` runs under
21
+ :func:`torch.no_grad`, ``"grad"`` runs under
22
+ :func:`torch.enable_grad`, and ``"none"`` preserves the ambient
23
+ PyTorch context.
24
+ """
25
+ if mode == "inference":
26
+ # `torch.inference_mode` is not supported inside a compiled region:
27
+ # https://github.com/pytorch/pytorch/issues/180823
28
+ context = (
29
+ torch.no_grad()
30
+ if torch.compiler.is_compiling()
31
+ else torch.inference_mode()
32
+ )
33
+ elif mode == "no_grad":
34
+ context = torch.no_grad()
35
+ elif mode == "grad":
36
+ context = torch.enable_grad()
37
+ else:
38
+ assert mode == "none"
39
+ context = contextlib.nullcontext()
40
+
41
+ with context:
42
+ yield
@@ -0,0 +1,10 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from sdm._kernels.segment_multi_reduce import segment_multi_reduce
5
+ from sdm._kernels.rmsnorm_cast import rmsnorm_cast
6
+
7
+ __all__ = [
8
+ "segment_multi_reduce",
9
+ "rmsnorm_cast",
10
+ ]
@@ -0,0 +1,92 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from collections.abc import Callable
5
+ from typing import TypeAlias
6
+
7
+ import torch
8
+ import torch.nn.functional as F
9
+ from torch import Tensor
10
+
11
+ _RMSNormCast: TypeAlias = Callable[
12
+ [Tensor, Tensor | None, float, tuple[Tensor, Tensor] | None], Tensor
13
+ ]
14
+
15
+ _triton_rmsnorm_cast: _RMSNormCast | None = None
16
+ try:
17
+ from sdm._kernels.triton.rmsnorm_cast import (
18
+ rmsnorm_cast as _triton_rmsnorm_cast_impl,
19
+ )
20
+ except ImportError:
21
+ pass
22
+ else:
23
+ _triton_rmsnorm_cast = _triton_rmsnorm_cast_impl
24
+
25
+
26
+ def rmsnorm_cast(
27
+ x: Tensor,
28
+ weight: Tensor | None,
29
+ eps: float,
30
+ rope: tuple[Tensor, Tensor] | None = None,
31
+ ) -> Tensor:
32
+ """Apply RMSNorm over the last dimension and cast to the autocast dtype.
33
+
34
+ The output uses the configured autocast dtype for the input device,
35
+ even when autocast is disabled.
36
+
37
+ Args:
38
+ x: Input tensor with shape ``[..., C]``, where ``C`` is the number
39
+ of channels.
40
+ weight: Scale tensor with shape ``[C]``, or ``None`` for no scaling.
41
+ eps: Constant added to the mean square before normalization.
42
+ rope: Optional cosine and sine tables with shape ``[S, C // 2]``
43
+ for split-half rotary embeddings. The input then has shape
44
+ ``[..., S, H, C]``, where ``S`` is the sequence length and ``H``
45
+ is the number of heads. Rotation rounds in the input dtype.
46
+ """
47
+ if (
48
+ _triton_rmsnorm_cast is not None
49
+ and x.is_cuda
50
+ and x.dtype in {torch.float16, torch.bfloat16, torch.float32}
51
+ and x.size(-1) in {32, 64, 128, 256, 512}
52
+ and (
53
+ weight is None
54
+ or (
55
+ weight.dtype in {torch.float16, torch.bfloat16, torch.float32}
56
+ and weight.device == x.device
57
+ and weight.shape == x.shape[-1:]
58
+ )
59
+ )
60
+ and (
61
+ rope is None
62
+ or (
63
+ x.dim() >= 3
64
+ and all(
65
+ table.shape == (x.size(-3), x.size(-1) // 2)
66
+ and table.dtype == x.dtype
67
+ and table.device == x.device
68
+ for table in rope
69
+ )
70
+ )
71
+ )
72
+ and x.numel() > 0
73
+ and not (
74
+ (
75
+ x.requires_grad
76
+ or (weight is not None and weight.requires_grad)
77
+ or (
78
+ rope is not None
79
+ and any(table.requires_grad for table in rope)
80
+ )
81
+ )
82
+ and torch.is_grad_enabled()
83
+ )
84
+ ):
85
+ return _triton_rmsnorm_cast(x, weight, eps, rope)
86
+ if rope is not None:
87
+ cos, sin = (table.unsqueeze(-2) for table in rope)
88
+ x1, x2 = x.chunk(2, dim=-1)
89
+ x = torch.cat((x1 * cos - x2 * sin, x2 * cos + x1 * sin), dim=-1)
90
+ return F.rms_norm(x, (x.size(-1),), weight, eps).to(
91
+ torch.get_autocast_dtype(x.device.type)
92
+ )
@@ -0,0 +1,141 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from collections.abc import Callable
5
+ from typing import TypeAlias
6
+
7
+ import torch
8
+ from torch import Tensor
9
+
10
+ _SegmentMultiReduce: TypeAlias = Callable[
11
+ [Tensor, Tensor, Tensor, Tensor, Tensor | None],
12
+ Tensor,
13
+ ]
14
+
15
+ _triton_segment_multi_reduce: _SegmentMultiReduce | None = None
16
+ try:
17
+ from sdm._kernels.triton.segment_multi_reduce import (
18
+ segment_multi_reduce as _triton_segment_multi_reduce_impl,
19
+ )
20
+ except ImportError:
21
+ pass
22
+ else:
23
+ _triton_segment_multi_reduce = _triton_segment_multi_reduce_impl
24
+
25
+
26
+ def _eager_segment_multi_reduce(
27
+ src: Tensor,
28
+ index: Tensor,
29
+ edge_attr: Tensor,
30
+ offsets: Tensor,
31
+ edge_type: Tensor | None = None,
32
+ ) -> Tensor:
33
+ if edge_type is not None:
34
+ edge_attr = edge_attr[edge_type]
35
+ values = src[index]
36
+ if src.dtype in {torch.float16, torch.bfloat16}:
37
+ values = values.float() + edge_attr.float()
38
+ else:
39
+ values = values + edge_attr
40
+ total = torch.segment_reduce(
41
+ values,
42
+ offsets=offsets,
43
+ reduce="sum",
44
+ unsafe=True,
45
+ initial=0,
46
+ )
47
+ mean = total / offsets.diff().clamp(min=1).unsqueeze(1)
48
+ variance = (
49
+ torch.segment_reduce(
50
+ values.square(),
51
+ offsets=offsets,
52
+ reduce="mean",
53
+ unsafe=True,
54
+ initial=0,
55
+ )
56
+ - mean.square()
57
+ )
58
+ std = torch.where(
59
+ variance <= 1e-5,
60
+ 0.0,
61
+ variance.clamp(min=1e-5).sqrt(),
62
+ )
63
+ minimum = torch.segment_reduce(
64
+ values,
65
+ offsets=offsets,
66
+ reduce="min",
67
+ unsafe=True,
68
+ )
69
+ maximum = torch.segment_reduce(
70
+ values,
71
+ offsets=offsets,
72
+ reduce="max",
73
+ unsafe=True,
74
+ )
75
+ minimum = torch.where(minimum.isinf(), 0.0, minimum)
76
+ maximum = torch.where(maximum.isinf(), 0.0, maximum)
77
+ return torch.stack(
78
+ (
79
+ total,
80
+ mean,
81
+ std,
82
+ minimum,
83
+ maximum,
84
+ ),
85
+ dim=1,
86
+ ).to(src.dtype)
87
+
88
+
89
+ def segment_multi_reduce(
90
+ src: Tensor,
91
+ index: Tensor,
92
+ edge_attr: Tensor,
93
+ offsets: Tensor,
94
+ edge_type: Tensor | None = None,
95
+ ) -> Tensor:
96
+ edge_attr_matches = (
97
+ edge_attr.shape == (index.numel(), src.size(1))
98
+ if edge_type is None
99
+ else (
100
+ edge_attr.dim() == 2
101
+ and edge_attr.size(1) == src.size(1)
102
+ and edge_type.numel() == index.numel()
103
+ )
104
+ )
105
+ if (
106
+ _triton_segment_multi_reduce is not None
107
+ and not (
108
+ (src.requires_grad or edge_attr.requires_grad)
109
+ and torch.is_grad_enabled()
110
+ )
111
+ and src.is_cuda
112
+ and src.dtype in {torch.float16, torch.bfloat16, torch.float32}
113
+ and edge_attr.dtype == src.dtype
114
+ and (
115
+ edge_type is None
116
+ or (
117
+ edge_type.device == src.device
118
+ and edge_type.dtype in {torch.int32, torch.int64}
119
+ )
120
+ )
121
+ and index.device == src.device
122
+ and edge_attr.device == src.device
123
+ and offsets.device == src.device
124
+ and index.dtype in {torch.int32, torch.int64}
125
+ and offsets.dtype in {torch.int32, torch.int64}
126
+ and edge_attr_matches
127
+ ):
128
+ return _triton_segment_multi_reduce(
129
+ src,
130
+ index,
131
+ edge_attr,
132
+ offsets,
133
+ edge_type,
134
+ )
135
+ return _eager_segment_multi_reduce(
136
+ src,
137
+ index,
138
+ edge_attr,
139
+ offsets,
140
+ edge_type,
141
+ )
@@ -0,0 +1,7 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import sys
5
+
6
+ if sys.platform != "linux":
7
+ raise ImportError("Triton kernels are only available on Linux")
@@ -0,0 +1,136 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from typing import Any, cast
5
+
6
+ import torch
7
+ import triton
8
+ import triton.language as tl
9
+ from torch import Tensor
10
+
11
+
12
+ @triton.jit
13
+ def _load_input(
14
+ x_ptr,
15
+ cos_ptr,
16
+ sin_ptr,
17
+ row,
18
+ c,
19
+ mask,
20
+ channels: tl.constexpr,
21
+ seq_len,
22
+ num_heads,
23
+ ) -> tl.tensor:
24
+ offsets = row[:, None] * channels
25
+ value = tl.load(x_ptr + offsets + c, mask=mask, other=0).to(tl.float32)
26
+ if cos_ptr is not None:
27
+ first = c < channels // 2
28
+ partner = tl.where(first, c + channels // 2, c - channels // 2)
29
+ other = tl.load(x_ptr + offsets + partner, mask=mask, other=0).to(
30
+ tl.float32
31
+ )
32
+ table = (row // num_heads % seq_len)[:, None] * (channels // 2)
33
+ table += c % (channels // 2)
34
+ cos = tl.load(cos_ptr + table, mask=mask, other=0).to(tl.float32)
35
+ sin = tl.load(sin_ptr + table, mask=mask, other=0).to(tl.float32)
36
+ # Eager RoPE rounds both products and their sum in the input dtype.
37
+ dtype = x_ptr.dtype.element_ty
38
+ a = (value * cos).to(dtype).to(tl.float32)
39
+ b = (other * sin).to(dtype).to(tl.float32)
40
+ value = tl.where(first, a - b, a + b).to(dtype).to(tl.float32)
41
+ return value
42
+
43
+
44
+ @triton.jit(do_not_specialize=["rows", "seq_len", "num_heads"])
45
+ def _rmsnorm_cast_kernel(
46
+ x_ptr,
47
+ weight_ptr,
48
+ cos_ptr,
49
+ sin_ptr,
50
+ out_ptr,
51
+ rows,
52
+ seq_len,
53
+ num_heads,
54
+ channels: tl.constexpr,
55
+ eps: tl.constexpr,
56
+ block_rows: tl.constexpr,
57
+ ) -> None:
58
+ row = tl.program_id(0).to(tl.int64) * block_rows + tl.arange(0, block_rows)
59
+ offsets = row[:, None] * channels
60
+ mask = row[:, None] < rows
61
+ groups = tl.arange(0, channels // 4)[None, :]
62
+ total = tl.full((block_rows, channels // 4), 0, tl.float32)
63
+ for i in tl.static_range(4):
64
+ v = _load_input( # ty: ignore[invalid-argument-type]
65
+ x_ptr,
66
+ cos_ptr,
67
+ sin_ptr,
68
+ row,
69
+ groups * 4 + i,
70
+ mask,
71
+ channels,
72
+ seq_len,
73
+ num_heads,
74
+ )
75
+ total = tl.fma(v, v, total)
76
+ if channels == 32:
77
+ # Preserve CUDA's shuffle order when Triton packs adjacent groups.
78
+ for shift in tl.static_range(2, -1, -1):
79
+ indices = tl.broadcast_to(groups ^ (1 << shift), (block_rows, 8))
80
+ total += tl.gather(total, indices, axis=1)
81
+ total = tl.where(groups == 0, total, 0)
82
+ variance = tl.sum(total, 1) # ty: ignore[invalid-argument-type]
83
+ c = tl.arange(0, channels)[None, :]
84
+ value = _load_input( # ty: ignore[invalid-argument-type]
85
+ x_ptr,
86
+ cos_ptr,
87
+ sin_ptr,
88
+ row,
89
+ c,
90
+ mask,
91
+ channels,
92
+ seq_len,
93
+ num_heads,
94
+ )
95
+ value *= tl.rsqrt(variance[:, None] / channels + eps)
96
+ if weight_ptr is not None:
97
+ value *= tl.load(weight_ptr + c).to(tl.float32)
98
+ tl.store(out_ptr + offsets + c, value, mask=mask)
99
+
100
+
101
+ def rmsnorm_cast(
102
+ x: Tensor,
103
+ weight: Tensor | None,
104
+ eps: float,
105
+ rope: tuple[Tensor, Tensor] | None = None,
106
+ ) -> Tensor:
107
+ x = x.contiguous()
108
+ if weight is not None:
109
+ weight = weight.contiguous()
110
+ cos, sin = (
111
+ (None, None)
112
+ if rope is None
113
+ else (table.contiguous() for table in rope)
114
+ )
115
+ channels = x.size(-1)
116
+ rows = x.numel() // channels
117
+ block_rows = 8
118
+ grid = ((rows + block_rows - 1) // block_rows,)
119
+ out = torch.empty_like(x, dtype=torch.get_autocast_dtype("cuda"))
120
+ with torch.cuda.device(x.device):
121
+ cast(Any, _rmsnorm_cast_kernel)[grid](
122
+ x,
123
+ weight,
124
+ cos,
125
+ sin,
126
+ out,
127
+ rows,
128
+ x.size(-3) if rope is not None else 1,
129
+ x.size(-2) if rope is not None else 1,
130
+ channels,
131
+ eps,
132
+ block_rows,
133
+ num_warps=4,
134
+ enable_fp_fusion=False,
135
+ )
136
+ return out
@@ -0,0 +1,173 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from typing import Any, cast
5
+
6
+ import torch
7
+ import triton
8
+ import triton.language as tl
9
+ from torch import Tensor
10
+
11
+ _NUM_STATISTICS = 5
12
+
13
+
14
+ @triton.jit
15
+ def _segment_multi_reduce_kernel(
16
+ src_ptr,
17
+ index_ptr,
18
+ edge_attr_ptr,
19
+ edge_type_ptr,
20
+ offsets_ptr,
21
+ output_ptr,
22
+ num_channels: tl.constexpr,
23
+ num_statistics: tl.constexpr,
24
+ block_channels: tl.constexpr,
25
+ has_edge_type: tl.constexpr,
26
+ ) -> None:
27
+ segment = tl.program_id(0)
28
+ channels = tl.program_id(1) * block_channels + tl.arange(0, block_channels)
29
+ channel_mask = channels < num_channels
30
+ start = tl.load(offsets_ptr + segment)
31
+ end = tl.load(offsets_ptr + segment + 1)
32
+
33
+ total = tl.zeros((block_channels,), tl.float32) # ty: ignore[invalid-argument-type]
34
+ square_total = tl.zeros((block_channels,), tl.float32) # ty: ignore[invalid-argument-type]
35
+ minimum = tl.full((block_channels,), float("inf"), tl.float32)
36
+ maximum = tl.full((block_channels,), -float("inf"), tl.float32)
37
+ has_nan = tl.zeros((block_channels,), tl.int1) # ty: ignore[invalid-argument-type]
38
+
39
+ edge = start
40
+ while edge < end:
41
+ row = tl.load(index_ptr + edge)
42
+ value = tl.load(
43
+ src_ptr + row.to(tl.int64) * num_channels + channels,
44
+ mask=channel_mask,
45
+ other=0.0,
46
+ ).to(tl.float32)
47
+ if has_edge_type:
48
+ edge_attr_row = tl.load(edge_type_ptr + edge)
49
+ else:
50
+ edge_attr_row = edge
51
+ value += tl.load(
52
+ edge_attr_ptr
53
+ + edge_attr_row.to(tl.int64) * num_channels
54
+ + channels,
55
+ mask=channel_mask,
56
+ other=0.0,
57
+ ).to(tl.float32)
58
+ total += value
59
+ square_total += value * value
60
+ minimum = tl.minimum(minimum, value)
61
+ maximum = tl.maximum(maximum, value)
62
+ has_nan |= value != value
63
+ edge += 1
64
+
65
+ count = end - start
66
+ divisor = tl.maximum(count, 1)
67
+ mean = total / divisor
68
+ variance = square_total / divisor - mean * mean
69
+ clamped_variance = tl.where(variance < 1e-5, 1e-5, variance)
70
+ std = tl.where(
71
+ variance <= 1e-5,
72
+ 0.0,
73
+ tl.sqrt(clamped_variance),
74
+ )
75
+ minimum_is_inf = (minimum == float("inf")) | (minimum == -float("inf"))
76
+ maximum_is_inf = (maximum == float("inf")) | (maximum == -float("inf"))
77
+ minimum = tl.where(minimum_is_inf, 0.0, minimum)
78
+ maximum = tl.where(maximum_is_inf, 0.0, maximum)
79
+ minimum = tl.where(has_nan, float("nan"), minimum)
80
+ maximum = tl.where(has_nan, float("nan"), maximum)
81
+
82
+ output_offsets = (
83
+ segment.to(tl.int64) * num_statistics * num_channels + channels
84
+ )
85
+ tl.store(output_ptr + output_offsets, total, mask=channel_mask)
86
+ tl.store(
87
+ output_ptr + output_offsets + num_channels,
88
+ mean,
89
+ mask=channel_mask,
90
+ )
91
+ tl.store(
92
+ output_ptr + output_offsets + 2 * num_channels,
93
+ std,
94
+ mask=channel_mask,
95
+ )
96
+ tl.store(
97
+ output_ptr + output_offsets + 3 * num_channels,
98
+ minimum,
99
+ mask=channel_mask,
100
+ )
101
+ tl.store(
102
+ output_ptr + output_offsets + 4 * num_channels,
103
+ maximum,
104
+ mask=channel_mask,
105
+ )
106
+
107
+
108
+ def segment_multi_reduce(
109
+ src: Tensor,
110
+ index: Tensor,
111
+ edge_attr: Tensor,
112
+ offsets: Tensor,
113
+ edge_type: Tensor | None = None,
114
+ ) -> Tensor:
115
+ src = src.contiguous()
116
+ index = index.contiguous()
117
+ edge_attr = edge_attr.contiguous()
118
+ offsets = offsets.contiguous()
119
+ if edge_type is not None:
120
+ edge_type = edge_type.contiguous()
121
+ if edge_type is None and edge_attr.shape != (
122
+ index.numel(),
123
+ src.size(1),
124
+ ):
125
+ raise ValueError(
126
+ "edge_attr must have shape (index.numel(), src.size(1))"
127
+ )
128
+ if edge_type is not None and (
129
+ edge_attr.dim() != 2
130
+ or edge_attr.size(1) != src.size(1)
131
+ or edge_type.numel() != index.numel()
132
+ ):
133
+ raise ValueError(
134
+ "edge_type must match index and edge_attr must have "
135
+ "src.size(1) columns"
136
+ )
137
+
138
+ shape = (offsets.numel() - 1, _NUM_STATISTICS, src.size(1))
139
+ output = src.new_empty(shape)
140
+ if shape[0] == 0 or shape[2] == 0:
141
+ return output
142
+
143
+ block_channels = min(
144
+ triton.next_power_of_2(shape[2]), # ty: ignore[invalid-argument-type]
145
+ 512,
146
+ )
147
+ if block_channels >= 256:
148
+ num_warps = 8
149
+ elif block_channels >= 128:
150
+ num_warps = 4
151
+ else:
152
+ num_warps = 1
153
+ grid = (
154
+ shape[0],
155
+ triton.cdiv( # ty: ignore[invalid-argument-type]
156
+ shape[2], block_channels
157
+ ),
158
+ )
159
+ with torch.cuda.device(src.device):
160
+ cast(Any, _segment_multi_reduce_kernel)[grid](
161
+ src,
162
+ index,
163
+ edge_attr,
164
+ edge_type if edge_type is not None else index,
165
+ offsets,
166
+ output,
167
+ num_channels=shape[2],
168
+ num_statistics=shape[1],
169
+ block_channels=block_channels,
170
+ num_warps=num_warps,
171
+ has_edge_type=edge_type is not None,
172
+ )
173
+ return output
sdm/_memory.py ADDED
@@ -0,0 +1,34 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import os
5
+
6
+ import torch
7
+
8
+
9
+ def chunk_memory_limit(device: torch.device) -> int:
10
+ r"""Bytes one chunk of a chunked operation may occupy on a CUDA device.
11
+
12
+ The limit is the ``SDM_CHUNK_MEMORY_FRACTION`` (default ``0.05``) share of
13
+ the device memory available to this process.
14
+ """
15
+ return int(
16
+ torch.cuda.get_device_properties(device).total_memory
17
+ * torch.cuda.get_per_process_memory_fraction(device)
18
+ * float(os.getenv("SDM_CHUNK_MEMORY_FRACTION", "0.05"))
19
+ )
20
+
21
+
22
+ def split_size(num_items: int, item_bytes: int, device: torch.device) -> int:
23
+ r"""Split size of balanced chunks of items of ``item_bytes`` bytes each.
24
+
25
+ On CUDA devices, chunks fit :func:`chunk_memory_limit`, unless a single
26
+ item exceeds the limit. Autograd keeps the memory of every chunk, so one
27
+ chunk holds all items elsewhere or while gradients are enabled.
28
+ """
29
+ if device.type != "cuda" or torch.is_grad_enabled() or item_bytes == 0:
30
+ return max(num_items, 1)
31
+ limit = max(chunk_memory_limit(device), 1)
32
+ capacity = max(limit // item_bytes, 1)
33
+ num_chunks = max(-(-num_items // capacity), 1)
34
+ return max(-(-num_items // num_chunks), 1)