structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,22 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Utilities for converting tensors to external data formats."""
5
+
6
+ from sdm.tensor.io.arrow import (
7
+ ARROW_TORCH_DTYPES,
8
+ TORCH_ARROW_DTYPES,
9
+ combine_arrow_chunks,
10
+ arrow_as_tensor,
11
+ to_arrow,
12
+ )
13
+ from sdm.tensor.io.cudf import to_cudf
14
+
15
+ __all__ = [
16
+ "ARROW_TORCH_DTYPES",
17
+ "TORCH_ARROW_DTYPES",
18
+ "combine_arrow_chunks",
19
+ "arrow_as_tensor",
20
+ "to_arrow",
21
+ "to_cudf",
22
+ ]
sdm/tensor/io/arrow.py ADDED
@@ -0,0 +1,120 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import warnings
5
+
6
+ import pyarrow as pa
7
+ import torch
8
+ from torch import Tensor
9
+
10
+ ARROW_TORCH_DTYPES = {
11
+ pa.bool_(): torch.bool,
12
+ pa.uint8(): torch.uint8,
13
+ pa.uint16(): torch.uint16,
14
+ pa.uint32(): torch.uint32,
15
+ pa.uint64(): torch.uint64,
16
+ pa.int8(): torch.int8,
17
+ pa.int16(): torch.int16,
18
+ pa.int32(): torch.int32,
19
+ pa.int64(): torch.int64,
20
+ pa.float16(): torch.float16,
21
+ pa.float32(): torch.float32,
22
+ pa.float64(): torch.float64,
23
+ }
24
+ TORCH_ARROW_DTYPES = {value: key for key, value in ARROW_TORCH_DTYPES.items()}
25
+
26
+
27
+ def combine_arrow_chunks(array: pa.ChunkedArray) -> pa.Array:
28
+ r"""Combine a :class:`pyarrow.ChunkedArray` into a single array.
29
+
30
+ String and string-dictionary chunks are promoted to large-string storage
31
+ before combining to prevent offset overflows.
32
+
33
+ Args:
34
+ array: The :class:`pyarrow.ChunkedArray`.
35
+ """
36
+ if array.num_chunks == 0:
37
+ return array.combine_chunks()
38
+ if array.num_chunks == 1:
39
+ return array.chunk(0)
40
+
41
+ if pa.types.is_string(array.type):
42
+ array = array.cast(pa.large_string())
43
+ elif pa.types.is_dictionary(array.type) and pa.types.is_string(
44
+ array.type.value_type
45
+ ):
46
+ array = array.cast(
47
+ pa.dictionary(
48
+ array.type.index_type,
49
+ pa.large_string(),
50
+ ordered=array.type.ordered,
51
+ )
52
+ )
53
+
54
+ return array.combine_chunks()
55
+
56
+
57
+ def arrow_as_tensor(
58
+ array: pa.Array | pa.ChunkedArray,
59
+ *,
60
+ dtype: torch.dtype | None = None,
61
+ device: torch.device | str | None = None,
62
+ ) -> Tensor:
63
+ r"""Convert a :class:`pyarrow.Array` to a tensor.
64
+
65
+ Args:
66
+ array: The :class:`pyarrow.Array` or :class:`pyarrow.ChunkedArray`.
67
+ dtype: The dtype.
68
+ device: The device.
69
+ """
70
+ if isinstance(array, pa.ChunkedArray):
71
+ array = combine_arrow_chunks(array)
72
+
73
+ if pa.types.is_boolean(array.type) and array.null_count > 0:
74
+ dtype = dtype or torch.get_default_dtype()
75
+ if dtype.is_floating_point:
76
+ array = array.cast(TORCH_ARROW_DTYPES[dtype])
77
+ else:
78
+ array = array.fill_null(False)
79
+
80
+ values = array.to_numpy(zero_copy_only=False)
81
+
82
+ with warnings.catch_warnings():
83
+ warnings.filterwarnings(
84
+ "ignore",
85
+ message="The given NumPy array is not writable",
86
+ )
87
+ return torch.as_tensor(values, dtype=dtype, device=device)
88
+
89
+
90
+ def to_arrow(tensor: Tensor, valid_mask: Tensor | None = None) -> pa.Array:
91
+ r"""Convert a tensor to a flat :class:`pyarrow.Array`.
92
+
93
+ Args:
94
+ tensor: The tensor.
95
+ valid_mask: Boolean mask indicating valid, non-null tensor elements.
96
+ """
97
+ tensor = tensor.detach().contiguous().view(-1).cpu()
98
+ if valid_mask is not None:
99
+ valid_mask = valid_mask.contiguous().view(-1).cpu()
100
+
101
+ arrow_type = TORCH_ARROW_DTYPES.get(tensor.dtype)
102
+ if arrow_type is None:
103
+ raise TypeError(f"Unsupported data type '{tensor.dtype}'")
104
+
105
+ if tensor.dtype == torch.bool:
106
+ buffer = pa.array(tensor.numpy(), type=pa.bool_()).buffers()[1]
107
+ else:
108
+ buffer = pa.py_buffer(tensor.numpy())
109
+
110
+ return pa.Array.from_buffers(
111
+ type=arrow_type,
112
+ length=tensor.numel(),
113
+ buffers=[
114
+ pa.array(valid_mask.numpy(), type=pa.bool_()).buffers()[1]
115
+ if valid_mask is not None
116
+ else None,
117
+ buffer,
118
+ ],
119
+ null_count=-1 if valid_mask is not None else 0,
120
+ )
sdm/tensor/io/cudf.py ADDED
@@ -0,0 +1,47 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from __future__ import annotations
5
+
6
+ from typing import TYPE_CHECKING
7
+
8
+ import torch
9
+ from torch import Tensor
10
+
11
+ if TYPE_CHECKING:
12
+ import cudf
13
+
14
+
15
+ def to_cudf(tensor: Tensor, valid_mask: Tensor | None = None) -> cudf.Series:
16
+ r"""Convert a CUDA tensor to a flat :class:`cudf.Series`.
17
+
18
+ Args:
19
+ tensor: The CUDA tensor.
20
+ valid_mask: Boolean mask indicating valid, non-null tensor elements.
21
+ """
22
+ import cudf
23
+
24
+ if not tensor.is_cuda:
25
+ raise ValueError(
26
+ f"Expected tensor to be on a CUDA device (got '{tensor.device}')"
27
+ )
28
+
29
+ tensor = tensor.detach().contiguous().view(-1)
30
+ with torch.cuda.device(tensor.device):
31
+ ser = cudf.Series(tensor, copy=False, nan_as_null=valid_mask is None)
32
+
33
+ if valid_mask is None:
34
+ return ser
35
+
36
+ if valid_mask.device != tensor.device:
37
+ raise ValueError(
38
+ f"Expected 'valid_mask' to be on device '{tensor.device}' "
39
+ f"(got '{valid_mask.device}')"
40
+ )
41
+
42
+ with torch.cuda.device(tensor.device):
43
+ mask = to_cudf(valid_mask.contiguous().view(-1))._column.as_mask()
44
+ if not isinstance(mask, tuple):
45
+ mask = (mask,)
46
+
47
+ return cudf.Series._from_column(ser._column.set_mask(*mask))
sdm/tensor/mixin.py ADDED
@@ -0,0 +1,124 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import abc
5
+ from collections.abc import Callable, Iterator
6
+ from typing import Self
7
+
8
+ import torch
9
+ from torch import Tensor
10
+
11
+
12
+ class DeviceMixin(abc.ABC):
13
+ r"""Add :class:`torch.Tensor`-like device capabilities to an object."""
14
+
15
+ def to(
16
+ self,
17
+ device: torch.device | str | None,
18
+ *,
19
+ non_blocking: bool = False,
20
+ ) -> Self:
21
+ r"""Perform device conversion.
22
+
23
+ Args:
24
+ device: The device.
25
+ non_blocking: If ``True`` and the source is in pinned memory, the
26
+ copy will be asynchronous with respect to the host. Otherwise,
27
+ the argument has no effect.
28
+ """
29
+ return self._apply_tensor(
30
+ lambda x: x.to(device, non_blocking=non_blocking)
31
+ )
32
+
33
+ def cpu(self) -> Self:
34
+ r"""Copy data in CPU memory."""
35
+ return self._apply_tensor(lambda x: x.cpu())
36
+
37
+ def cuda(
38
+ self,
39
+ device: torch.device | str | int | None = None,
40
+ *,
41
+ non_blocking: bool = False,
42
+ ) -> Self:
43
+ r"""Copy data in CUDA memory.
44
+
45
+ Args:
46
+ device: The device.
47
+ non_blocking: If ``True`` and the source is in pinned memory, the
48
+ copy will be asynchronous with respect to the host. Otherwise,
49
+ the argument has no effect.
50
+ """
51
+ return self._apply_tensor(
52
+ lambda x: x.cuda(device, non_blocking=non_blocking)
53
+ )
54
+
55
+ @property
56
+ def device(self) -> torch.device:
57
+ r"""The :class:`torch.device` where the data is."""
58
+ devices = {tensor.device for tensor in self._tensors()}
59
+ if len(devices) == 0:
60
+ raise RuntimeError(
61
+ f"Could not determine 'device' of empty "
62
+ f"{self.__class__.__name__!r}"
63
+ )
64
+ if len(devices) > 1:
65
+ raise RuntimeError(
66
+ f"Expected tensors in {self.__class__.__name__!r} to be on "
67
+ f"the same device (got {list(devices)})"
68
+ )
69
+ return next(iter(devices))
70
+
71
+ @property
72
+ def is_cpu(self) -> bool:
73
+ r"""Whether the data is stored on the CPU."""
74
+ devices = {tensor.device for tensor in self._tensors()}
75
+ if len(devices) == 0:
76
+ raise RuntimeError(
77
+ f"Could not determine 'is_cpu' of empty "
78
+ f"{self.__class__.__name__!r}"
79
+ )
80
+ return all(device.type == "cpu" for device in devices)
81
+
82
+ @property
83
+ def is_cuda(self) -> bool:
84
+ r"""Whether the data is stored on the GPU."""
85
+ devices = {tensor.device for tensor in self._tensors()}
86
+ if len(devices) == 0:
87
+ raise RuntimeError(
88
+ f"Could not determine 'is_cuda' of empty "
89
+ f"{self.__class__.__name__!r}"
90
+ )
91
+ return all(device.type == "cuda" for device in devices)
92
+
93
+ def pin_memory(self) -> Self:
94
+ r"""Copy data to pinned memory, if it is not already pinned."""
95
+ return self._apply_tensor(lambda x: x.pin_memory())
96
+
97
+ def is_pinned(self) -> bool:
98
+ r"""Returns ``True`` if data resides in pinned memory."""
99
+ tensors = tuple(tensor for tensor in self._tensors())
100
+ if len(tensors) == 0:
101
+ raise RuntimeError(
102
+ f"Could not determine 'in_pinned' of empty "
103
+ f"{self.__class__.__name__!r}"
104
+ )
105
+ return all(tensor.is_pinned() for tensor in tensors)
106
+
107
+ @abc.abstractmethod
108
+ def _tensors(self) -> Iterator[Tensor]:
109
+ pass
110
+
111
+ @abc.abstractmethod
112
+ def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self:
113
+ pass
114
+
115
+
116
+ def _resolve_device(device: torch.device | str | None) -> torch.device | None:
117
+ if device is None:
118
+ return None
119
+ device = torch.device(device)
120
+ if device.type == "cuda" and device.index is None:
121
+ return torch.device("cuda", torch.cuda.current_device())
122
+ if device.type == "mps" and device.index is None:
123
+ return torch.device("mps", 0)
124
+ return device