structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,196 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from collections.abc import Sequence
5
+ from typing import Literal, cast
6
+
7
+ import torch
8
+ from torch import Tensor
9
+
10
+ from sdm import NaT, Stype, TableTensor
11
+ from sdm.processing import Processor
12
+
13
+ US_PER_MINUTE = 60 * 1_000_000
14
+ US_PER_HOUR = 60 * US_PER_MINUTE
15
+ US_PER_DAY = 24 * US_PER_HOUR
16
+
17
+
18
+ class AddCalendarFields(Processor):
19
+ r"""Add numerical calendar fields derived from datetime columns.
20
+
21
+ Args:
22
+ fields: The calendar fields to add.
23
+ encoding: How to encode each field. ``"raw"`` uses raw integer values.
24
+ ``"cyclic"`` uses sine and cosine values.
25
+ """
26
+
27
+ handles_stypes = frozenset({Stype.datetime})
28
+ requires_fit = False
29
+
30
+ def __init__(
31
+ self,
32
+ fields: Sequence[
33
+ Literal[
34
+ "minute",
35
+ "hour",
36
+ "weekday",
37
+ "day_of_month",
38
+ "month",
39
+ ]
40
+ ],
41
+ encoding: Literal["raw", "cyclic"] = "raw",
42
+ ) -> None:
43
+ super().__init__()
44
+
45
+ if len(fields) != len(set(fields)):
46
+ raise ValueError("Expected datetime fields to be unique")
47
+
48
+ self.fields = fields
49
+ self.encoding = encoding
50
+
51
+ def _transform(self, table: TableTensor) -> TableTensor:
52
+ if len(self.fields) == 0:
53
+ return table
54
+
55
+ datetime = table.datetime
56
+ na_mask = datetime == NaT
57
+
58
+ days: Tensor | None = None
59
+ if len({"weekday", "day_of_month", "month"} & set(self.fields)) > 0:
60
+ days = datetime.div(US_PER_DAY, rounding_mode="floor")
61
+
62
+ year = month = day = None
63
+ if len({"day_of_month", "month"} & set(self.fields)) > 0:
64
+ assert days is not None
65
+ year, month, day = _civil_from_days(days)
66
+
67
+ time_of_day: Tensor | None = None
68
+ if len({"minute", "hour"} & set(self.fields)) > 0:
69
+ time_of_day = datetime.remainder(US_PER_DAY)
70
+
71
+ outs: dict[str, Tensor] = {}
72
+ for field in self.fields:
73
+ if field == "minute":
74
+ assert time_of_day is not None
75
+ out = time_of_day.div(US_PER_MINUTE, rounding_mode="floor")
76
+ outs[field] = out.remainder(60)
77
+ continue
78
+
79
+ if field == "hour":
80
+ assert time_of_day is not None
81
+ out = time_of_day.div(US_PER_HOUR, rounding_mode="floor")
82
+ outs[field] = out
83
+ continue
84
+
85
+ if field == "weekday":
86
+ assert days is not None
87
+ outs[field] = (days + 3).remainder(7)
88
+ continue
89
+
90
+ if field == "month":
91
+ assert month is not None
92
+ outs[field] = month - 1
93
+ continue
94
+
95
+ if field == "day_of_month":
96
+ assert day is not None
97
+ outs[field] = day - 1
98
+ continue
99
+
100
+ raise ValueError(
101
+ f"{self.__class__.__name__!r} received unsupported "
102
+ f"field {field!r}"
103
+ )
104
+
105
+ if self.encoding == "cyclic":
106
+ for field, out in outs.items():
107
+ if field == "minute":
108
+ outs[field] = out / 60.0
109
+ elif field == "hour":
110
+ outs[field] = out / 24.0
111
+ elif field == "weekday":
112
+ outs[field] = out / 7.0
113
+ elif field == "month":
114
+ outs[field] = out / 12.0
115
+ else:
116
+ assert field == "day_of_month"
117
+ assert year is not None
118
+ assert month is not None
119
+ period = torch.tensor(
120
+ [-1, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31],
121
+ device=year.device,
122
+ )
123
+ period = period[month]
124
+ leap_year = (year.remainder(4) == 0) & (
125
+ (year.remainder(100) != 0) | (year.remainder(400) == 0)
126
+ )
127
+ period = torch.where(leap_year & (month == 2), 29, period)
128
+ outs[field] = out / period
129
+
130
+ out = torch.stack(list(outs.values()), dim=-1)
131
+ out = out.to(table.numerical.dtype)
132
+ out = out.masked_fill(na_mask.unsqueeze(-1), float("nan"))
133
+ out = out.flatten(-2, -1)
134
+
135
+ if self.encoding == "cyclic":
136
+ out *= 2 * torch.pi
137
+ out = torch.stack([out.sin(), out.cos()], dim=-1).flatten(-2, -1)
138
+
139
+ columns = tuple(
140
+ f"{column}__{field}__{fn}"
141
+ for column in table.columns[Stype.datetime]
142
+ for field in self.fields
143
+ for fn in ("sin", "cos")
144
+ )
145
+ else:
146
+ assert self.encoding == "raw"
147
+ columns = tuple(
148
+ f"{column}__{field}"
149
+ for column in table.columns[Stype.datetime]
150
+ for field in self.fields
151
+ )
152
+
153
+ out_table = TableTensor(
154
+ columns={Stype.numerical: columns},
155
+ numerical=out,
156
+ )
157
+
158
+ return cast(TableTensor, torch.cat([table, out_table], dim=-1))
159
+
160
+ def __repr__(self, *, indent: int = 0) -> str:
161
+ field_repr = "".join(
162
+ f"{' ' * (indent + 4)}{field!r},\n" for field in self.fields
163
+ )
164
+ return (
165
+ f"{' ' * indent}{self.__class__.__name__}(\n"
166
+ f"{' ' * (indent + 2)}fields=[\n"
167
+ f"{field_repr}"
168
+ f"{' ' * (indent + 2)}],\n"
169
+ f"{' ' * (indent + 2)}encoding={self.encoding!r},\n"
170
+ f"{' ' * indent})"
171
+ )
172
+
173
+
174
+ def _civil_from_days(days: Tensor) -> tuple[Tensor, Tensor, Tensor]:
175
+ """Convert days since the Unix epoch to Gregorian year, month, and day."""
176
+ shifted = days + 719_468
177
+ era = shifted.div(146_097, rounding_mode="floor")
178
+ day_of_era = shifted - era * 146_097
179
+ year_of_era = (
180
+ day_of_era
181
+ - day_of_era.div(1_460, rounding_mode="floor")
182
+ + day_of_era.div(36_524, rounding_mode="floor")
183
+ - day_of_era.div(146_096, rounding_mode="floor")
184
+ ).div(365, rounding_mode="floor")
185
+ year = year_of_era + era * 400
186
+ day_of_year = day_of_era - (
187
+ 365 * year_of_era
188
+ + year_of_era.div(4, rounding_mode="floor")
189
+ - year_of_era.div(100, rounding_mode="floor")
190
+ )
191
+ month_prime = (5 * day_of_year + 2).div(153, rounding_mode="floor")
192
+ month_offset = (153 * month_prime + 2).div(5, rounding_mode="floor")
193
+ day = day_of_year - month_offset + 1
194
+ month = month_prime + torch.where(month_prime < 10, 3, -9)
195
+ year = year + (month <= 2)
196
+ return year, month, day
@@ -0,0 +1,203 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ from __future__ import annotations
5
+
6
+ import abc
7
+ from typing import Self
8
+
9
+ import torch
10
+
11
+ from sdm import EnsembleTable, TableTensor
12
+ from sdm.processing import InvertibleMixin, Processor
13
+
14
+
15
+ class EnsembleProcessor(Processor):
16
+ r"""Base processor for ensemble-aware table transformations.
17
+
18
+ An :class:`EnsembleProcessor` defines a reusable transformation on
19
+ :class:`~sdm.EnsembleTable` for feature, target and output
20
+ pre/post-processing across ensemble members.
21
+ An :class:`EnsembleProcessor` learns any required state via
22
+ :meth:`fit_ensemble` and applies the transformation via
23
+ :meth:`transform_ensemble`. :meth:`fit_ensemble`,
24
+ :meth:`transform_ensemble`, and :meth:`fit_transform_ensemble`
25
+ are no-ops for stypes outside of
26
+ :attr:`~sdm.processing.base.Processor.handles_stypes`.
27
+
28
+ As a :class:`~sdm.processing.base.Processor`, it also accepts a
29
+ :class:`~sdm.tensor.TableTensor` and processes it as an ensemble
30
+ with one member.
31
+ """
32
+
33
+ @staticmethod
34
+ def as_processor(processor: object) -> EnsembleProcessor:
35
+ r"""Normalize a processor-like object to a :class:`EnsembleProcessor`.
36
+
37
+ Args:
38
+ processor: A processor-like object. An :class:`EnsembleProcessor`
39
+ is returned as-is, and any ordinary processor-like object is
40
+ adapted to ensemble processing.
41
+ """
42
+ from sdm.processing import EnsembleProcessorAdapter # noqa: PLC0415
43
+
44
+ processor = Processor.as_processor(processor)
45
+ if isinstance(processor, EnsembleProcessor):
46
+ return processor
47
+ return EnsembleProcessorAdapter(processor)
48
+
49
+ def _fit(
50
+ self,
51
+ table: TableTensor,
52
+ *,
53
+ generator: torch.Generator | None = None,
54
+ ) -> None:
55
+ self._fit_ensemble(
56
+ EnsembleTable.from_table(table, num_members=1),
57
+ generator=generator,
58
+ )
59
+
60
+ def _transform(self, table: TableTensor) -> TableTensor:
61
+ output = self._transform_ensemble(
62
+ EnsembleTable.from_table(table, num_members=1)
63
+ )
64
+ return output[0]
65
+
66
+ def _fit_transform(
67
+ self,
68
+ table: TableTensor,
69
+ *,
70
+ generator: torch.Generator | None = None,
71
+ ) -> TableTensor:
72
+ if not self.requires_fit:
73
+ return self._transform(table)
74
+ output = self._fit_transform_ensemble(
75
+ EnsembleTable.from_table(table, num_members=1),
76
+ generator=generator,
77
+ )
78
+ return output[0]
79
+
80
+ def _fit_ensemble(
81
+ self,
82
+ ensemble_table: EnsembleTable,
83
+ *,
84
+ generator: torch.Generator | None = None,
85
+ ) -> None:
86
+ raise NotImplementedError
87
+
88
+ @abc.abstractmethod
89
+ def _transform_ensemble(
90
+ self,
91
+ ensemble_table: EnsembleTable,
92
+ ) -> EnsembleTable:
93
+ pass
94
+
95
+ def _fit_transform_ensemble(
96
+ self,
97
+ ensemble_table: EnsembleTable,
98
+ *,
99
+ generator: torch.Generator | None = None,
100
+ ) -> EnsembleTable:
101
+ if self.requires_fit:
102
+ self._fit_ensemble(ensemble_table, generator=generator)
103
+ return self._transform_ensemble(ensemble_table)
104
+
105
+ def fit_ensemble(
106
+ self,
107
+ ensemble_table: EnsembleTable,
108
+ *,
109
+ generator: torch.Generator | None = None,
110
+ ) -> Self:
111
+ """Fit the processor on an ensemble table.
112
+
113
+ Args:
114
+ ensemble_table: Ensemble table used to compute the processor state.
115
+ generator: Pseudorandom number generator used for sampling.
116
+ """
117
+ if not any(
118
+ group.active_stypes & self.handles_stypes
119
+ for group in ensemble_table._iter_groups()
120
+ ):
121
+ return self
122
+ if self.requires_fit:
123
+ self._fit_ensemble(ensemble_table, generator=generator)
124
+ self._set_fitted(ensemble_table[0].device)
125
+ return self
126
+
127
+ def transform_ensemble(
128
+ self,
129
+ ensemble_table: EnsembleTable,
130
+ ) -> EnsembleTable:
131
+ """Transform an ensemble table with fitted state.
132
+
133
+ Args:
134
+ ensemble_table: Ensemble table to transform.
135
+
136
+ Returns:
137
+ The transformed ensemble table.
138
+ """
139
+ if not any(
140
+ group.active_stypes & self.handles_stypes
141
+ for group in ensemble_table._iter_groups()
142
+ ):
143
+ return ensemble_table
144
+ self._check_is_fitted()
145
+ return self._transform_ensemble(ensemble_table)
146
+
147
+ def fit_transform_ensemble(
148
+ self,
149
+ ensemble_table: EnsembleTable,
150
+ *,
151
+ generator: torch.Generator | None = None,
152
+ ) -> EnsembleTable:
153
+ """Fit the processor and transform an ensemble table.
154
+
155
+ Args:
156
+ ensemble_table: Ensemble table to fit on and transform.
157
+ generator: Pseudorandom number generator used for sampling.
158
+
159
+ Returns:
160
+ The transformed ensemble table.
161
+ """
162
+ if not any(
163
+ group.active_stypes & self.handles_stypes
164
+ for group in ensemble_table._iter_groups()
165
+ ):
166
+ return ensemble_table
167
+ output = self._fit_transform_ensemble(
168
+ ensemble_table,
169
+ generator=generator,
170
+ )
171
+ if self.requires_fit:
172
+ self._set_fitted(ensemble_table[0].device)
173
+ return output
174
+
175
+
176
+ class EnsembleInvertibleMixin(InvertibleMixin):
177
+ r"""Extend a :class:`EnsembleProcessor` by an inverse transformation."""
178
+
179
+ def _inverse_transform(self, table: TableTensor) -> TableTensor:
180
+ output = self._inverse_transform_ensemble(
181
+ EnsembleTable.from_table(table, num_members=1)
182
+ )
183
+ return output[0]
184
+
185
+ @abc.abstractmethod
186
+ def _inverse_transform_ensemble(
187
+ self, ensemble_table: EnsembleTable
188
+ ) -> EnsembleTable: ...
189
+
190
+ def inverse_transform_ensemble(
191
+ self, ensemble_table: EnsembleTable
192
+ ) -> EnsembleTable:
193
+ r"""Apply the inverse transformation to ``ensemble_table``.
194
+
195
+ Args:
196
+ ensemble_table: The ensemble table in transformed representation.
197
+
198
+ Returns:
199
+ The table restored to the representation before
200
+ :meth:`~EnsembleProcessor.transform_ensemble`.
201
+ """
202
+ self._check_is_fitted()
203
+ return self._inverse_transform_ensemble(ensemble_table)