structured-data-models 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdm/__init__.py +58 -0
- sdm/_constants.py +6 -0
- sdm/_inference.py +42 -0
- sdm/_kernels/__init__.py +10 -0
- sdm/_kernels/rmsnorm_cast.py +92 -0
- sdm/_kernels/segment_multi_reduce.py +141 -0
- sdm/_kernels/triton/__init__.py +7 -0
- sdm/_kernels/triton/rmsnorm_cast.py +136 -0
- sdm/_kernels/triton/segment_multi_reduce.py +173 -0
- sdm/_memory.py +34 -0
- sdm/_warnings.py +29 -0
- sdm/cache.py +163 -0
- sdm/ensemble.py +501 -0
- sdm/evaluation/__init__.py +11 -0
- sdm/evaluation/classification.py +205 -0
- sdm/explain/__init__.py +12 -0
- sdm/explain/base.py +123 -0
- sdm/explain/gradient.py +112 -0
- sdm/models/__init__.py +20 -0
- sdm/models/_huggingface.py +58 -0
- sdm/models/base.py +966 -0
- sdm/models/callback.py +106 -0
- sdm/models/ecoc.py +197 -0
- sdm/models/kumo/__init__.py +13 -0
- sdm/models/kumo/relational/NOTICE +5 -0
- sdm/models/kumo/relational/__init__.py +11 -0
- sdm/models/kumo/relational/graph.py +101 -0
- sdm/models/kumo/relational/invariant_gnn.py +122 -0
- sdm/models/kumo/relational/model.py +602 -0
- sdm/models/kumo/relational/recipe.py +18 -0
- sdm/models/kumo/relational/task.py +113 -0
- sdm/models/kumo/tabular/__init__.py +17 -0
- sdm/models/kumo/tabular/batching.py +139 -0
- sdm/models/kumo/tabular/block.py +100 -0
- sdm/models/kumo/tabular/cell_embedding.py +85 -0
- sdm/models/kumo/tabular/icl.py +136 -0
- sdm/models/kumo/tabular/model.py +299 -0
- sdm/models/kumo/tabular/recipe.py +71 -0
- sdm/models/kumo/tabular/row_embedding.py +214 -0
- sdm/models/tabfm/LICENSE +209 -0
- sdm/models/tabfm/__init__.py +11 -0
- sdm/models/tabfm/block.py +94 -0
- sdm/models/tabfm/cell_embedding.py +219 -0
- sdm/models/tabfm/ckpt.py +242 -0
- sdm/models/tabfm/icl.py +107 -0
- sdm/models/tabfm/model.py +261 -0
- sdm/models/tabfm/recipe.py +46 -0
- sdm/models/tabfm/row_embedding.py +241 -0
- sdm/models/tabiclv2/LICENSE +33 -0
- sdm/models/tabiclv2/__init__.py +11 -0
- sdm/models/tabiclv2/block.py +81 -0
- sdm/models/tabiclv2/ckpt.py +134 -0
- sdm/models/tabiclv2/icl.py +420 -0
- sdm/models/tabiclv2/model.py +293 -0
- sdm/models/tabiclv2/recipe.py +46 -0
- sdm/models/tabiclv2/row_embedding.py +254 -0
- sdm/nn/__init__.py +28 -0
- sdm/nn/_buffer.py +91 -0
- sdm/nn/attention.py +825 -0
- sdm/nn/glu.py +78 -0
- sdm/nn/linear.py +46 -0
- sdm/nn/rms_norm.py +59 -0
- sdm/nn/rope.py +107 -0
- sdm/nn/scaling.py +240 -0
- sdm/nn/set_transformer.py +162 -0
- sdm/nn/softplus.py +65 -0
- sdm/processing/__init__.py +96 -0
- sdm/processing/base.py +254 -0
- sdm/processing/categorical/__init__.py +16 -0
- sdm/processing/categorical/_categorical.py +60 -0
- sdm/processing/categorical/align.py +540 -0
- sdm/processing/categorical/count.py +121 -0
- sdm/processing/categorical/impute.py +107 -0
- sdm/processing/categorical/shuffle.py +204 -0
- sdm/processing/common/__init__.py +32 -0
- sdm/processing/common/callable.py +29 -0
- sdm/processing/common/choice.py +185 -0
- sdm/processing/common/drop.py +28 -0
- sdm/processing/common/ensemble.py +187 -0
- sdm/processing/common/identity.py +18 -0
- sdm/processing/common/select.py +107 -0
- sdm/processing/common/sequential.py +158 -0
- sdm/processing/common/shuffle.py +195 -0
- sdm/processing/common/stype.py +199 -0
- sdm/processing/common/table.py +120 -0
- sdm/processing/common/task.py +135 -0
- sdm/processing/common/to_numerical.py +60 -0
- sdm/processing/datetime/__init__.py +8 -0
- sdm/processing/datetime/calendar.py +196 -0
- sdm/processing/ensemble.py +203 -0
- sdm/processing/execution.py +418 -0
- sdm/processing/numerical/__init__.py +38 -0
- sdm/processing/numerical/_stats.py +29 -0
- sdm/processing/numerical/cast.py +33 -0
- sdm/processing/numerical/clip.py +48 -0
- sdm/processing/numerical/clip_soft.py +54 -0
- sdm/processing/numerical/constant.py +235 -0
- sdm/processing/numerical/flip_sign.py +78 -0
- sdm/processing/numerical/impute.py +52 -0
- sdm/processing/numerical/pca.py +62 -0
- sdm/processing/numerical/power.py +325 -0
- sdm/processing/numerical/quantile.py +306 -0
- sdm/processing/numerical/quantile_clip.py +63 -0
- sdm/processing/numerical/random_projection.py +86 -0
- sdm/processing/numerical/rank_gaussian.py +168 -0
- sdm/processing/numerical/robust_scale.py +88 -0
- sdm/processing/numerical/sigma_clip.py +123 -0
- sdm/processing/numerical/standardize.py +71 -0
- sdm/processing/output/__init__.py +14 -0
- sdm/processing/output/reduce.py +86 -0
- sdm/processing/output/softmax.py +48 -0
- sdm/processing/output/sort.py +16 -0
- sdm/processing/recipe.py +154 -0
- sdm/processing/text/__init__.py +12 -0
- sdm/processing/text/sentence_transformer.py +451 -0
- sdm/processing/text/tfidf.py +478 -0
- sdm/relational/__init__.py +16 -0
- sdm/relational/backend/__init__.py +12 -0
- sdm/relational/backend/_cugraph.py +546 -0
- sdm/relational/backend/_pyg_lib.py +175 -0
- sdm/relational/data.py +393 -0
- sdm/relational/join.py +173 -0
- sdm/relational/sampler.py +259 -0
- sdm/relational/task.py +465 -0
- sdm/stype.py +396 -0
- sdm/task.py +53 -0
- sdm/tensor/__init__.py +20 -0
- sdm/tensor/categorical.py +953 -0
- sdm/tensor/columnar.py +1127 -0
- sdm/tensor/io/__init__.py +22 -0
- sdm/tensor/io/arrow.py +120 -0
- sdm/tensor/io/cudf.py +47 -0
- sdm/tensor/mixin.py +124 -0
- sdm/tensor/nullable.py +963 -0
- sdm/tensor/string.py +526 -0
- sdm/tensor/table.py +2035 -0
- sdm/tensor/var_len.py +1486 -0
- sdm/testing/__init__.py +14 -0
- sdm/testing/decorators.py +61 -0
- sdm/testing/memory.py +92 -0
- structured_data_models-0.2.0.dist-info/METADATA +142 -0
- structured_data_models-0.2.0.dist-info/RECORD +152 -0
- structured_data_models-0.2.0.dist-info/WHEEL +5 -0
- structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
- structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
- structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
- structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
- structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/stype.py
ADDED
|
@@ -0,0 +1,396 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import importlib.util
|
|
7
|
+
import re
|
|
8
|
+
import warnings
|
|
9
|
+
from collections.abc import Callable, Hashable, Iterable, Mapping
|
|
10
|
+
from enum import StrEnum
|
|
11
|
+
from typing import TYPE_CHECKING, Literal, TypeAlias
|
|
12
|
+
|
|
13
|
+
import pyarrow as pa
|
|
14
|
+
import pyarrow.compute as pc
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
import cudf
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Stype(StrEnum):
|
|
22
|
+
r"""The semantic type of a table column.
|
|
23
|
+
|
|
24
|
+
A semantic type denotes the semantic meaning of a column, and denotes how
|
|
25
|
+
columns are encoded into a feature space.
|
|
26
|
+
Possible values are:
|
|
27
|
+
|
|
28
|
+
Attributes:
|
|
29
|
+
numerical: Numerical columns.
|
|
30
|
+
categorical: Categorical columns.
|
|
31
|
+
datetime: Date or date-time columns.
|
|
32
|
+
text: Text columns.
|
|
33
|
+
id: Identifier values used to distinguish or link entities. Identifier
|
|
34
|
+
columns are not used as model features by default.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
numerical = "numerical"
|
|
38
|
+
categorical = "categorical"
|
|
39
|
+
datetime = "datetime"
|
|
40
|
+
text = "text"
|
|
41
|
+
id = "id"
|
|
42
|
+
|
|
43
|
+
def __repr__(self) -> str:
|
|
44
|
+
return f"{self.__class__.__name__}.{self.name}"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
StypeLike: TypeAlias = Stype | str
|
|
48
|
+
|
|
49
|
+
# Semantic Type Inference #####################################################
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
Policy: TypeAlias = Literal["off", "infer", "drop"]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def infer_stypes(
|
|
56
|
+
table: pa.Table | pd.DataFrame | cudf.DataFrame,
|
|
57
|
+
overrides: Mapping[str, StypeLike] | None = None,
|
|
58
|
+
*,
|
|
59
|
+
text: Literal["off", "infer", "drop"] = "off",
|
|
60
|
+
id: Literal["off", "infer", "drop"] = "off",
|
|
61
|
+
_low_cardinality: Literal["off", "infer"] = "off",
|
|
62
|
+
unsupported: Literal["error", "warn", "drop"] = "error",
|
|
63
|
+
) -> dict[str, StypeLike]:
|
|
64
|
+
r"""Infer semantic types from raw data statistics.
|
|
65
|
+
|
|
66
|
+
Semantic types are inferred as follows:
|
|
67
|
+
|
|
68
|
+
* Integer, floating-point, and decimal columns are inferred as
|
|
69
|
+
:attr:`~Stype.numerical`.
|
|
70
|
+
* String, boolean and dictionary-encoded columns are inferred as
|
|
71
|
+
:attr:`~Stype.categorical`.
|
|
72
|
+
* Datetime columns are inferred as :attr:`~Stype.datetime`.
|
|
73
|
+
|
|
74
|
+
Optionally, infer the following semantic types based on best-effort
|
|
75
|
+
heuristics:
|
|
76
|
+
|
|
77
|
+
* String columns are inferred as :attr:`~Stype.text` if they contain at
|
|
78
|
+
least 200 distinct string values, a distinct non-null value ratio of at
|
|
79
|
+
least 5%, and an average of at least ``3`` words per distinct value.
|
|
80
|
+
* Integer or (non-dictionary) string columns are inferred as
|
|
81
|
+
:attr:`~Stype.id` if its name contains ``"id"`` as a whole word
|
|
82
|
+
(*e.g.*, ``"user_id"``, ``"userId"``, ``"id"``, but not ``"solid"`` or
|
|
83
|
+
``"covid"``).
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
table: A :class:`pandas.DataFrame`, :class:`pyarrow.Table`, or
|
|
87
|
+
:class:`cudf.DataFrame`.
|
|
88
|
+
overrides: Optional semantic type overrides by column name.
|
|
89
|
+
text: The :attr:`~Stype.text` detection policy.
|
|
90
|
+
``"off"`` disables :attr:`~Stype.text` column detection.
|
|
91
|
+
``"infer"`` includes inferred :attr:`~Stype.text` columns.
|
|
92
|
+
``"drop"`` omits inferred :attr:`~Stype.text` columns.
|
|
93
|
+
id: The :attr:`~Stype.id` detection policy.
|
|
94
|
+
``"off"`` disables :attr:`~Stype.id` column detection.
|
|
95
|
+
``"infer"`` includes inferred :attr:`~Stype.id` columns.
|
|
96
|
+
``"drop"`` omits inferred :attr:`~Stype.id` columns.
|
|
97
|
+
unsupported: How to handle unsupported dtypes.
|
|
98
|
+
``"error"`` raises a :class:`TypeError`.
|
|
99
|
+
``"warn"`` emits a warning and omits the column.
|
|
100
|
+
``"drop"`` omits the column silently.
|
|
101
|
+
|
|
102
|
+
Returns:
|
|
103
|
+
Dictionary mapping column names to inferred semantic type.
|
|
104
|
+
"""
|
|
105
|
+
overrides = overrides or {}
|
|
106
|
+
|
|
107
|
+
fn: (
|
|
108
|
+
Callable[
|
|
109
|
+
[str, object, Policy, Policy, Literal["off", "infer"]],
|
|
110
|
+
Stype | None,
|
|
111
|
+
]
|
|
112
|
+
| None
|
|
113
|
+
) = None
|
|
114
|
+
columns: Iterable[tuple[Hashable, object]] | None = None
|
|
115
|
+
if isinstance(table, pa.Table):
|
|
116
|
+
fn = _infer_arrow_stype
|
|
117
|
+
columns = zip(table.column_names, table.columns)
|
|
118
|
+
if fn is None and importlib.util.find_spec("pandas") is not None:
|
|
119
|
+
import pandas as pd
|
|
120
|
+
|
|
121
|
+
if isinstance(table, pd.DataFrame):
|
|
122
|
+
fn = _infer_pandas_stype
|
|
123
|
+
columns = table.items()
|
|
124
|
+
if fn is None and importlib.util.find_spec("cudf") is not None:
|
|
125
|
+
import cudf
|
|
126
|
+
|
|
127
|
+
if isinstance(table, cudf.DataFrame):
|
|
128
|
+
fn = _infer_cudf_stype
|
|
129
|
+
columns = table.items()
|
|
130
|
+
|
|
131
|
+
if fn is None or columns is None:
|
|
132
|
+
raise TypeError(
|
|
133
|
+
f"Expected input to be a 'pandas.DataFrame', 'pyarrow.Table', "
|
|
134
|
+
f"or 'cudf.DataFrame' (got {type(table).__name__!r})"
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
stypes = {}
|
|
138
|
+
unsupported_columns: list[str] = []
|
|
139
|
+
for name, column in columns:
|
|
140
|
+
assert isinstance(name, str)
|
|
141
|
+
if name in overrides:
|
|
142
|
+
stypes[name] = Stype(overrides[name])
|
|
143
|
+
continue
|
|
144
|
+
|
|
145
|
+
try:
|
|
146
|
+
stype = fn(name, column, text, id, _low_cardinality)
|
|
147
|
+
except TypeError:
|
|
148
|
+
if unsupported == "error":
|
|
149
|
+
raise
|
|
150
|
+
if unsupported == "warn":
|
|
151
|
+
unsupported_columns.append(name)
|
|
152
|
+
continue
|
|
153
|
+
|
|
154
|
+
if stype is not None:
|
|
155
|
+
stypes[name] = stype
|
|
156
|
+
|
|
157
|
+
if len(unsupported_columns) == 1:
|
|
158
|
+
warnings.warn(
|
|
159
|
+
f"Dropping unsupported column {unsupported_columns[0]!r}",
|
|
160
|
+
stacklevel=2,
|
|
161
|
+
)
|
|
162
|
+
elif len(unsupported_columns) > 1:
|
|
163
|
+
warnings.warn(
|
|
164
|
+
f"Dropping unsupported columns {unsupported_columns}",
|
|
165
|
+
stacklevel=2,
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
return stypes
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# Helpers #####################################################################
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _infer_arrow_stype(
|
|
175
|
+
name: str,
|
|
176
|
+
array: object,
|
|
177
|
+
text: Policy,
|
|
178
|
+
id: Policy,
|
|
179
|
+
low_cardinality: Literal["off", "infer"],
|
|
180
|
+
) -> Stype | None:
|
|
181
|
+
assert isinstance(array, pa.Array | pa.ChunkedArray)
|
|
182
|
+
dtype = array.type
|
|
183
|
+
|
|
184
|
+
if (
|
|
185
|
+
id != "off"
|
|
186
|
+
and (
|
|
187
|
+
pa.types.is_integer(dtype)
|
|
188
|
+
or pa.types.is_string(dtype)
|
|
189
|
+
or pa.types.is_large_string(dtype)
|
|
190
|
+
)
|
|
191
|
+
and _has_id_token(name)
|
|
192
|
+
):
|
|
193
|
+
return None if id == "drop" else Stype.id
|
|
194
|
+
|
|
195
|
+
if (
|
|
196
|
+
pa.types.is_integer(dtype)
|
|
197
|
+
or pa.types.is_floating(dtype)
|
|
198
|
+
or pa.types.is_decimal(dtype)
|
|
199
|
+
):
|
|
200
|
+
if low_cardinality != "off" and _is_arrow_low_cardinality(array):
|
|
201
|
+
return Stype.categorical
|
|
202
|
+
return Stype.numerical
|
|
203
|
+
|
|
204
|
+
if pa.types.is_boolean(dtype) or pa.types.is_dictionary(dtype):
|
|
205
|
+
return Stype.categorical
|
|
206
|
+
|
|
207
|
+
if pa.types.is_string(dtype) or pa.types.is_large_string(dtype):
|
|
208
|
+
if text != "off" and _is_arrow_text(array):
|
|
209
|
+
return None if text == "drop" else Stype.text
|
|
210
|
+
return Stype.categorical
|
|
211
|
+
|
|
212
|
+
if pa.types.is_timestamp(dtype) or pa.types.is_date(dtype):
|
|
213
|
+
return Stype.datetime
|
|
214
|
+
|
|
215
|
+
raise TypeError(f"Unsupported arrow type '{dtype}' for column {name!r}")
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _infer_pandas_stype(
|
|
219
|
+
name: str,
|
|
220
|
+
ser: object,
|
|
221
|
+
text: Policy,
|
|
222
|
+
id: Policy,
|
|
223
|
+
low_cardinality: Literal["off", "infer"],
|
|
224
|
+
) -> Stype | None:
|
|
225
|
+
import pandas as pd
|
|
226
|
+
from pandas.api.types import (
|
|
227
|
+
infer_dtype,
|
|
228
|
+
is_bool_dtype,
|
|
229
|
+
is_datetime64_any_dtype,
|
|
230
|
+
is_float_dtype,
|
|
231
|
+
is_integer_dtype,
|
|
232
|
+
is_object_dtype,
|
|
233
|
+
is_string_dtype,
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
assert isinstance(ser, pd.Series)
|
|
237
|
+
dtype = ser.dtype
|
|
238
|
+
|
|
239
|
+
is_string = is_string_dtype(dtype)
|
|
240
|
+
if is_object_dtype(dtype):
|
|
241
|
+
is_string = infer_dtype(ser, skipna=True) == "string"
|
|
242
|
+
|
|
243
|
+
if (
|
|
244
|
+
id != "off"
|
|
245
|
+
and (is_integer_dtype(dtype) or is_string)
|
|
246
|
+
and _has_id_token(name)
|
|
247
|
+
):
|
|
248
|
+
return None if id == "drop" else Stype.id
|
|
249
|
+
|
|
250
|
+
if is_integer_dtype(dtype) or is_float_dtype(dtype):
|
|
251
|
+
if low_cardinality != "off" and _is_series_low_cardinality(ser):
|
|
252
|
+
return Stype.categorical
|
|
253
|
+
return Stype.numerical
|
|
254
|
+
|
|
255
|
+
if is_bool_dtype(dtype) or isinstance(dtype, pd.CategoricalDtype):
|
|
256
|
+
return Stype.categorical
|
|
257
|
+
|
|
258
|
+
if is_string:
|
|
259
|
+
if text != "off" and _is_arrow_text(pa.array(ser, from_pandas=True)):
|
|
260
|
+
return None if text == "drop" else Stype.text
|
|
261
|
+
return Stype.categorical
|
|
262
|
+
|
|
263
|
+
if is_datetime64_any_dtype(dtype) or isinstance(dtype, pd.PeriodDtype):
|
|
264
|
+
return Stype.datetime
|
|
265
|
+
|
|
266
|
+
raise TypeError(f"Unsupported pandas type '{dtype}' for column {name!r}")
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _infer_cudf_stype(
|
|
270
|
+
name: str,
|
|
271
|
+
ser: object,
|
|
272
|
+
text: Policy,
|
|
273
|
+
id: Policy,
|
|
274
|
+
low_cardinality: Literal["off", "infer"],
|
|
275
|
+
) -> Stype | None:
|
|
276
|
+
import cudf
|
|
277
|
+
from cudf.api.types import (
|
|
278
|
+
is_bool_dtype,
|
|
279
|
+
is_datetime64_any_dtype,
|
|
280
|
+
is_decimal_dtype,
|
|
281
|
+
is_float_dtype,
|
|
282
|
+
is_integer_dtype,
|
|
283
|
+
is_string_dtype,
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
assert isinstance(ser, cudf.Series)
|
|
287
|
+
dtype = ser.dtype
|
|
288
|
+
|
|
289
|
+
if (
|
|
290
|
+
id != "off"
|
|
291
|
+
and (is_integer_dtype(dtype) or is_string_dtype(dtype))
|
|
292
|
+
and _has_id_token(name)
|
|
293
|
+
):
|
|
294
|
+
return None if id == "drop" else Stype.id
|
|
295
|
+
|
|
296
|
+
if (
|
|
297
|
+
is_integer_dtype(dtype)
|
|
298
|
+
or is_float_dtype(dtype)
|
|
299
|
+
or is_decimal_dtype(dtype)
|
|
300
|
+
):
|
|
301
|
+
if low_cardinality != "off" and _is_series_low_cardinality(ser):
|
|
302
|
+
return Stype.categorical
|
|
303
|
+
return Stype.numerical
|
|
304
|
+
|
|
305
|
+
if is_bool_dtype(dtype) or isinstance(dtype, cudf.CategoricalDtype):
|
|
306
|
+
return Stype.categorical
|
|
307
|
+
|
|
308
|
+
if is_string_dtype(dtype):
|
|
309
|
+
if text != "off" and _is_cudf_text(ser):
|
|
310
|
+
return None if text == "drop" else Stype.text
|
|
311
|
+
return Stype.categorical
|
|
312
|
+
|
|
313
|
+
if is_datetime64_any_dtype(dtype):
|
|
314
|
+
return Stype.datetime
|
|
315
|
+
|
|
316
|
+
raise TypeError(f"Unsupported cudf type '{dtype}' for column {name!r}")
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
# Tokenize strings on separators (non-letters/digits) and camelCase boundaries:
|
|
320
|
+
_WORD_PATTERN = re.compile(r"[^a-zA-Z0-9]+|(?<=[a-z0-9])(?=[A-Z])")
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _has_id_token(name: str) -> bool:
|
|
324
|
+
return "id" in (word.lower() for word in _WORD_PATTERN.split(name))
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
_TEXT_MIN_UNIQUE_VALUES = 200
|
|
328
|
+
_TEXT_MIN_UNIQUE_RATIO = 0.05
|
|
329
|
+
_TEXT_MIN_AVERAGE_WORD_COUNT = 3
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _is_arrow_text(array: pa.Array | pa.ChunkedArray) -> bool:
|
|
333
|
+
if (num_values := len(array) - array.null_count) == 0:
|
|
334
|
+
return False
|
|
335
|
+
|
|
336
|
+
array = pc.call_function("drop_null", [array])
|
|
337
|
+
num_unique = pc.call_function("count_distinct", [array]).as_py()
|
|
338
|
+
if num_unique < _TEXT_MIN_UNIQUE_VALUES:
|
|
339
|
+
return False
|
|
340
|
+
if num_unique / num_values <= _TEXT_MIN_UNIQUE_RATIO:
|
|
341
|
+
return False
|
|
342
|
+
|
|
343
|
+
unique = pc.call_function("unique", [array])
|
|
344
|
+
tokens = pc.call_function("utf8_split_whitespace", [unique])
|
|
345
|
+
words = pc.call_function("list_value_length", [tokens])
|
|
346
|
+
avg_words = pc.call_function("mean", [words]).as_py()
|
|
347
|
+
return avg_words >= _TEXT_MIN_AVERAGE_WORD_COUNT
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _is_cudf_text(ser: cudf.Series) -> bool:
|
|
351
|
+
if (num_values := ser.count()) == 0:
|
|
352
|
+
return False
|
|
353
|
+
|
|
354
|
+
if (num_unique := ser.nunique(dropna=True)) < _TEXT_MIN_UNIQUE_VALUES:
|
|
355
|
+
return False
|
|
356
|
+
if num_unique / num_values <= _TEXT_MIN_UNIQUE_RATIO:
|
|
357
|
+
return False
|
|
358
|
+
|
|
359
|
+
unique = ser.dropna().unique()
|
|
360
|
+
avg_words = unique.str.token_count().mean()
|
|
361
|
+
return avg_words >= _TEXT_MIN_AVERAGE_WORD_COUNT
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
_LOW_CARDINALITY_MIN_ROWS = 151
|
|
365
|
+
_LOW_CARDINALITY_MAX_UNIQUE_VALUES = 3
|
|
366
|
+
_LOW_CARDINALITY_PREFIX_ROWS = 1024
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _is_arrow_low_cardinality(array: pa.Array | pa.ChunkedArray) -> bool:
|
|
370
|
+
if len(array) < _LOW_CARDINALITY_MIN_ROWS:
|
|
371
|
+
return False
|
|
372
|
+
|
|
373
|
+
# A prefix holds a subset of the distinct values, so most columns are
|
|
374
|
+
# ruled out without a full pass.
|
|
375
|
+
options = pc.CountOptions(mode="all")
|
|
376
|
+
prefix = array.slice(0, _LOW_CARDINALITY_PREFIX_ROWS)
|
|
377
|
+
num_unique = pc.call_function("count_distinct", [prefix], options).as_py()
|
|
378
|
+
if num_unique > _LOW_CARDINALITY_MAX_UNIQUE_VALUES:
|
|
379
|
+
return False
|
|
380
|
+
|
|
381
|
+
num_unique = pc.call_function("count_distinct", [array], options).as_py()
|
|
382
|
+
return 1 < num_unique <= _LOW_CARDINALITY_MAX_UNIQUE_VALUES
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _is_series_low_cardinality(ser: pd.Series | cudf.Series) -> bool:
|
|
386
|
+
if len(ser) < _LOW_CARDINALITY_MIN_ROWS:
|
|
387
|
+
return False
|
|
388
|
+
|
|
389
|
+
# A prefix holds a subset of the distinct values, so most columns are
|
|
390
|
+
# ruled out without a full pass.
|
|
391
|
+
prefix = ser.iloc[:_LOW_CARDINALITY_PREFIX_ROWS]
|
|
392
|
+
if prefix.nunique(dropna=False) > _LOW_CARDINALITY_MAX_UNIQUE_VALUES:
|
|
393
|
+
return False
|
|
394
|
+
|
|
395
|
+
num_unique = ser.nunique(dropna=False)
|
|
396
|
+
return 1 < num_unique <= _LOW_CARDINALITY_MAX_UNIQUE_VALUES
|
sdm/task.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from enum import StrEnum
|
|
7
|
+
from typing import TypeAlias
|
|
8
|
+
|
|
9
|
+
from sdm import Stype, StypeLike
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Task(StrEnum):
|
|
13
|
+
r"""The prediction task type.
|
|
14
|
+
|
|
15
|
+
A task type describes the kind of target a model predicts.
|
|
16
|
+
Possible values are:
|
|
17
|
+
|
|
18
|
+
Attributes:
|
|
19
|
+
classification: Binary, multi-class, or multi-label classification.
|
|
20
|
+
regression: Continuous numerical prediction.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
classification = "classification"
|
|
24
|
+
regression = "regression"
|
|
25
|
+
|
|
26
|
+
@classmethod
|
|
27
|
+
def from_stype(cls, stype: StypeLike) -> Task:
|
|
28
|
+
r"""The prediction task associated with a target semantic type.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
stype: The target semantic type.
|
|
32
|
+
"""
|
|
33
|
+
if stype == Stype.categorical:
|
|
34
|
+
return cls.classification
|
|
35
|
+
if stype == Stype.numerical:
|
|
36
|
+
return cls.regression
|
|
37
|
+
raise ValueError(
|
|
38
|
+
f"Cannot map semantic type {str(stype)!r} to prediction task"
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def stype(self) -> Stype:
|
|
43
|
+
r"""The semantic type associated with a prediction task."""
|
|
44
|
+
if self == Task.classification:
|
|
45
|
+
return Stype.categorical
|
|
46
|
+
assert self == Task.regression
|
|
47
|
+
return Stype.numerical
|
|
48
|
+
|
|
49
|
+
def __repr__(self) -> str:
|
|
50
|
+
return f"{self.__class__.__name__}.{self.name}"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
TaskLike: TypeAlias = Task | str
|
sdm/tensor/__init__.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Structured data containers."""
|
|
5
|
+
|
|
6
|
+
from sdm.tensor.var_len import VarLenTensor
|
|
7
|
+
from sdm.tensor.string import StringTensor
|
|
8
|
+
from sdm.tensor.nullable import NullableTensor
|
|
9
|
+
from sdm.tensor.categorical import CategoricalTensor
|
|
10
|
+
from sdm.tensor.columnar import ColumnarTensor
|
|
11
|
+
from sdm.tensor.table import TableTensor
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"VarLenTensor",
|
|
15
|
+
"StringTensor",
|
|
16
|
+
"NullableTensor",
|
|
17
|
+
"CategoricalTensor",
|
|
18
|
+
"ColumnarTensor",
|
|
19
|
+
"TableTensor",
|
|
20
|
+
]
|