mlbundle 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mlbundle/__init__.py +82 -0
- mlbundle/_base.py +19 -0
- mlbundle/_canonical_work.py +67 -0
- mlbundle/_catboost/__init__.py +159 -0
- mlbundle/_catboost/budget.py +22 -0
- mlbundle/_catboost/flatbuffer.py +104 -0
- mlbundle/_catboost/native.py +244 -0
- mlbundle/_catboost/prediction.py +67 -0
- mlbundle/_catboost/scope.py +109 -0
- mlbundle/_catboost/state.py +62 -0
- mlbundle/_composite/__init__.py +1 -0
- mlbundle/_composite/claims.py +133 -0
- mlbundle/_composite/io.py +158 -0
- mlbundle/_composite/prediction.py +77 -0
- mlbundle/_container_rules.py +73 -0
- mlbundle/_export_manifest.py +78 -0
- mlbundle/_native_io.py +53 -0
- mlbundle/_onnx/__init__.py +115 -0
- mlbundle/_onnx/_wire_runs.py +103 -0
- mlbundle/_onnx/description.py +252 -0
- mlbundle/_onnx/prediction.py +136 -0
- mlbundle/_onnx/protobuf.py +111 -0
- mlbundle/_onnx/session.py +87 -0
- mlbundle/_onnx/signatures.py +63 -0
- mlbundle/_onnx/validation.py +164 -0
- mlbundle/_pipeline_contract.py +104 -0
- mlbundle/_prediction_input.py +63 -0
- mlbundle/_schema_fields.py +68 -0
- mlbundle/_sklearn/__init__.py +183 -0
- mlbundle/_sklearn/_width_checks.py +59 -0
- mlbundle/_sklearn/archive.py +32 -0
- mlbundle/_sklearn/arrays.py +150 -0
- mlbundle/_sklearn/column_width.py +198 -0
- mlbundle/_sklearn/consistency.py +127 -0
- mlbundle/_sklearn/dtypes.py +41 -0
- mlbundle/_sklearn/encoder_width.py +123 -0
- mlbundle/_sklearn/fitted.py +113 -0
- mlbundle/_sklearn/labels.py +52 -0
- mlbundle/_sklearn/nodes.py +75 -0
- mlbundle/_sklearn/parameters.py +94 -0
- mlbundle/_sklearn/payload.py +234 -0
- mlbundle/_sklearn/pipeline_scope.py +141 -0
- mlbundle/_sklearn/polynomial_names.py +31 -0
- mlbundle/_sklearn/prediction.py +119 -0
- mlbundle/_sklearn/prediction_state.py +61 -0
- mlbundle/_sklearn/references.py +110 -0
- mlbundle/_sklearn/serialization.py +80 -0
- mlbundle/_sklearn/state.py +144 -0
- mlbundle/_sklearn/state_fields.json +228 -0
- mlbundle/_sklearn/structure.py +265 -0
- mlbundle/_sklearn/transform_names.py +111 -0
- mlbundle/_sklearn/transform_width.py +270 -0
- mlbundle/_sklearn/transformers.py +69 -0
- mlbundle/_sklearn/trusted.py +89 -0
- mlbundle/_xgboost/__init__.py +262 -0
- mlbundle/_xgboost/arrays.py +47 -0
- mlbundle/_xgboost/categories.py +109 -0
- mlbundle/_xgboost/native.py +193 -0
- mlbundle/_xgboost/prediction.py +59 -0
- mlbundle/_xgboost/snapshots.py +41 -0
- mlbundle/_xgboost/state.py +147 -0
- mlbundle/_xgboost/structure.py +47 -0
- mlbundle/_xgboost/ubjson.py +119 -0
- mlbundle/_zip_directory.py +173 -0
- mlbundle/_zip_members.py +116 -0
- mlbundle/adapters.py +33 -0
- mlbundle/canonical.py +290 -0
- mlbundle/claims.py +124 -0
- mlbundle/cli.py +193 -0
- mlbundle/container.py +93 -0
- mlbundle/contract.py +55 -0
- mlbundle/declarations.py +68 -0
- mlbundle/environment.py +85 -0
- mlbundle/errors.py +92 -0
- mlbundle/inspection.py +30 -0
- mlbundle/integrity.py +30 -0
- mlbundle/limits.py +38 -0
- mlbundle/manifest.py +222 -0
- mlbundle/model_io.py +151 -0
- mlbundle/py.typed +0 -0
- mlbundle/registry.py +103 -0
- mlbundle/results.py +68 -0
- mlbundle/samples.py +71 -0
- mlbundle/schema.py +145 -0
- mlbundle/training.py +104 -0
- mlbundle/writing.py +256 -0
- mlbundle-0.1.0a1.dist-info/METADATA +347 -0
- mlbundle-0.1.0a1.dist-info/RECORD +91 -0
- mlbundle-0.1.0a1.dist-info/WHEEL +4 -0
- mlbundle-0.1.0a1.dist-info/entry_points.txt +3 -0
- mlbundle-0.1.0a1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""Self-contained inference graph validation before checker or runtime loading."""
|
|
2
|
+
|
|
3
|
+
from math import prod
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import onnx
|
|
7
|
+
|
|
8
|
+
from mlbundle.errors import InvalidModelError, LimitExceededError, MLBundleError
|
|
9
|
+
from mlbundle.limits import MAX_FEATURES, ArchiveLimits
|
|
10
|
+
|
|
11
|
+
from .protobuf import validate_wire
|
|
12
|
+
|
|
13
|
+
DOMAINS = frozenset({"", "ai.onnx", "ai.onnx.ml"})
|
|
14
|
+
MAX_NODES = 100_000
|
|
15
|
+
MAX_RANK = 32
|
|
16
|
+
BYTE_CAP = ArchiveLimits().max_member_bytes
|
|
17
|
+
# Storage field, bytes per element, primitive values per element.
|
|
18
|
+
TENSOR_TYPES = {
|
|
19
|
+
1: ("float_data", 4, 1),
|
|
20
|
+
2: ("int32_data", 1, 1),
|
|
21
|
+
3: ("int32_data", 1, 1),
|
|
22
|
+
4: ("int32_data", 2, 1),
|
|
23
|
+
5: ("int32_data", 2, 1),
|
|
24
|
+
6: ("int32_data", 4, 1),
|
|
25
|
+
7: ("int64_data", 8, 1),
|
|
26
|
+
8: ("string_data", 0, 1),
|
|
27
|
+
9: ("int32_data", 1, 1),
|
|
28
|
+
10: ("int32_data", 2, 1),
|
|
29
|
+
11: ("double_data", 8, 1),
|
|
30
|
+
12: ("uint64_data", 4, 1),
|
|
31
|
+
13: ("uint64_data", 8, 1),
|
|
32
|
+
14: ("float_data", 8, 2),
|
|
33
|
+
15: ("double_data", 16, 2),
|
|
34
|
+
16: ("int32_data", 2, 1),
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _dimensions(dims: Any, *, allow_zero: bool) -> int:
|
|
39
|
+
if len(dims) > MAX_RANK or any(
|
|
40
|
+
dimension < (0 if allow_zero else 1) or dimension > BYTE_CAP
|
|
41
|
+
for dimension in dims
|
|
42
|
+
):
|
|
43
|
+
raise LimitExceededError("ONNX tensor dimensions exceed the limit.")
|
|
44
|
+
count = prod(dims)
|
|
45
|
+
if count > BYTE_CAP:
|
|
46
|
+
raise LimitExceededError("ONNX tensor element count exceeds the limit.")
|
|
47
|
+
return count
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _tensor(tensor: Any) -> int:
|
|
51
|
+
if tensor.data_location != onnx.TensorProto.DEFAULT or tensor.external_data:
|
|
52
|
+
raise InvalidModelError("ONNX external tensor data is unsupported.")
|
|
53
|
+
if tensor.HasField("segment") or tensor.data_type not in TENSOR_TYPES:
|
|
54
|
+
raise InvalidModelError("Unsupported ONNX tensor storage.")
|
|
55
|
+
count = _dimensions(tensor.dims, allow_zero=True)
|
|
56
|
+
field, size, factor = TENSOR_TYPES[tensor.data_type]
|
|
57
|
+
if count * max(size, 1) > BYTE_CAP:
|
|
58
|
+
raise LimitExceededError("ONNX tensor storage exceeds the limit.")
|
|
59
|
+
populated = [
|
|
60
|
+
name
|
|
61
|
+
for name in {value[0] for value in TENSOR_TYPES.values()}
|
|
62
|
+
if len(getattr(tensor, name))
|
|
63
|
+
]
|
|
64
|
+
if tensor.HasField("raw_data"):
|
|
65
|
+
valid = size > 0 and not populated and len(tensor.raw_data) == count * size
|
|
66
|
+
else:
|
|
67
|
+
valid = (
|
|
68
|
+
all(name == field for name in populated)
|
|
69
|
+
and len(getattr(tensor, field)) == count * factor
|
|
70
|
+
)
|
|
71
|
+
if not valid:
|
|
72
|
+
raise InvalidModelError("ONNX tensor dimensions and stored values disagree.")
|
|
73
|
+
return max(count * max(size, 1), len(tensor.raw_data))
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _shape(shape: Any) -> None:
|
|
77
|
+
if len(shape.dim) > MAX_RANK:
|
|
78
|
+
raise LimitExceededError("ONNX value rank exceeds the limit.")
|
|
79
|
+
static = []
|
|
80
|
+
for dim in shape.dim:
|
|
81
|
+
if dim.HasField("dim_value"):
|
|
82
|
+
static.append(dim.dim_value)
|
|
83
|
+
elif len(dim.dim_param) > 1024:
|
|
84
|
+
raise LimitExceededError("ONNX dim_param exceeds the 1024-character limit.")
|
|
85
|
+
_dimensions(static, allow_zero=True)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _operator(value: Any) -> None:
|
|
89
|
+
if value.domain not in DOMAINS:
|
|
90
|
+
raise InvalidModelError("Unsupported ONNX operator domain.")
|
|
91
|
+
if value.DESCRIPTOR.name == "NodeProto" and (
|
|
92
|
+
value.overload or value.device_configurations
|
|
93
|
+
):
|
|
94
|
+
raise InvalidModelError("Unsupported ONNX node configuration.")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _sparse(value: Any) -> None:
|
|
98
|
+
count = _dimensions(value.dims, allow_zero=True)
|
|
99
|
+
size = max(TENSOR_TYPES.get(value.values.data_type, (None, BYTE_CAP, 1))[1], 1)
|
|
100
|
+
if count * size > BYTE_CAP:
|
|
101
|
+
raise LimitExceededError("ONNX sparse tensor dense storage exceeds the limit.")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _children(value: Any) -> list[Any]:
|
|
105
|
+
children = []
|
|
106
|
+
for field, child in value.ListFields():
|
|
107
|
+
if field.type == field.TYPE_MESSAGE:
|
|
108
|
+
children.extend(child if field.is_repeated else [child])
|
|
109
|
+
return children
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _visit(model: Any) -> None:
|
|
113
|
+
pending = [model]
|
|
114
|
+
nodes = storage = 0
|
|
115
|
+
while pending:
|
|
116
|
+
value = pending.pop()
|
|
117
|
+
name = value.DESCRIPTOR.name
|
|
118
|
+
if name == "TensorProto":
|
|
119
|
+
storage += _tensor(value)
|
|
120
|
+
if storage > BYTE_CAP:
|
|
121
|
+
raise LimitExceededError(
|
|
122
|
+
"ONNX aggregate tensor storage exceeds the limit."
|
|
123
|
+
)
|
|
124
|
+
elif name == "SparseTensorProto":
|
|
125
|
+
_sparse(value)
|
|
126
|
+
elif name == "TensorShapeProto":
|
|
127
|
+
_shape(value)
|
|
128
|
+
elif name in ("NodeProto", "FunctionProto", "OperatorSetIdProto"):
|
|
129
|
+
_operator(value)
|
|
130
|
+
if name == "NodeProto":
|
|
131
|
+
nodes += 1
|
|
132
|
+
if nodes > MAX_NODES:
|
|
133
|
+
raise LimitExceededError("ONNX node count exceeds the limit.")
|
|
134
|
+
pending.extend(_children(value))
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def parse_model(data: bytes) -> Any:
|
|
138
|
+
try:
|
|
139
|
+
validate_wire(data, onnx.ModelProto.DESCRIPTOR)
|
|
140
|
+
model = onnx.ModelProto()
|
|
141
|
+
model.ParseFromString(data)
|
|
142
|
+
_visit(model)
|
|
143
|
+
if model.training_info or model.functions or model.configuration:
|
|
144
|
+
raise InvalidModelError(
|
|
145
|
+
"ONNX training, local functions and device configuration are unsupported."
|
|
146
|
+
)
|
|
147
|
+
if any(item.key == "ort_config" for item in model.metadata_props):
|
|
148
|
+
raise InvalidModelError(
|
|
149
|
+
"ONNX runtime configuration metadata is unsupported."
|
|
150
|
+
)
|
|
151
|
+
if not 1 <= len(model.graph.input) <= MAX_FEATURES:
|
|
152
|
+
raise InvalidModelError("ONNX requires bounded tabular inputs.")
|
|
153
|
+
initializers = {item.name for item in model.graph.initializer} | {
|
|
154
|
+
item.values.name for item in model.graph.sparse_initializer
|
|
155
|
+
}
|
|
156
|
+
if initializers & {item.name for item in model.graph.input}:
|
|
157
|
+
raise InvalidModelError("ONNX overridable initializers are unsupported.")
|
|
158
|
+
# full_check=False avoids shape inference executing constant propagation.
|
|
159
|
+
onnx.checker.check_model(model, full_check=False)
|
|
160
|
+
return model
|
|
161
|
+
except MLBundleError:
|
|
162
|
+
raise
|
|
163
|
+
except Exception:
|
|
164
|
+
raise InvalidModelError("Invalid ONNX protobuf or graph.") from None
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Core-only composite records; declarations never select Python constructors."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Annotated, Literal
|
|
6
|
+
|
|
7
|
+
from pydantic import Field, JsonValue, field_validator, model_validator
|
|
8
|
+
|
|
9
|
+
from ._base import ContractModel
|
|
10
|
+
from .canonical import canonical_dumps
|
|
11
|
+
from .claims import FeatureNames, ModelClaims, validate_labels
|
|
12
|
+
from .contract import MODEL_MEMBERS, Library, MemberName, SerializationFormat
|
|
13
|
+
from .limits import HYPERPARAMETER_BYTES, MAX_FEATURES
|
|
14
|
+
|
|
15
|
+
StepName = Annotated[str, Field(min_length=1, max_length=256)]
|
|
16
|
+
Digest = Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")]
|
|
17
|
+
OPTIONS = {"steps", "memory", "verbose", "transform_input"}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def valid_step_names(names: list[str]) -> bool:
|
|
21
|
+
return len(set(names)) == len(names) and all(
|
|
22
|
+
"__" not in name and name not in OPTIONS for name in names
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class TransformerInfo(ContractModel):
|
|
27
|
+
member: Literal["model.skops"]
|
|
28
|
+
estimator_class: Literal["Pipeline"]
|
|
29
|
+
hyperparameters: dict[str, JsonValue] = Field(max_length=4096)
|
|
30
|
+
state_sha256: Digest
|
|
31
|
+
|
|
32
|
+
@field_validator("hyperparameters")
|
|
33
|
+
@classmethod
|
|
34
|
+
def bounded_parameters(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
|
|
35
|
+
canonical_dumps(value, max_bytes=HYPERPARAMETER_BYTES, max_depth=16)
|
|
36
|
+
return value
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class NativeComponentInfo(ContractModel):
|
|
40
|
+
library: Literal[Library.XGBOOST, Library.CATBOOST]
|
|
41
|
+
serialization: SerializationFormat
|
|
42
|
+
member: MemberName
|
|
43
|
+
claims: ModelClaims
|
|
44
|
+
|
|
45
|
+
@model_validator(mode="after")
|
|
46
|
+
def native_pairing(self) -> NativeComponentInfo:
|
|
47
|
+
if (self.serialization, self.member) != MODEL_MEMBERS[self.library]:
|
|
48
|
+
raise ValueError("Invalid native component pairing.")
|
|
49
|
+
classes = {
|
|
50
|
+
Library.XGBOOST: {"XGBClassifier", "XGBRegressor"},
|
|
51
|
+
Library.CATBOOST: {"CatBoostClassifier", "CatBoostRegressor"},
|
|
52
|
+
}
|
|
53
|
+
if self.claims.estimator_class not in classes[self.library]:
|
|
54
|
+
raise ValueError("Invalid native component class.")
|
|
55
|
+
if self.claims.onnx is not None:
|
|
56
|
+
raise ValueError("Composite components cannot declare ONNX fields.")
|
|
57
|
+
validate_labels(self.claims, self.claims.class_labels)
|
|
58
|
+
return self
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class TransformedFeatures(ContractModel):
|
|
62
|
+
n_features: int = Field(gt=0, le=MAX_FEATURES)
|
|
63
|
+
names: FeatureNames
|
|
64
|
+
origin: Literal["transformer", "model", "input", "generated"]
|
|
65
|
+
|
|
66
|
+
@model_validator(mode="after")
|
|
67
|
+
def aligned_names(self) -> TransformedFeatures:
|
|
68
|
+
if len(self.names) != self.n_features or len(set(self.names)) != len(
|
|
69
|
+
self.names
|
|
70
|
+
):
|
|
71
|
+
raise ValueError("Invalid transformed names.")
|
|
72
|
+
if self.origin == "generated" and self.names != [
|
|
73
|
+
f"transformed_feature_{i}" for i in range(1, self.n_features + 1)
|
|
74
|
+
]:
|
|
75
|
+
raise ValueError("Invalid generated transformed names.")
|
|
76
|
+
return self
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class PipelineInfo(ContractModel):
|
|
80
|
+
step_names: list[StepName] = Field(min_length=1, max_length=128)
|
|
81
|
+
transformers: TransformerInfo | None
|
|
82
|
+
estimator: NativeComponentInfo
|
|
83
|
+
transformed_features: TransformedFeatures
|
|
84
|
+
|
|
85
|
+
@model_validator(mode="after")
|
|
86
|
+
def consistent_recipe(self) -> PipelineInfo:
|
|
87
|
+
if not valid_step_names(self.step_names):
|
|
88
|
+
raise ValueError("Invalid Pipeline names.")
|
|
89
|
+
if (len(self.step_names) > 1) != (self.transformers is not None):
|
|
90
|
+
raise ValueError("Transformer recipe and steps disagree.")
|
|
91
|
+
if self.transformed_features.n_features != self.estimator.claims.n_features:
|
|
92
|
+
raise ValueError("Composite boundary widths disagree.")
|
|
93
|
+
return self
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def expected_model_members(
|
|
97
|
+
library: Library, pipeline: PipelineInfo | None
|
|
98
|
+
) -> set[MemberName]:
|
|
99
|
+
if pipeline is None:
|
|
100
|
+
return {MODEL_MEMBERS[library][1]}
|
|
101
|
+
members = {pipeline.estimator.member}
|
|
102
|
+
if pipeline.transformers is not None:
|
|
103
|
+
members.add(MemberName.SKOPS)
|
|
104
|
+
return members
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Numeric input checks shared by optional sklearn and XGBoost predictors."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
from scipy.sparse import issparse
|
|
8
|
+
|
|
9
|
+
from .errors import InvalidModelError, MLBundleError, PredictionInputError
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def tabular(
|
|
13
|
+
data: Any, width: int, *, categorical: bool = False, strings: bool = False
|
|
14
|
+
) -> tuple[Any, int]:
|
|
15
|
+
try:
|
|
16
|
+
shape = getattr(data, "shape", None)
|
|
17
|
+
if shape is None:
|
|
18
|
+
data = np.asarray(data)
|
|
19
|
+
shape = data.shape
|
|
20
|
+
if len(shape) != 2 or shape[1] != width:
|
|
21
|
+
raise PredictionInputError(
|
|
22
|
+
"Prediction input must have the model's feature width."
|
|
23
|
+
)
|
|
24
|
+
if issparse(data):
|
|
25
|
+
arrays = [data.data]
|
|
26
|
+
elif categorical and hasattr(data, "dtypes"):
|
|
27
|
+
arrays = [
|
|
28
|
+
np.asarray(data.iloc[:, index])
|
|
29
|
+
for index, dtype in enumerate(data.dtypes)
|
|
30
|
+
if str(dtype) != "category"
|
|
31
|
+
]
|
|
32
|
+
else:
|
|
33
|
+
arrays = [np.asarray(data)]
|
|
34
|
+
for values in arrays:
|
|
35
|
+
_numeric(values, strings=strings)
|
|
36
|
+
return data, int(shape[0])
|
|
37
|
+
except MLBundleError:
|
|
38
|
+
raise
|
|
39
|
+
except Exception:
|
|
40
|
+
raise InvalidModelError("Invalid tabular prediction input.") from None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _numeric(values: np.ndarray, *, strings: bool) -> None:
|
|
44
|
+
if values.dtype.kind == "O":
|
|
45
|
+
if any(
|
|
46
|
+
not (strings and isinstance(value, str))
|
|
47
|
+
and (
|
|
48
|
+
not isinstance(
|
|
49
|
+
value, (int, float, np.integer, np.floating, bool, np.bool_)
|
|
50
|
+
)
|
|
51
|
+
or math.isinf(value)
|
|
52
|
+
)
|
|
53
|
+
for value in values.flat
|
|
54
|
+
):
|
|
55
|
+
raise InvalidModelError(
|
|
56
|
+
"Prediction input must contain numeric values without infinity."
|
|
57
|
+
)
|
|
58
|
+
elif strings and values.dtype.kind == "U":
|
|
59
|
+
return
|
|
60
|
+
elif values.dtype.kind not in "biuf" or np.isinf(values).any():
|
|
61
|
+
raise InvalidModelError(
|
|
62
|
+
"Prediction input must contain numeric values without infinity."
|
|
63
|
+
)
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Closed inference-only mlschema 0.2.1 wire vocabulary; no optional imports."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Annotated, Literal
|
|
6
|
+
|
|
7
|
+
from pydantic import Field, model_validator
|
|
8
|
+
|
|
9
|
+
from ._base import ContractModel, Name
|
|
10
|
+
from .limits import MAX_FEATURES
|
|
11
|
+
|
|
12
|
+
FieldLabel = Annotated[str, Field(min_length=1, max_length=100)]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class FieldBase(ContractModel):
|
|
16
|
+
label: FieldLabel
|
|
17
|
+
required: bool
|
|
18
|
+
mappedTo: Name
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class SimpleField(FieldBase):
|
|
22
|
+
kind: Literal["text", "boolean", "date"]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class NumberField(FieldBase):
|
|
26
|
+
kind: Literal["number"]
|
|
27
|
+
step: Annotated[float, Field(gt=0)]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class CategoryField(FieldBase):
|
|
31
|
+
kind: Literal["category"]
|
|
32
|
+
options: Annotated[list[str], Field(min_length=1, max_length=MAX_FEATURES)]
|
|
33
|
+
|
|
34
|
+
@model_validator(mode="after")
|
|
35
|
+
def unique_options(self) -> CategoryField:
|
|
36
|
+
if len(set(self.options)) != len(self.options):
|
|
37
|
+
raise ValueError("Category options must be unique.")
|
|
38
|
+
return self
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
ScalarField = Annotated[
|
|
42
|
+
SimpleField | NumberField | CategoryField, Field(discriminator="kind")
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class SeriesField(FieldBase):
|
|
47
|
+
kind: Literal["series"]
|
|
48
|
+
field1: ScalarField
|
|
49
|
+
field2: ScalarField
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class OneHotOption(ContractModel):
|
|
53
|
+
label: Name
|
|
54
|
+
value: Name
|
|
55
|
+
mappedTo: Name
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class OneHotField(ContractModel):
|
|
59
|
+
kind: Literal["onehot-category"]
|
|
60
|
+
label: FieldLabel
|
|
61
|
+
required: bool
|
|
62
|
+
options: Annotated[list[OneHotOption], Field(min_length=2, max_length=MAX_FEATURES)]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
SchemaField = Annotated[
|
|
66
|
+
SimpleField | NumberField | CategoryField | SeriesField | OneHotField,
|
|
67
|
+
Field(discriminator="kind"),
|
|
68
|
+
]
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""Skops-only sklearn adapter; importing the base package never imports this module."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, cast
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from pydantic import JsonValue, ValidationError
|
|
7
|
+
from sklearn.base import BaseEstimator, is_classifier, is_regressor
|
|
8
|
+
from sklearn.exceptions import NotFittedError
|
|
9
|
+
from sklearn.multioutput import ClassifierChain, MultiOutputRegressor, RegressorChain
|
|
10
|
+
from sklearn.pipeline import Pipeline
|
|
11
|
+
from sklearn.utils import get_tags
|
|
12
|
+
from sklearn.utils.validation import check_is_fitted
|
|
13
|
+
|
|
14
|
+
from mlbundle.canonical import canonical_value
|
|
15
|
+
from mlbundle.claims import ClassLabels, FeatureNames, ModelClaims
|
|
16
|
+
from mlbundle.contract import Library, MemberName, ModelKind
|
|
17
|
+
from mlbundle.declarations import ExportDeclarations
|
|
18
|
+
from mlbundle.errors import (
|
|
19
|
+
InvalidJSONError,
|
|
20
|
+
InvalidModelError,
|
|
21
|
+
MLBundleError,
|
|
22
|
+
UnsupportedModelError,
|
|
23
|
+
)
|
|
24
|
+
from mlbundle.limits import MAX_CLASSES, MAX_OUTPUTS
|
|
25
|
+
|
|
26
|
+
from .parameters import parameters
|
|
27
|
+
from .payload import validate_payload
|
|
28
|
+
from .prediction import Predictor
|
|
29
|
+
from .serialization import SkopsIO
|
|
30
|
+
from .structure import extract_structure, has_structure, validate_tree_storage
|
|
31
|
+
from .trusted import TRUSTED_TYPES as TRUSTED_TYPES
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _outputs(model: Any, classifier: bool) -> tuple[int, ClassLabels | None]:
|
|
35
|
+
if classifier:
|
|
36
|
+
labels = _labels(model)
|
|
37
|
+
return len(labels), labels
|
|
38
|
+
estimator = model.steps[-1][1] if isinstance(model, Pipeline) else model
|
|
39
|
+
if hasattr(estimator, "n_outputs_"):
|
|
40
|
+
outputs = estimator.n_outputs_
|
|
41
|
+
if (
|
|
42
|
+
not isinstance(outputs, (int, np.integer))
|
|
43
|
+
or isinstance(outputs, (bool, np.bool_))
|
|
44
|
+
or not 1 <= outputs <= MAX_OUTPUTS
|
|
45
|
+
):
|
|
46
|
+
raise InvalidModelError("Invalid regressor output count.")
|
|
47
|
+
return int(outputs), None
|
|
48
|
+
if hasattr(estimator, "coef_"):
|
|
49
|
+
coef = np.asarray(estimator.coef_)
|
|
50
|
+
if coef.ndim not in (1, 2) or coef.shape[-1] != estimator.n_features_in_:
|
|
51
|
+
raise InvalidModelError("Invalid regressor coefficient dimensions.")
|
|
52
|
+
return 1 if coef.ndim == 1 else coef.shape[0], None
|
|
53
|
+
if hasattr(estimator, "_y"):
|
|
54
|
+
target = np.asarray(estimator._y)
|
|
55
|
+
return 1 if target.ndim == 1 else target.shape[1], None
|
|
56
|
+
if isinstance(estimator, (MultiOutputRegressor, RegressorChain)):
|
|
57
|
+
return len(list(estimator.estimators_)), None
|
|
58
|
+
if not get_tags(estimator).target_tags.multi_output:
|
|
59
|
+
return 1, None
|
|
60
|
+
raise UnsupportedModelError(
|
|
61
|
+
reason="Regressor output count cannot be recovered faithfully."
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _labels(model: Any) -> ClassLabels:
|
|
66
|
+
classes = getattr(model, "classes_", None)
|
|
67
|
+
if classes is None:
|
|
68
|
+
raise UnsupportedModelError(reason="Classifier must expose original classes_.")
|
|
69
|
+
outputs = classes if isinstance(classes, list) else [classes]
|
|
70
|
+
if not 1 <= len(outputs) <= MAX_OUTPUTS or any(
|
|
71
|
+
np.asarray(output).ndim != 1 or not 1 <= len(output) <= MAX_CLASSES
|
|
72
|
+
for output in outputs
|
|
73
|
+
):
|
|
74
|
+
raise InvalidModelError("Invalid classifier label dimensions.")
|
|
75
|
+
return cast(ClassLabels, canonical_value(outputs))
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class Adapter:
|
|
79
|
+
library = Library.SKLEARN
|
|
80
|
+
model_member = MemberName.SKOPS
|
|
81
|
+
|
|
82
|
+
def __init__(self) -> None:
|
|
83
|
+
# Each registry call creates fresh synchronous operation state.
|
|
84
|
+
self._io = SkopsIO(self._validate, validate_payload)
|
|
85
|
+
|
|
86
|
+
def _state(self, model: object) -> str:
|
|
87
|
+
return self._io.state(model)
|
|
88
|
+
|
|
89
|
+
def recognizes(self, model: object) -> bool:
|
|
90
|
+
return (
|
|
91
|
+
isinstance(model, BaseEstimator)
|
|
92
|
+
and not type(model).__module__.startswith(("xgboost.", "catboost."))
|
|
93
|
+
and (is_classifier(model) or is_regressor(model))
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
def prepare_export(self, model: object, declarations: ExportDeclarations) -> object:
|
|
97
|
+
if declarations.class_labels is not None:
|
|
98
|
+
raise UnsupportedModelError(
|
|
99
|
+
reason="Sklearn already carries class labels; class_labels is unsupported."
|
|
100
|
+
)
|
|
101
|
+
return model
|
|
102
|
+
|
|
103
|
+
def _validate(self, model: Any) -> None:
|
|
104
|
+
if not self.recognizes(model):
|
|
105
|
+
raise UnsupportedModelError(
|
|
106
|
+
reason="Sklearn payload must be a classifier or regressor."
|
|
107
|
+
)
|
|
108
|
+
try:
|
|
109
|
+
check_is_fitted(model)
|
|
110
|
+
except NotFittedError:
|
|
111
|
+
raise UnsupportedModelError(
|
|
112
|
+
reason="Sklearn estimator must be fitted before export/load."
|
|
113
|
+
) from None
|
|
114
|
+
if is_classifier(model) and not callable(getattr(model, "predict_proba", None)):
|
|
115
|
+
raise UnsupportedModelError(reason="Classifier must expose predict_proba.")
|
|
116
|
+
estimator = model.steps[-1][1] if isinstance(model, Pipeline) else model
|
|
117
|
+
binarizer = getattr(estimator, "_label_binarizer", None)
|
|
118
|
+
if (
|
|
119
|
+
isinstance(estimator, ClassifierChain)
|
|
120
|
+
or getattr(estimator, "multilabel_", False)
|
|
121
|
+
or getattr(binarizer, "y_type_", None) == "multilabel-indicator"
|
|
122
|
+
):
|
|
123
|
+
raise UnsupportedModelError(
|
|
124
|
+
reason="Multilabel/chain probabilities cannot satisfy the output-major contract."
|
|
125
|
+
)
|
|
126
|
+
if not hasattr(model, "n_features_in_"):
|
|
127
|
+
raise UnsupportedModelError(
|
|
128
|
+
reason="Estimator must expose fitted n_features_in_."
|
|
129
|
+
)
|
|
130
|
+
validate_tree_storage(model)
|
|
131
|
+
|
|
132
|
+
def serialize(self, model: object) -> bytes:
|
|
133
|
+
return self._io.serialize(model)
|
|
134
|
+
|
|
135
|
+
def deserialize(self, data: bytes) -> object:
|
|
136
|
+
return self._io.deserialize(data)
|
|
137
|
+
|
|
138
|
+
def extract_claims(self, model: Any) -> ModelClaims:
|
|
139
|
+
try:
|
|
140
|
+
self._validate(model)
|
|
141
|
+
classifier = is_classifier(model)
|
|
142
|
+
outputs, labels = _outputs(model, classifier)
|
|
143
|
+
names = getattr(model, "feature_names_in_", None)
|
|
144
|
+
if names is not None and (
|
|
145
|
+
np.asarray(names).ndim != 1 or len(names) != model.n_features_in_
|
|
146
|
+
):
|
|
147
|
+
raise InvalidModelError("Invalid model feature-name dimensions.")
|
|
148
|
+
return ModelClaims(
|
|
149
|
+
estimator_class=type(model).__name__,
|
|
150
|
+
kind=ModelKind.CLASSIFIER if classifier else ModelKind.REGRESSOR,
|
|
151
|
+
hyperparameters=parameters(model),
|
|
152
|
+
state_sha256=self._state(model) if not has_structure(model) else None,
|
|
153
|
+
n_features=int(model.n_features_in_),
|
|
154
|
+
n_classes=None
|
|
155
|
+
if labels is None
|
|
156
|
+
else [len(output) for output in labels],
|
|
157
|
+
n_outputs=outputs,
|
|
158
|
+
feature_names=None
|
|
159
|
+
if names is None
|
|
160
|
+
else cast(FeatureNames, canonical_value(names)),
|
|
161
|
+
feature_dtypes=None,
|
|
162
|
+
class_labels=labels,
|
|
163
|
+
)
|
|
164
|
+
except (ValidationError, InvalidJSONError):
|
|
165
|
+
raise UnsupportedModelError(
|
|
166
|
+
reason="Model claims cannot be represented faithfully."
|
|
167
|
+
) from None
|
|
168
|
+
except MLBundleError:
|
|
169
|
+
raise
|
|
170
|
+
except Exception:
|
|
171
|
+
raise InvalidModelError("Invalid sklearn model claims.") from None
|
|
172
|
+
|
|
173
|
+
def extract_structure(self, model: object) -> dict[str, JsonValue] | None:
|
|
174
|
+
try:
|
|
175
|
+
self._validate(model)
|
|
176
|
+
return extract_structure(model, state_sha256=self._state(model))
|
|
177
|
+
except MLBundleError:
|
|
178
|
+
raise
|
|
179
|
+
except Exception:
|
|
180
|
+
raise InvalidModelError("Invalid sklearn model structure.") from None
|
|
181
|
+
|
|
182
|
+
def predictor(self, model: object) -> Predictor:
|
|
183
|
+
return Predictor(model, self.extract_claims(model))
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Strict fitted dimensions shared by the closed width validators."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
from mlbundle.errors import InvalidModelError
|
|
8
|
+
from mlbundle.limits import MAX_FEATURES
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def width(value: Any, *, zero: bool = False) -> int:
|
|
12
|
+
if (
|
|
13
|
+
isinstance(value, (bool, np.bool_))
|
|
14
|
+
or not isinstance(value, (int, np.integer))
|
|
15
|
+
or not (0 if zero else 1) <= value <= MAX_FEATURES
|
|
16
|
+
):
|
|
17
|
+
raise InvalidModelError("Invalid transformer feature width.")
|
|
18
|
+
return int(value)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def dimensions(condition: bool) -> None:
|
|
22
|
+
if not condition:
|
|
23
|
+
raise InvalidModelError("Transformer fitted dimensions disagree.")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def vector(
|
|
27
|
+
model: Any, name: str, count: int, *, required: bool = True
|
|
28
|
+
) -> np.ndarray | None:
|
|
29
|
+
value = getattr(model, name, None)
|
|
30
|
+
if value is None:
|
|
31
|
+
dimensions(not required)
|
|
32
|
+
return None
|
|
33
|
+
result = np.asarray(value)
|
|
34
|
+
dimensions(result.shape == (count,))
|
|
35
|
+
return result
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def input_width(model: Any, count: int) -> None:
|
|
39
|
+
native = getattr(model, "n_features_in_", None)
|
|
40
|
+
if native is not None and width(native) != count:
|
|
41
|
+
raise InvalidModelError("Transformer input widths disagree.")
|
|
42
|
+
names = getattr(model, "feature_names_in_", None)
|
|
43
|
+
if names is not None:
|
|
44
|
+
values = np.asarray(names)
|
|
45
|
+
dimensions(
|
|
46
|
+
values.shape == (count,)
|
|
47
|
+
and all(type(x) is str and 1 <= len(x) <= 1024 for x in values)
|
|
48
|
+
and len(set(values)) == count
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def indices(value: Any, count: int) -> list[int]:
|
|
53
|
+
array = np.asarray(value)
|
|
54
|
+
dimensions(array.ndim == 1 and array.dtype.kind in "iu" and len(array) <= count)
|
|
55
|
+
result = [int(item) for item in array]
|
|
56
|
+
dimensions(
|
|
57
|
+
all(0 <= item < count for item in result) and len(set(result)) == len(result)
|
|
58
|
+
)
|
|
59
|
+
return result
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Bound nested ZIP directory work before ZipFile creates entry objects."""
|
|
2
|
+
|
|
3
|
+
import stat
|
|
4
|
+
from io import BytesIO
|
|
5
|
+
from zipfile import ZipFile
|
|
6
|
+
|
|
7
|
+
from mlbundle._zip_directory import CENTRAL, _directory_bounds, unpack
|
|
8
|
+
from mlbundle.errors import InvalidContainerError, InvalidModelError
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def bounded_zip(data: bytes, max_files: int) -> ZipFile:
|
|
12
|
+
try:
|
|
13
|
+
offset, size, count = _directory_bounds(data)
|
|
14
|
+
except InvalidContainerError:
|
|
15
|
+
raise InvalidModelError("Invalid nested skops archive.") from None
|
|
16
|
+
if count > max_files:
|
|
17
|
+
raise InvalidModelError("Skops nested member count exceeds the limit.")
|
|
18
|
+
position = offset
|
|
19
|
+
boundary = offset + size
|
|
20
|
+
for _ in range(count):
|
|
21
|
+
record = unpack(CENTRAL, data, position)
|
|
22
|
+
if record[0] != b"PK\x01\x02" or stat.S_IFMT(record[15] >> 16) not in (
|
|
23
|
+
0,
|
|
24
|
+
stat.S_IFREG,
|
|
25
|
+
):
|
|
26
|
+
raise InvalidModelError("Invalid skops nested ZIP directory.")
|
|
27
|
+
position += CENTRAL.size + sum(record[10:13])
|
|
28
|
+
if position > boundary:
|
|
29
|
+
raise InvalidModelError("Invalid skops nested ZIP directory.")
|
|
30
|
+
if position != boundary:
|
|
31
|
+
raise InvalidModelError("Invalid skops nested ZIP member count.")
|
|
32
|
+
return ZipFile(BytesIO(data))
|