mlbundle 0.1.0a1__py3-none-any.whl

mlbundle/_sklearn/fitted.py

@@ -0,0 +1,113 @@
+"""Fitted dimensions that inference uses must agree with displayed counts."""
+
+from typing import Any
+
+import numpy as np
+from sklearn.dummy import DummyRegressor
+from sklearn.neural_network import MLPClassifier, MLPRegressor
+from sklearn.svm._base import BaseLibSVM
+
+from mlbundle.errors import InvalidModelError
+
+
+def validate_fitted(model: Any, width: int | None) -> None:
+    _statistics(model, width)
+    _dummy(model)
+    if width is None:
+        return
+    if hasattr(model, "_fit_X") and hasattr(model, "_y"):
+        rows = model._fit_X.shape[0]
+        if (
+            model._fit_X.shape[1] != width
+            or len(model._y) != rows
+            or getattr(model, "n_samples_fit_", rows) != rows
+        ):
+            raise InvalidModelError("Neighbor fitted dimensions disagree.")
+    if isinstance(model, (MLPClassifier, MLPRegressor)) and hasattr(model, "coefs_"):
+        _mlp(model, width)
+
+
+def _mlp(model: Any, width: int) -> None:
+    sizes = [width, *np.atleast_1d(model.hidden_layer_sizes).tolist()]
+    sizes.append(model.n_outputs_)
+    if (
+        model.n_layers_ != len(sizes)
+        or len(model.coefs_) != len(sizes) - 1
+        or len(model.intercepts_) != len(model.coefs_)
+        or any(
+            coef.shape != (left, right) or intercept.shape != (right,)
+            for coef, intercept, left, right in zip(
+                model.coefs_, model.intercepts_, sizes, sizes[1:], strict=False
+            )
+        )
+    ):
+        raise InvalidModelError("MLP fitted layer dimensions disagree.")
+    if isinstance(model, MLPClassifier):
+        classes = len(model.classes_)
+        if (
+            classes < 2
+            or model.n_outputs_ != (1 if classes <= 2 else classes)
+            or model.out_activation_ != ("logistic" if classes <= 2 else "softmax")
+        ):
+            raise InvalidModelError("MLP class count and output activation disagree.")
+    elif model.out_activation_ != ("exp" if model.loss == "poisson" else "identity"):
+        raise InvalidModelError("MLP regression loss and output activation disagree.")
+
+
+def _statistics(model: Any, width: int | None) -> None:
+    if not hasattr(model, "classes_") or isinstance(model.classes_, list):
+        return
+    classes = len(model.classes_)
+    if width is not None and hasattr(model, "coef_"):
+        _coefficient(model, classes, width)
+    for name in ("theta_", "var_", "means_", "centroids_"):
+        if (
+            width is not None
+            and hasattr(model, name)
+            and np.asarray(getattr(model, name)).shape
+            != (
+                classes,
+                width,
+            )
+        ):
+            raise InvalidModelError(
+                "Classifier fitted statistics and dimensions disagree."
+            )
+    for name in ("class_prior_", "class_log_prior_", "class_count_"):
+        if hasattr(model, name) and np.asarray(getattr(model, name)).shape != (
+            classes,
+        ):
+            raise InvalidModelError(
+                "Classifier fitted priors and class counts disagree."
+            )
+    if hasattr(model, "feature_log_prob_"):
+        values = model.feature_log_prob_
+        shapes = (
+            [np.asarray(value).shape for value in values]
+            if isinstance(values, list)
+            else [np.asarray(values).shape]
+        )
+        if any(len(shape) != 2 or shape[0] != classes for shape in shapes):
+            raise InvalidModelError("Classifier likelihoods and class counts disagree.")
+        if (isinstance(values, list) and len(values) != width) or (
+            not isinstance(values, list) and shapes[0][1] != width
+        ):
+            raise InvalidModelError(
+                "Classifier likelihoods and feature counts disagree."
+            )
+
+
+def _coefficient(model: Any, classes: int, width: int) -> None:
+    if isinstance(model, BaseLibSVM):
+        return
+    if np.asarray(model.coef_).shape != (1 if classes <= 2 else classes, width):
+        raise InvalidModelError("Classifier coefficients and class counts disagree.")
+
+
+def _dummy(model: Any) -> None:
+    if (
+        isinstance(model, DummyRegressor)
+        and hasattr(model, "constant_")
+        and np.asarray(model.constant_).shape != (1, model.n_outputs_)
+    ):
+        raise InvalidModelError("Dummy regressor constant and output counts disagree.")

mlbundle/_sklearn/labels.py

@@ -0,0 +1,52 @@
+"""One label/count check for every fitted classifier, including nested models."""
+
+from typing import Any, cast
+
+import numpy as np
+from pydantic import ConfigDict, TypeAdapter
+from sklearn.neural_network import MLPClassifier
+
+from mlbundle.canonical import canonical_dumps, canonical_value
+from mlbundle.claims import ClassLabels, validate_label_sets
+from mlbundle.errors import InvalidModelError
+
+# The pinned pre-commit Pyright predates Annotated aliases as TypeForm arguments.
+_LABELS = TypeAdapter(
+    cast(Any, ClassLabels), config=ConfigDict(strict=True, allow_inf_nan=False)
+)
+
+
+def validate_classes(model: Any) -> None:
+    classes = model.classes_ if isinstance(model.classes_, list) else [model.classes_]
+    if not classes or any(np.asarray(labels).ndim != 1 for labels in classes):
+        raise InvalidModelError("Invalid classifier label dimensions.")
+    counts = np.asarray([len(labels) for labels in classes])
+    labels = _LABELS.validate_python(canonical_value(classes))
+    validate_label_sets(labels)
+    binarizer = getattr(model, "_label_binarizer", None)
+    if binarizer is not None and canonical_dumps(model.classes_) != canonical_dumps(
+        binarizer.classes_
+    ):
+        raise InvalidModelError("Classifier and label binarizer classes disagree.")
+    _outputs(model, len(classes))
+    if hasattr(model, "n_classes_"):
+        native = np.asarray(model.n_classes_)
+        if (
+            native.dtype.kind not in "iu"
+            or native.ndim != (0 if len(classes) == 1 else 1)
+            or not np.array_equal(np.atleast_1d(native), counts)
+        ):
+            raise InvalidModelError(
+                "Classifier labels and native class counts disagree."
+            )
+    if hasattr(model, "tree_") and not np.array_equal(model.tree_.n_classes, counts):
+        raise InvalidModelError("Classifier labels and tree class counts disagree.")
+
+
+def _outputs(model: Any, count: int) -> None:
+    if (
+        not isinstance(model, MLPClassifier)
+        and hasattr(model, "n_outputs_")
+        and model.n_outputs_ != count
+    ):
+        raise InvalidModelError("Classifier labels and native output counts disagree.")

mlbundle/_sklearn/nodes.py

@@ -0,0 +1,75 @@
+"""Closed skops node layouts: no unused state, loaders or hidden child objects."""
+
+from typing import Any
+
+from mlbundle.errors import InvalidModelError, UnsupportedModelError
+
+from .trusted import FUNCTIONS, OPTIMIZER_TYPES, SCALAR_NAMES
+
+FIELDS = {
+    "ObjectNode": {"content"},
+    "TreeNode": {"content", "__reduce__"},
+    "LossNode": {"content", "__reduce__"},
+    "DictNode": {"content", "key_types"},
+    "ListNode": {"content"},
+    "TupleNode": {"content"},
+    "SetNode": {"content"},
+    "JsonNode": {"content", "is_json"},
+    "TypeNode": set(),
+    "FunctionNode": set(),
+    "CachedNode": set(),
+    "NdArrayNode": {"type", "file", "content", "shape"},
+    "SparseMatrixNode": {"type", "file"},
+    "RandomStateNode": {"content"},
+    "RandomGeneratorNode": {"content"},
+    "MaskedArrayNode": {"content"},
+    "DTypeNode": {"content"},
+    "SliceNode": {"content"},
+    "ConstructorFromReduceNode": {"content"},
+    "BytesNode": {"file"},
+    "BytearrayNode": {"file"},
+}
+COMMON = {"__class__", "__module__", "__loader__", "__id__"}
+
+
+def validate_nodes(nodes: list[dict[str, Any]], root: Any) -> None:
+    for node in nodes:
+        loader = node["__loader__"]
+        name = node.get("__module__", "") + "." + node.get("__class__", "")
+        _special_loader(node, name, loader)
+        if name in OPTIMIZER_TYPES and loader not in ("ObjectNode", "CachedNode"):
+            raise InvalidModelError("Invalid skops optimizer loader.")
+        if (
+            node.get("__module__") == "sklearn.tree._tree"
+            and node.get("__class__") == "Tree"
+            and loader not in ("TreeNode", "CachedNode")
+        ):
+            raise InvalidModelError("Invalid skops native tree loader.")
+        if loader not in FIELDS:
+            raise UnsupportedModelError((name,))
+        allowed = (
+            COMMON
+            | FIELDS[loader]
+            | ({"protocol", "_skops_version"} if node is root else set())
+        )
+        if set(node) - allowed:
+            raise InvalidModelError("Unknown skops node state.")
+        if loader in ("TreeNode", "LossNode") and set(node["__reduce__"]) != {"args"}:
+            raise InvalidModelError("Unknown skops constructor state.")
+
+
+def _special_loader(node, name, loader) -> None:
+    if name in FUNCTIONS and loader not in ("FunctionNode", "CachedNode"):
+        raise InvalidModelError("Invalid skops function loader.")
+    if loader == "FunctionNode" and name not in FUNCTIONS:
+        raise UnsupportedModelError((name,))
+    if name == "numpy.dtype" and loader not in ("DTypeNode", "CachedNode"):
+        raise InvalidModelError("Invalid skops dtype loader.")
+    if loader == "DTypeNode" and name != "numpy.dtype":
+        raise InvalidModelError("Invalid skops dtype identity.")
+    if (
+        loader == "TypeNode"
+        and node.get("__module__") == "numpy"
+        and name not in SCALAR_NAMES
+    ):
+        raise UnsupportedModelError((name,))

mlbundle/_sklearn/parameters.py

@@ -0,0 +1,94 @@
+"""Deterministic sklearn parameters; these records never select loading code."""
+
+from typing import cast
+
+import numpy as np
+from pydantic import JsonValue
+from sklearn.base import BaseEstimator
+
+from mlbundle.canonical import canonical_value
+from mlbundle.errors import LimitExceededError, UnsupportedModelError
+from mlbundle.limits import HYPERPARAMETER_BYTES
+
+from .trusted import dtype_identifier, reference_identifier
+
+MAX_VISITS = 100_000
+
+
+def _parameter(
+    value: object,
+    ancestors: frozenset[int],
+    budget: list[int] | None = None,
+    *,
+    estimator_ref: bool = False,
+) -> JsonValue:
+    budget = [0] if budget is None else budget
+    budget[0] += 1
+    if budget[0] > MAX_VISITS or len(ancestors) > 16:
+        raise LimitExceededError("Sklearn parameter work exceeds the limit.")
+    if id(value) in ancestors:
+        raise UnsupportedModelError(reason="Cyclic hyperparameters are unsupported.")
+    path = ancestors | {id(value)}
+    if isinstance(value, BaseEstimator):
+        if not type(value).__module__.startswith("sklearn."):
+            raise UnsupportedModelError(
+                (f"{type(value).__module__}.{type(value).__name__}",)
+            )
+        return _estimator(value, path, budget, estimator_ref)
+    if isinstance(value, np.random.RandomState):
+        return {"$random_state": canonical_value(value.get_state())}
+    if isinstance(value, dict):
+        if any(type(key) is not str for key in value):
+            raise UnsupportedModelError(reason="Hyperparameter keys must be strings.")
+        return {key: _parameter(item, path, budget) for key, item in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [
+            _parameter(item, path, budget, estimator_ref=estimator_ref)
+            for item in value
+        ]
+    return _leaf(value)
+
+
+def _leaf(value: object) -> JsonValue:
+    if isinstance(value, np.dtype):
+        return {"$dtype": dtype_identifier(value)}
+    if callable(value):
+        identifier = reference_identifier(value)
+        if identifier is not None:
+            return {"$type" if isinstance(value, type) else "$function": identifier}
+        raise UnsupportedModelError(
+            reason="Callable and type hyperparameters are unsupported."
+        )
+    return canonical_value(value, max_bytes=HYPERPARAMETER_BYTES, max_depth=16)
+
+
+def _estimator(value, ancestors, budget, reference) -> JsonValue:
+    # Named deep keys own step parameters; constructor lists retain routing/order.
+    if reference:
+        return {"$estimator_ref": type(value).__name__}
+    return {
+        "$estimator": {
+            "class": type(value).__name__,
+            "params": parameters(value, ancestors, budget),
+        }
+    }
+
+
+def parameters(
+    model: BaseEstimator,
+    ancestors: frozenset[int] = frozenset(),
+    budget: list[int] | None = None,
+) -> dict[str, JsonValue]:
+    budget = [0] if budget is None else budget
+    return cast(
+        dict[str, JsonValue],
+        {
+            key: _parameter(
+                value,
+                ancestors,
+                budget,
+                estimator_ref=key in ("steps", "transformers", "transformer_list"),
+            )
+            for key, value in model.get_params(deep=True).items()
+        },
+    )

mlbundle/_sklearn/payload.py

@@ -0,0 +1,234 @@
+"""Bound the nested skops ZIP before skops reads schema or numeric buffers."""
+
+from io import BytesIO
+from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile
+
+import numpy as np
+from pydantic import JsonValue
+
+from mlbundle.canonical import canonical_loads
+from mlbundle.errors import InvalidModelError
+from mlbundle.limits import (
+    MANIFEST_BYTES,
+    MAX_CLASSES,
+    MAX_FEATURES,
+    MAX_OUTPUTS,
+    ArchiveLimits,
+)
+
+from .archive import bounded_zip
+from .arrays import MAX_FILES, array_header, sparse_headers
+from .dtypes import validate_dtypes
+from .nodes import validate_nodes
+from .references import References
+from .state import state_digest, validate_state
+
+BUFFER_BYTES = ArchiveLimits().max_member_bytes
+
+
+def validate_payload(data: bytes) -> str:
+    limits = ArchiveLimits()
+    with bounded_zip(data, MAX_FILES) as archive:
+        entries = archive.infolist()
+        names = [entry.filename for entry in entries]
+        if (
+            len(names) > MAX_FILES
+            or len(names) != len(set(names))
+            or "schema.json" not in names
+            or any("/" in name or "\\" in name or ".." in name for name in names)
+            or sum(entry.file_size for entry in entries) > limits.max_member_bytes
+        ):
+            raise InvalidModelError("Invalid nested skops archive.")
+        schema_entry = archive.getinfo("schema.json")
+        if schema_entry.file_size > MANIFEST_BYTES:
+            raise InvalidModelError("Skops schema exceeds the byte limit.")
+        for entry in entries:
+            if (
+                entry.flag_bits & 1
+                or entry.compress_type not in (ZIP_STORED, ZIP_DEFLATED)
+                or entry.file_size > limits.max_member_bytes
+                or entry.file_size > limits.max_compression_ratio * entry.compress_size
+            ):
+                raise InvalidModelError("Unsafe nested skops member bounds.")
+        schema = canonical_loads(archive.read(schema_entry))
+        references = References(schema)
+        validate_nodes(references.nodes, schema)
+        validate_state(references.nodes, references.resolve)
+        arrays = {name for name in names if name.endswith(".npy")} | _array_references(
+            schema
+        )
+        sparse = {
+            node["file"]
+            for node in references.nodes
+            if node.get("__loader__") == "SparseMatrixNode"
+        }
+        remaining = limits.max_member_bytes - sum(
+            archive.getinfo(name).file_size for name in arrays if name in names
+        )
+        sparse_sizes = {}
+        for name in sparse:
+            sparse_sizes[name] = sparse_headers(archive.read(name), remaining)
+            remaining -= sparse_sizes[name]
+        _buffer_budget(references, archive, sparse_sizes)
+        headers = {}
+        for name in arrays:
+            if name not in names:
+                raise InvalidModelError("Missing skops numeric array member.")
+            headers[name] = _array_header(
+                archive, name, archive.getinfo(name).file_size
+            )
+        validate_dtypes(references.nodes, references.resolve, archive)
+        _tree_constructors(references, archive, headers)
+        files = _array_references(schema) | sparse | {"schema.json"}
+        if set(names) != files:
+            raise InvalidModelError("Unreferenced skops member.")
+        return state_digest(schema, archive, references.resolve)
+
+
+def _array_references(schema: JsonValue) -> set[str]:
+    arrays: set[str] = set()
+    pending = [schema]
+    while pending:
+        node = pending.pop()
+        if isinstance(node, dict):
+            if node.get("__loader__") == "NdArrayNode" and "file" in node:
+                name = node["file"]
+                if not isinstance(name, str):
+                    raise InvalidModelError("Invalid skops numeric array reference.")
+                arrays.add(name)
+            pending.extend(node.values())
+        elif isinstance(node, list):
+            pending.extend(node)
+    return arrays
+
+
+def _buffer_budget(
+    references: References, archive: ZipFile, sparse: dict[str, int]
+) -> None:
+    seen: set[int] = set()
+    total = 0
+    for node in references.nodes:
+        if (
+            node.get("__loader__") not in ("NdArrayNode", "SparseMatrixNode")
+            or "file" not in node
+        ):
+            continue
+        identity = node.get("__id__", id(node))
+        if identity in seen:
+            continue
+        seen.add(identity)
+        total += sparse.get(node["file"], archive.getinfo(node["file"]).file_size)
+        if total > BUFFER_BYTES:
+            raise InvalidModelError("Skops materialized buffer bytes exceed the limit.")
+
+
+_array_header = array_header
+
+
+def _tree_constructors(references: References, archive: ZipFile, headers) -> None:
+    seen: set[int] = set()
+    for node in references.nodes:
+        identity = node.get("__id__", id(node))
+        if node.get("__loader__") == "TreeNode" and identity not in seen:
+            seen.add(identity)
+            reduced = node.get("__reduce__")
+            if not isinstance(reduced, dict):
+                raise InvalidModelError("Invalid skops tree constructor.")
+            args = references.resolve(reduced.get("args"))
+            state = references.resolve(node.get("content"))
+            _tree_constructor(
+                {**node, "__reduce__": {"args": args}, "content": state},
+                references.memo,
+                archive,
+                headers,
+            )
+
+
+def _resolve(
+    value: JsonValue, memo: dict[int, dict[str, JsonValue]]
+) -> dict[str, JsonValue]:
+    if not isinstance(value, dict):
+        raise InvalidModelError("Invalid skops tree constructor schema.")
+    identity = value.get("__id__")
+    if type(identity) is int and identity in memo:
+        return memo[identity]
+    if value.get("__loader__") == "CachedNode":
+        raise InvalidModelError("Invalid skops tree constructor reference.")
+    return value
+
+
+def _integer(value: JsonValue, memo: dict[int, dict[str, JsonValue]]) -> int:
+    node = _resolve(value, memo)
+    content = node.get("content")
+    if (
+        node.get("__loader__") != "JsonNode"
+        or node.get("is_json") is not True
+        or not isinstance(content, str)
+    ):
+        raise InvalidModelError("Invalid skops tree constructor integer.")
+    result = canonical_loads(content.encode())
+    if type(result) is not int:
+        raise InvalidModelError("Invalid skops tree constructor integer.")
+    return result
+
+
+def _numeric_header(
+    value: JsonValue, memo: dict[int, dict[str, JsonValue]], headers
+) -> tuple[str, tuple[int, ...], np.dtype]:
+    node = _resolve(value, memo)
+    name = node.get("file")
+    if (
+        node.get("__loader__") != "NdArrayNode"
+        or node.get("type") != "numpy"
+        or not isinstance(name, str)
+    ):
+        raise InvalidModelError("Invalid skops tree numeric storage.")
+    shape, dtype = headers[name]
+    return name, shape, dtype
+
+
+def _tree_constructor(
+    node: dict[str, JsonValue],
+    memo: dict[int, dict[str, JsonValue]],
+    archive: ZipFile,
+    headers,
+) -> None:
+    reduced = node.get("__reduce__")
+    arguments = reduced.get("args") if isinstance(reduced, dict) else None
+    if not isinstance(arguments, dict):
+        raise InvalidModelError("Invalid skops tree constructor arguments.")
+    args = arguments.get("content")
+    if (
+        arguments.get("__loader__") != "TupleNode"
+        or not isinstance(args, list)
+        or len(args) != 3
+    ):
+        raise InvalidModelError("Invalid skops tree constructor arguments.")
+    width, outputs = _integer(args[0], memo), _integer(args[2], memo)
+    if not 1 <= width <= MAX_FEATURES or not 1 <= outputs <= MAX_OUTPUTS:
+        raise InvalidModelError("Unsafe skops tree constructor dimensions.")
+    name, shape, dtype = _numeric_header(args[1], memo, headers)
+    if shape != (outputs,) or dtype != np.dtype(np.intp):
+        raise InvalidModelError("Invalid skops tree class-count dimensions.")
+    classes = np.load(BytesIO(archive.read(name)), allow_pickle=False)
+    if np.any(classes < 1) or np.any(classes > MAX_CLASSES):
+        raise InvalidModelError("Unsafe skops tree class counts.")
+    state = node.get("content")
+    if not isinstance(state, dict):
+        raise InvalidModelError("Invalid skops tree state.")
+    attributes = state.get("content")
+    if not isinstance(attributes, dict):
+        raise InvalidModelError("Invalid skops tree state.")
+    if set(attributes) != {"node_count", "max_depth", "nodes", "values"}:
+        raise InvalidModelError("Unknown skops tree state.")
+    count = _integer(attributes.get("node_count"), memo)
+    depth = _integer(attributes.get("max_depth"), memo)
+    _, nodes_shape, _ = _numeric_header(attributes.get("nodes"), memo, headers)
+    _, values_shape, _ = _numeric_header(attributes.get("values"), memo, headers)
+    if (
+        count < 1
+        or not 0 <= depth < count
+        or nodes_shape != (count,)
+        or values_shape != (count, outputs, int(max(classes)))
+    ):
+        raise InvalidModelError("Invalid skops tree storage dimensions.")