mlbundle 0.1.0a1__py3-none-any.whl

mlbundle/_sklearn/arrays.py

@@ -0,0 +1,150 @@
+"""Check declared numeric and sparse storage against real bytes before allocation."""
+
+from io import BytesIO
+from math import prod
+from zipfile import ZipFile
+
+import numpy as np
+
+from mlbundle.errors import InvalidModelError
+from mlbundle.limits import ArchiveLimits
+
+from .archive import bounded_zip
+
+MAX_RANK = 32
+MAX_FILES = 100_000
+BYTE_CAP = ArchiveLimits().max_member_bytes
+
+
+def array_header(
+    archive: ZipFile, name: str, size: int
+) -> tuple[tuple[int, ...], np.dtype]:
+    with archive.open(name) as stream:
+        version = np.lib.format.read_magic(stream)
+        if version == (1, 0):
+            shape, _, dtype = np.lib.format.read_array_header_1_0(stream)
+        elif version == (2, 0):
+            shape, _, dtype = np.lib.format.read_array_header_2_0(stream)
+        else:
+            raise InvalidModelError("Unsupported skops numeric array header.")
+        if (
+            dtype.hasobject
+            or dtype.itemsize == 0
+            or len(shape) > MAX_RANK
+            or any(not 0 <= dimension <= BYTE_CAP for dimension in shape)
+            or prod(shape) * dtype.itemsize != size - stream.tell()
+        ):
+            raise InvalidModelError("Invalid skops numeric array dimensions.")
+        expected = size - stream.tell()
+        actual = 0
+        while chunk := stream.read(min(64 * 1024, expected - actual + 1)):
+            actual += len(chunk)
+            if actual > expected:
+                raise InvalidModelError("Invalid skops numeric array stored size.")
+        if actual != expected:
+            raise InvalidModelError("Invalid skops numeric array stored size.")
+        return shape, dtype
+
+
+def sparse_headers(data: bytes, max_bytes: int = BYTE_CAP) -> int:
+    with bounded_zip(data, 9) as archive:
+        entries = archive.infolist()
+        names = {entry.filename for entry in entries}
+        if (
+            len(names) != len(entries)
+            or not {"format.npy", "shape.npy", "data.npy"} <= names
+            or names
+            - {
+                "format.npy",
+                "shape.npy",
+                "data.npy",
+                "indices.npy",
+                "indptr.npy",
+                "row.npy",
+                "col.npy",
+                "offsets.npy",
+                "_is_array.npy",
+            }
+            or sum(entry.file_size for entry in entries) > max_bytes
+        ):
+            raise InvalidModelError("Invalid skops sparse archive.")
+        headers = {}
+        for entry in entries:
+            if (
+                entry.flag_bits & 1
+                or entry.compress_type not in (0, 8)
+                or entry.file_size > BYTE_CAP
+                or entry.file_size > 200 * entry.compress_size
+            ):
+                raise InvalidModelError("Unsafe skops sparse member bounds.")
+            headers[entry.filename] = array_header(
+                archive, entry.filename, entry.file_size
+            )
+        if headers["shape.npy"][0] != (2,) or headers["format.npy"][0] != ():
+            raise InvalidModelError("Invalid skops sparse shape or format.")
+        shape = np.load(BytesIO(archive.read("shape.npy")), allow_pickle=False)
+        if (
+            shape.dtype.kind not in "iu"
+            or any(not 0 <= dim <= BYTE_CAP for dim in shape)
+            or prod(int(dim) for dim in shape) * headers["data.npy"][1].itemsize
+            > BYTE_CAP
+        ):
+            raise InvalidModelError("Unsafe skops sparse declared shape.")
+        # All buffers have already passed exact stored-byte checks. Loading them
+        # here cannot allocate in proportion to an unsupported header declaration.
+        values = {
+            name[:-4]: np.load(BytesIO(archive.read(name)), allow_pickle=False)
+            for name in names
+        }
+        _sparse_indices(values, shape)
+        return sum(entry.file_size for entry in entries)
+
+
+def _sparse_indices(values, shape) -> None:
+    kind = values["format"].item()
+    rows, columns = (int(dim) for dim in shape)
+    if kind in (b"csr", b"csc", b"bsr"):
+        indices, pointers = values["indices"], values["indptr"]
+        blocks = len(values["data"])
+        major, minor = (columns, rows) if kind == b"csc" else (rows, columns)
+        if kind == b"bsr":
+            block_shape = values["data"].shape
+            if len(block_shape) != 3 or min(block_shape[1:]) < 1:
+                raise InvalidModelError("Invalid skops sparse block shape.")
+            major //= block_shape[1]
+            minor //= block_shape[2]
+        if (
+            pointers.dtype.kind not in "iu"
+            or indices.dtype.kind not in "iu"
+            or pointers.shape != (major + 1,)
+            or indices.shape != (blocks,)
+            or pointers[0] != 0
+            or pointers[-1] != blocks
+            or np.any(pointers[1:] < pointers[:-1])
+            or np.any(pointers < 0)
+            or np.any(indices < 0)
+            or np.any(indices >= minor)
+        ):
+            raise InvalidModelError("Invalid skops sparse index storage.")
+    elif kind == b"coo":
+        for name, width in (("row", rows), ("col", columns)):
+            index = values[name]
+            if (
+                index.dtype.kind not in "iu"
+                or index.shape != values["data"].shape
+                or np.any(index < 0)
+                or np.any(index >= width)
+            ):
+                raise InvalidModelError("Invalid skops sparse coordinate storage.")
+    elif kind == b"dia":
+        offsets = values["offsets"]
+        if (
+            offsets.dtype.kind not in "iu"
+            or offsets.ndim != 1
+            or values["data"].ndim != 2
+            or len(offsets) != len(values["data"])
+            or len(set(offsets.tolist())) != len(offsets)
+        ):
+            raise InvalidModelError("Invalid skops sparse diagonal storage.")
+    else:
+        raise InvalidModelError("Unsupported skops sparse format.")

mlbundle/_sklearn/column_width.py

@@ -0,0 +1,198 @@
+"""Cross-check ColumnTransformer selectors, fitted clones and output slices."""
+
+from collections.abc import Callable
+from typing import Any
+
+import numpy as np
+from sklearn.preprocessing import FunctionTransformer
+
+from mlbundle.canonical import canonical_dumps
+from mlbundle.errors import InvalidModelError
+
+from ._width_checks import width
+from .parameters import parameters
+
+
+def require(condition: bool, message: str) -> None:
+    if not condition:
+        raise InvalidModelError(message)
+
+
+def selector(value: Any, count: int, names: list[str] | None) -> list[int]:
+    message = "Invalid ColumnTransformer input mapping."
+
+    def position(item: Any) -> int:
+        if type(item) in (str, np.str_):
+            require(names is not None and item in names, message)
+            return (names or []).index(item)
+        require(
+            isinstance(item, (int, np.integer))
+            and not isinstance(item, (bool, np.bool_)),
+            message,
+        )
+        result = int(item)
+        result = result + count if result < 0 else result
+        require(0 <= result < count, message)
+        return result
+
+    if isinstance(value, slice):
+        if type(value.start) is str or type(value.stop) is str:
+            require(value.step is None and names is not None, message)
+            start = 0 if value.start is None else position(value.start)
+            stop = count if value.stop is None else position(value.stop) + 1
+            return list(range(start, stop))
+        require(
+            all(
+                item is None or type(item) is int
+                for item in (value.start, value.stop, value.step)
+            )
+            and value.step != 0,
+            message,
+        )
+        return list(range(count))[value]
+    if type(value) in (str, np.str_) or isinstance(value, (int, np.integer)):
+        return [position(value)]
+    require(isinstance(value, (list, tuple, np.ndarray)), message)
+    array = np.asarray(value)
+    require(array.ndim == 1 and len(array) <= 100_000, message)
+    if array.dtype.kind == "b":
+        require(len(array) == count, message)
+        return np.flatnonzero(array).tolist()
+    return [position(item) for item in value]
+
+
+def same_transformer(declared: Any, fitted: Any) -> bool:
+    if type(declared) is str:
+        if declared == "drop":
+            return type(fitted) is str and fitted == "drop"
+        return (
+            declared == "passthrough"
+            and type(fitted) is FunctionTransformer
+            and fitted.func is None
+            and fitted.inverse_func is None
+            and fitted.feature_names_out == "one-to-one"
+            and fitted.validate is False
+        )
+    return type(declared) is type(fitted) and canonical_dumps(
+        parameters(declared)
+    ) == canonical_dumps(parameters(fitted))
+
+
+def column_width(
+    model: Any, count: int, depth: int, output_width: Callable[..., int]
+) -> int:
+
+    mapping_error = "Invalid ColumnTransformer input mapping."
+    clone_error = "ColumnTransformer declared and fitted transformers disagree."
+    output_error = "Invalid ColumnTransformer output mapping."
+    declared = model.transformers
+    fitted = model.transformers_
+    require(
+        isinstance(declared, (list, tuple))
+        and isinstance(fitted, list)
+        and len(declared) <= 128
+        and len(model._columns) == len(declared),
+        mapping_error,
+    )
+    require(
+        all(
+            isinstance(item, (tuple, list)) and len(item) == 3
+            for item in [*declared, *fitted]
+        ),
+        mapping_error,
+    )
+    declared_names = [item[0] for item in declared]
+    require(
+        all(
+            type(name) is str
+            and 1 <= len(name) <= 256
+            and name != "remainder"
+            and "__" not in name
+            for name in declared_names
+        )
+        and len(set(declared_names)) == len(declared_names),
+        mapping_error,
+    )
+    names = getattr(model, "feature_names_in_", None)
+    names = None if names is None else list(names)
+    mapping = model._transformer_to_input_indices
+    require(
+        isinstance(mapping, dict) and set(mapping) == {*declared_names, "remainder"},
+        mapping_error,
+    )
+    for selected in mapping.values():
+        require(
+            isinstance(selected, list)
+            and all(type(i) is int and 0 <= i < count for i in selected),
+            mapping_error,
+        )
+    used = set()
+    for (name, _, columns), cached in zip(declared, model._columns, strict=True):
+        resolved = selector(columns, count, names)
+        require(
+            resolved == selector(cached, count, names) and resolved == mapping[name],
+            mapping_error,
+        )
+        used.update(resolved)
+    remaining = sorted(set(range(count)) - used)
+    require(mapping["remainder"] == remaining, mapping_error)
+    remainder = model._remainder
+    require(
+        isinstance(remainder, tuple)
+        and len(remainder) == 3
+        and remainder[0] == "remainder"
+        and selector(remainder[2], count, names) == remaining,
+        mapping_error,
+    )
+    left, right = model.remainder, remainder[1]
+    require(
+        type(left) is type(right)
+        and (
+            canonical_dumps(left) == canonical_dumps(right)
+            if type(left) is str
+            else canonical_dumps(parameters(left)) == canonical_dumps(parameters(right))
+        ),
+        clone_error,
+    )
+    expected = [*declared, *((remainder,) if remaining else ())]
+    require(len(fitted) == len(expected), clone_error)
+    require(type(model.sparse_output_) is bool, output_error)
+    outputs = model.output_indices_
+    require(
+        isinstance(outputs, dict) and set(outputs) == {*declared_names, "remainder"},
+        output_error,
+    )
+    total = 0
+    for (name, original, columns), (actual_name, transformer, actual_columns) in zip(
+        expected, fitted, strict=True
+    ):
+        require(
+            name == actual_name
+            and selector(columns, count, names)
+            == selector(actual_columns, count, names),
+            mapping_error,
+        )
+        selected = mapping[name]
+        if not selected:
+            require(
+                type(original) is type(transformer)
+                or same_transformer(original, transformer),
+                clone_error,
+            )
+            require(outputs[name] == slice(0, 0), output_error)
+            continue
+        require(same_transformer(original, transformer), clone_error)
+        if type(transformer) is str and transformer == "drop":
+            require(outputs[name] == slice(0, 0), output_error)
+            continue
+        require(
+            not isinstance(columns, (str, np.str_, int, np.integer))
+            and not isinstance(actual_columns, (str, np.str_, int, np.integer)),
+            mapping_error,
+        )
+        result = output_width(transformer, len(selected), depth=depth + 1)
+        require(outputs[name] == slice(total, total + result), output_error)
+        total = width(total + result)
+    if not remaining:
+        require(outputs["remainder"] == slice(0, 0), output_error)
+    return width(total)

mlbundle/_sklearn/consistency.py

@@ -0,0 +1,127 @@
+"""Check alternate inference state against the estimator's displayed parameters."""
+
+from typing import Any
+
+import numpy as np
+from scipy.sparse import issparse
+from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor
+from sklearn.ensemble._forest import ForestClassifier, ForestRegressor
+from sklearn.linear_model._glm.glm import _GeneralizedLinearRegressor
+from sklearn.svm._base import BaseLibSVM
+
+from mlbundle.errors import InvalidModelError
+
+
+def validate_consistency(model: Any) -> None:
+    if isinstance(
+        model, (GradientBoostingClassifier, GradientBoostingRegressor)
+    ) and hasattr(model, "estimators_"):
+        columns = (
+            model.n_classes_
+            if isinstance(model, GradientBoostingClassifier) and model.n_classes_ > 2
+            else 1
+        )
+        trees = model.estimators_
+        if (
+            not isinstance(trees, np.ndarray)
+            or trees.ndim != 2
+            or trees.shape != (model.n_estimators_, columns)
+            or model.n_trees_per_iteration_ != columns
+        ):
+            raise InvalidModelError("Invalid gradient boosting stage arrangement.")
+        expected = model._get_loss(None)
+        _loss(model._loss, expected)
+        _initializer(model)
+    if isinstance(model, (ForestClassifier, ForestRegressor)) and hasattr(
+        model, "estimators_"
+    ):
+        for estimator in model.estimators_:
+            if estimator.n_outputs_ != model.n_outputs_ or (
+                isinstance(model, ForestClassifier)
+                and not np.array_equal(
+                    np.atleast_1d(estimator.n_classes_), np.atleast_1d(model.n_classes_)
+                )
+            ):
+                raise InvalidModelError(
+                    "Forest and constituent output counts disagree."
+                )
+    if isinstance(model, _GeneralizedLinearRegressor) and hasattr(model, "_base_loss"):
+        _loss(model._base_loss, model._get_loss())
+    if isinstance(model, BaseLibSVM) and hasattr(model, "_dual_coef_"):
+        _svm(model)
+
+
+def _loss(actual: Any, expected: Any) -> None:
+    if (
+        type(actual) is not type(expected)
+        or type(actual.link) is not type(expected.link)
+        or getattr(actual, "n_classes", None) != getattr(expected, "n_classes", None)
+    ):
+        raise InvalidModelError("Fitted loss and declared parameters disagree.")
+
+
+def _initializer(model: Any) -> None:
+    initializer = model.init_
+    if (
+        hasattr(initializer, "n_features_in_")
+        and initializer.n_features_in_ != model.n_features_in_
+    ):
+        raise InvalidModelError("Gradient boosting initializer input width disagrees.")
+
+
+def _same(left: Any, right: Any, sign: int) -> bool:
+    if issparse(left) or issparse(right):
+        return (
+            issparse(left)
+            and issparse(right)
+            and left.format == right.format
+            and left.shape == right.shape
+            and np.array_equal(left.indices, right.indices)
+            and np.array_equal(left.indptr, right.indptr)
+            and np.array_equal(left.data, sign * right.data)
+        )
+    return np.array_equal(left, sign * right)
+
+
+def _svm(model: Any) -> None:
+    if (
+        type(model._sparse) is not bool
+        or issparse(model.support_vectors_) != model._sparse
+        or issparse(model._dual_coef_) != model._sparse
+    ):
+        raise InvalidModelError("SVM sparse flag and fitted storage disagree.")
+    classes = len(model.classes_) if hasattr(model, "classes_") else None
+    count = len(model.support_)
+    supports = np.asarray(model._n_support)
+    rows = 1 if classes is None else classes - 1
+    intercepts = 1 if classes is None else classes * (classes - 1) // 2
+    if (
+        supports.dtype.kind not in "iu"
+        or supports.shape != ((2,) if classes is None else (classes,))
+        or np.any(supports < 0)
+        or np.any(supports > count)
+        or int(supports.sum()) != count * (2 if classes is None else 1)
+        or model._dual_coef_.shape != (rows, count)
+        or model._intercept_.shape != (intercepts,)
+        or model.support_.dtype.kind not in "iu"
+        or np.any(model.support_ < 0)
+        or np.any(model.support_ >= model.shape_fit_[0])
+        or model.shape_fit_[1] != model.n_features_in_
+    ):
+        raise InvalidModelError("Invalid SVM fitted dimensions or support indices.")
+    if model.kernel != "precomputed" and model.support_vectors_.shape != (
+        count,
+        model.n_features_in_,
+    ):
+        raise InvalidModelError("Invalid SVM support-vector dimensions.")
+    if (
+        classes is not None
+        and (model._probA.size or callable(getattr(model, "predict_proba", None)))
+        and (model._probA.shape != (intercepts,) or model._probB.shape != (intercepts,))
+    ):
+        raise InvalidModelError("Invalid SVM probability calibration dimensions.")
+    sign = -1 if model._impl in ("c_svc", "nu_svc") and len(model.classes_) == 2 else 1
+    if not _same(model.dual_coef_, model._dual_coef_, sign) or not np.array_equal(
+        model.intercept_, sign * model._intercept_
+    ):
+        raise InvalidModelError("SVM public and inference coefficients disagree.")

mlbundle/_sklearn/dtypes.py

@@ -0,0 +1,41 @@
+"""Validate skops' empty-array dtype representation before construction."""
+
+from mlbundle.canonical import canonical_loads
+from mlbundle.errors import InvalidModelError
+
+from .arrays import array_header
+from .trusted import dtype_identifier
+
+
+def validate_dtypes(nodes, resolve, archive) -> None:
+    for node in nodes:
+        if node.get("__loader__") != "DTypeNode":
+            continue
+        content = resolve(node.get("content"))
+        if content.get("__loader__") != "NdArrayNode" or (
+            content.get("__module__"),
+            content.get("__class__"),
+        ) != ("numpy", "ndarray"):
+            raise InvalidModelError("Invalid skops dtype storage.")
+        if content.get("type") == "numpy":
+            name = content["file"]
+            shape, dtype = array_header(archive, name, archive.getinfo(name).file_size)
+            dtype_identifier(dtype)
+            if shape != (0,):
+                raise InvalidModelError("Skops dtype storage must be empty and scalar.")
+        elif content.get("type") == "json":
+            # Only the canonical empty object array emitted by dtype_get_state.
+            shape = resolve(content.get("shape"))
+            if shape.get("__loader__") != "TupleNode" or len(shape["content"]) != 1:
+                raise InvalidModelError("Invalid skops object dtype shape.")
+            size = resolve(shape["content"][0])
+            if (
+                size.get("__loader__") != "JsonNode"
+                or size.get("is_json") is not True
+                or canonical_loads(size["content"].encode()) != 0
+                or size["content"] != "0"
+                or content.get("content") != []
+            ):
+                raise InvalidModelError("Skops object dtype storage must be empty.")
+        else:
+            raise InvalidModelError("Invalid skops dtype storage kind.")

mlbundle/_sklearn/encoder_width.py

@@ -0,0 +1,123 @@
+"""Validate fitted category/drop/grouping dimensions before deriving width."""
+
+from typing import Any
+
+import numpy as np
+from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder
+
+from mlbundle.canonical import canonical_dumps
+
+from ._width_checks import dimensions, indices, width
+
+
+def encoder_width(model: Any, count: int) -> int:
+    categories = model.categories_
+    dimensions(isinstance(categories, list) and len(categories) == count)
+    lengths = []
+    for values in categories:
+        dimensions(type(values) is np.ndarray)
+        array = np.asarray(values)
+        dimensions(array.ndim == 1 and 1 <= len(array) <= 100_000)
+        encoded = [canonical_dumps(item) for item in array]
+        dimensions(len(set(encoded)) == len(encoded))
+        lengths.append(len(array))
+    enabled = model._infrequent_enabled
+    dimensions(
+        type(enabled) is bool
+        and enabled
+        == (model.min_frequency is not None or model.max_categories is not None)
+    )
+    missing = model._missing_indices if type(model) is OrdinalEncoder else {}
+    dimensions(
+        isinstance(missing, dict)
+        and all(type(i) is int and 0 <= i < count for i in missing)
+    )
+    for i, index in missing.items():
+        dimensions(width(index, zero=True) == lengths[i] - 1)
+        dimensions(bool(np.asarray(categories[i][-1] != categories[i][-1])))
+    grouped = grouped_counts(model, lengths, missing, count)
+    if type(model) is not OneHotEncoder:
+        return count
+    dropped_counts(model, lengths, grouped, count)
+    cached = model._n_features_outs
+    dimensions(isinstance(cached, list) and len(cached) == count)
+    dimensions([width(item, zero=True) for item in cached] == grouped)
+    return width(sum(grouped))
+
+
+def grouped_counts(
+    model: Any, lengths: list[int], missing: dict, count: int
+) -> list[int]:
+    grouped = lengths.copy()
+    enabled = model._infrequent_enabled
+    if enabled:
+        infrequent = model._infrequent_indices
+        mappings = model._default_to_infrequent_mappings
+        dimensions(
+            isinstance(infrequent, list)
+            and isinstance(mappings, list)
+            and len(infrequent) == len(mappings) == count
+        )
+        for i, (selection, mapping) in enumerate(
+            zip(infrequent, mappings, strict=True)
+        ):
+            if selection is None:
+                dimensions(mapping is None)
+                continue
+            size = lengths[i] - int(i in missing)
+            selected = indices(selection, size)
+            dimensions(bool(selected) and selected == sorted(selected))
+            grouped[i] -= len(selected) - 1
+            array = np.asarray(mapping)
+            dimensions(array.shape == (size,) and array.dtype.kind in "iu")
+            expected = np.empty(size, dtype=int)
+            frequent = sorted(set(range(size)) - set(selected))
+            expected[frequent] = np.arange(len(frequent))
+            expected[selected] = len(frequent)
+            dimensions(np.array_equal(array, expected))
+    return grouped
+
+
+def dropped_counts(
+    model: Any, lengths: list[int], grouped: list[int], count: int
+) -> None:
+    enabled = model._infrequent_enabled
+    drop = model.drop_idx_
+    after = model._drop_idx_after_grouping
+    dimensions((drop is None) == (after is None) == (model.drop is None))
+    if drop is not None:
+        dimensions(
+            type(drop) is np.ndarray
+            and type(after) is np.ndarray
+            and drop.dtype.kind == after.dtype.kind == "O"
+        )
+        dimensions(np.asarray(drop).shape == np.asarray(after).shape == (count,))
+        for i, (raw_original, raw_transformed) in enumerate(
+            zip(drop, np.asarray(after), strict=True)
+        ):
+            dimensions((raw_original is None) == (raw_transformed is None))
+            if raw_original is None:
+                dimensions(model.drop == "if_binary" and grouped[i] != 2)
+                continue
+            original = width(raw_original, zero=True)
+            transformed = width(raw_transformed, zero=True)
+            dimensions(original < lengths[i] and transformed < grouped[i])
+            mapping = model._default_to_infrequent_mappings[i] if enabled else None
+            dimensions(
+                transformed == (original if mapping is None else int(mapping[original]))
+            )
+            if type(model.drop) is str:
+                dimensions(
+                    transformed == 0
+                    and (
+                        model.drop == "first"
+                        or (model.drop == "if_binary" and grouped[i] == 2)
+                    )
+                )
+            else:
+                dimensions(np.asarray(model.drop).shape == (count,))
+                dimensions(
+                    canonical_dumps(np.asarray(model.drop)[i])
+                    == canonical_dumps(model.categories_[i][original])
+                )
+            grouped[i] -= 1