mlbundle 0.1.0a1__py3-none-any.whl

mlbundle/_sklearn/pipeline_scope.py

@@ -0,0 +1,141 @@
+"""Bounded native-placement discovery and closed Pipeline options."""
+
+from typing import Any
+
+import numpy as np
+from sklearn import get_config
+from sklearn.base import BaseEstimator
+from sklearn.pipeline import Pipeline
+
+from mlbundle._pipeline_contract import OPTIONS, valid_step_names
+from mlbundle.errors import InvalidModelError, LimitExceededError, UnsupportedModelError
+
+
+def objects(root: Any):
+    pending = [root]
+    seen = set()
+    visits = 0
+    while pending:
+        value = pending.pop()
+        visits += 1
+        if visits > 100_000:
+            raise LimitExceededError("Pipeline state work exceeds the limit.")
+        if id(value) in seen:
+            continue
+        seen.add(id(value))
+        if isinstance(value, dict):
+            children = value.values()
+        elif isinstance(value, (list, tuple)):
+            children = value
+        elif isinstance(value, np.ndarray) and value.dtype.kind == "O":
+            children = value.flat
+        else:
+            yield value
+            children = vars(value).values() if isinstance(value, BaseEstimator) else ()
+        if len(pending) + len(children) > 100_000:
+            raise LimitExceededError("Pipeline state work exceeds the limit.")
+        pending.extend(children)
+
+
+def native_types(root: Any) -> tuple[str, ...]:
+    return tuple(
+        sorted(
+            {
+                f"{type(value).__module__}.{type(value).__name__}"
+                for value in objects(root)
+                if {base.__module__.split(".")[0] for base in type(value).__mro__}
+                & {"xgboost", "catboost"}
+            }
+        )
+    )
+
+
+def composite_candidate(model: Any) -> bool:
+    found = native_types(model)
+    if not found:
+        return False
+    if (
+        isinstance(model, Pipeline)
+        and model.steps
+        and {base.__module__.split(".")[0] for base in type(model.steps[-1][1]).__mro__}
+        & {"xgboost", "catboost"}
+    ):
+        if any(native_types(step) for _, step in model.steps[:-1]):
+            reject_placement(found)
+        validate_options(model)
+        return True
+    reject_placement(found)
+    return False
+
+
+def reject_placement(types: tuple[str, ...]) -> None:
+    raise UnsupportedModelError(
+        types,
+        reason=(
+            "Native estimators are supported only as the last step of the outermost Pipeline. "
+            + str(UnsupportedModelError(types))
+        ),
+    )
+
+
+def validate_options(model: Any) -> None:
+    if (
+        type(model) is not Pipeline
+        or not isinstance(model.steps, (list, tuple))
+        or not 1 <= len(model.steps) <= 128
+    ):
+        raise UnsupportedModelError(
+            reason="Composite models require an exact outer Pipeline with 1 to 128 steps."
+        )
+    if "_metadata_request" in vars(model) or model.transform_input is not None:
+        raise UnsupportedModelError(
+            reason="Composite Pipeline transform_input and metadata routing are unsupported."
+        )
+    if set(vars(model)) != OPTIONS:
+        raise InvalidModelError("Unknown outer Pipeline state.")
+    if any(
+        not isinstance(item, (list, tuple)) or len(item) != 2 for item in model.steps
+    ):
+        raise UnsupportedModelError(
+            reason="Pipeline step names must be unique bounded sklearn names."
+        )
+    names = [name for name, _ in model.steps]
+    if any(
+        type(name) is not str or not 1 <= len(name) <= 256 for name in names
+    ) or not valid_step_names(names):
+        raise UnsupportedModelError(
+            reason="Pipeline step names must be unique bounded sklearn names."
+        )
+    if model.memory is not None:
+        raise UnsupportedModelError(
+            reason="Composite Pipeline memory must be None; caching is unsupported."
+        )
+    if type(model.verbose) is not bool:
+        raise UnsupportedModelError(
+            reason="Composite Pipeline verbose must be a boolean."
+        )
+
+
+def validate_components(model: Pipeline, *, exporting: bool = False) -> None:
+    validate_options(model)
+    if exporting and get_config()["transform_output"] != "default":
+        raise UnsupportedModelError(
+            reason="Composite export requires default global transform_output; use per-step set_output."
+        )
+    for value in objects(model):
+        if isinstance(value, BaseEstimator):
+            if "_metadata_request" in vars(value):
+                raise UnsupportedModelError(
+                    reason="Composite Pipeline transform_input and metadata routing are unsupported."
+                )
+            config = vars(value).get("_sklearn_output_config", {})
+            if (
+                not isinstance(config, dict)
+                or set(config) - {"transform"}
+                or config.get("transform", "default") not in ("default", "pandas")
+            ):
+                raise UnsupportedModelError(
+                    reason="Composite transformers support only default or pandas set_output."
+                )
+            if isinstance(value, Pipeline) and value is not model:
+                validate_options(value)

mlbundle/_sklearn/polynomial_names.py

@@ -0,0 +1,31 @@
+"""Bounded polynomial names without sklearn's dense powers_ allocation."""
+
+from collections import Counter
+from itertools import combinations, combinations_with_replacement
+from typing import Any
+
+from mlbundle.errors import InvalidModelError, LimitExceededError
+
+
+def polynomial_names(model: Any, names: list[str]) -> list[str]:
+    # Count term visits before constructing tuples; unlike powers_, work follows
+    # emitted monomials, not output width multiplied by original feature width.
+    maximum = (
+        min(model._max_degree, len(names))
+        if model.interaction_only
+        else model._max_degree
+    )
+    if model.n_output_features_ * max(1, maximum) > 100_000:
+        raise LimitExceededError("Transformer feature-name work exceeds the limit.")
+    choose = combinations if model.interaction_only else combinations_with_replacement
+    result = ["1"] if model.include_bias else []
+    for degree in range(max(1, model._min_degree), maximum + 1):
+        for term in choose(range(len(names)), degree):
+            parts = [
+                names[index] + ("" if power == 1 else f"^{power}")
+                for index, power in sorted(Counter(term).items())
+            ]
+            if sum(map(len, parts)) + len(parts) - 1 > 1024:
+                raise InvalidModelError("Invalid transformed feature names.")
+            result.append(" ".join(parts))
+    return result

mlbundle/_sklearn/prediction.py

@@ -0,0 +1,119 @@
+"""Tabular prediction with explicit output dimensions and probability validation."""
+
+import warnings
+from typing import Any, cast
+
+import numpy as np
+from sklearn.compose import ColumnTransformer
+from sklearn.pipeline import FeatureUnion, Pipeline
+from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, TargetEncoder
+
+from mlbundle._prediction_input import tabular
+from mlbundle.adapters import ProbabilityOutput, RegressionOutput
+from mlbundle.claims import ModelClaims
+from mlbundle.contract import ModelKind
+from mlbundle.errors import InvalidModelError, MLBundleError
+
+
+def _string_inputs(model: Any) -> bool:
+    pending = [model]
+    seen: set[int] = set()
+    while pending:
+        value = pending.pop()
+        if id(value) in seen:
+            continue
+        seen.add(id(value))
+        if isinstance(
+            value, (OneHotEncoder, OrdinalEncoder, TargetEncoder)
+        ) and hasattr(value, "categories_"):
+            return True
+        if isinstance(value, Pipeline):
+            pending.extend(step for _, step in value.steps)
+        elif isinstance(value, ColumnTransformer):
+            pending.extend(step for _, step, _ in getattr(value, "transformers_", []))
+        elif isinstance(value, FeatureUnion):
+            pending.extend(step for _, step in value.transformer_list)
+    return False
+
+
+class Predictor:
+    def __init__(self, model: Any, claims: ModelClaims) -> None:
+        self.model = model
+        self.claims = claims
+        self.strings = isinstance(self.model, Pipeline) and _string_inputs(self.model)
+
+    def _data(self, data: object) -> tuple[object, int]:
+        return tabular(data, self.claims.n_features, strings=self.strings)
+
+    def _call(self, data: object, *, probabilities: bool) -> Any:
+        # Positional arrays/lists remain supported for named models; sklearn
+        # still enforces DataFrame name/order when those names are supplied.
+        try:
+            return self._native_call(data, probabilities=probabilities)
+        except MLBundleError:
+            raise
+        except Exception:
+            raise InvalidModelError("Sklearn prediction failed.") from None
+
+    def _native_call(self, data: object, *, probabilities: bool) -> Any:
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore",
+                message="X does not have valid feature names, but .*",
+                category=UserWarning,
+            )
+            warnings.filterwarnings(
+                "ignore", message="X has feature names, but .*", category=UserWarning
+            )
+            raw = (
+                self.model.predict_proba(data)
+                if probabilities
+                else self.model.predict(data)
+            )
+            if probabilities:
+                return (
+                    [np.asarray(item, dtype=float) for item in raw]
+                    if isinstance(raw, list)
+                    else np.asarray(raw, dtype=float)
+                )
+            return np.asarray(raw, dtype=float)
+
+    def predict(self, data: object) -> RegressionOutput:
+        if self.claims.kind != ModelKind.REGRESSOR:
+            raise ValueError("predict requires a regressor.")
+        data, rows = self._data(data)
+        result = np.asarray(self._call(data, probabilities=False), dtype=float)
+        expected = (
+            (rows,) if self.claims.n_outputs == 1 else (rows, self.claims.n_outputs)
+        )
+        if self.claims.n_outputs == 1 and result.shape == (rows, 1):
+            result = result[:, 0]
+        if result.shape != expected or not np.isfinite(result).all():
+            raise InvalidModelError(
+                "Invalid regression prediction dimensions or values."
+            )
+        return cast(RegressionOutput, result.tolist())
+
+    def predict_proba(self, data: object) -> ProbabilityOutput:
+        if self.claims.kind != ModelKind.CLASSIFIER:
+            raise ValueError("predict_proba requires a classifier.")
+        data, rows = self._data(data)
+        raw = self._call(data, probabilities=True)
+        matrices = raw if isinstance(raw, list) else [raw]
+        if not isinstance(matrices, list) or len(matrices) != self.claims.n_outputs:
+            raise InvalidModelError("Invalid probability output count.")
+        result: list[list[list[float]]] = []
+        for matrix, classes in zip(matrices, self.claims.n_classes or [], strict=True):
+            values = np.asarray(matrix, dtype=float)
+            if (
+                values.shape != (rows, classes)
+                or not np.isfinite(values).all()
+                or np.any(values < 0)
+                or np.any(values > 1)
+                or not np.allclose(values.sum(axis=1), 1.0, rtol=1e-7, atol=1e-8)
+            ):
+                raise InvalidModelError(
+                    "Invalid probability dimensions, values or row sums."
+                )
+            result.append(values.tolist())
+        return result[0] if self.claims.n_outputs == 1 else result

mlbundle/_sklearn/prediction_state.py

@@ -0,0 +1,61 @@
+"""Readable composition facts supplement complete fitted-state commitments."""
+
+from typing import Any
+
+import numpy as np
+import skops.io as sio
+from sklearn.base import BaseEstimator
+from sklearn.dummy import DummyClassifier, DummyRegressor
+
+from .parameters import parameters
+from .payload import validate_payload
+
+
+def prediction_state(model: Any) -> dict[str, Any]:
+    state = {}
+    if hasattr(model, "estimators_"):
+        estimators = np.asarray(model.estimators_, dtype=object)
+        state["estimator_labels"] = [
+            getattr(estimator, "classes_", None) for estimator in estimators.flat
+        ]
+    for attribute in (
+        "estimator_weights_",
+        "estimators_features_",
+        "_x_mean",
+        "_x_std",
+    ):
+        if hasattr(model, attribute):
+            state[attribute] = getattr(model, attribute)
+    loss = getattr(model, "_loss", getattr(model, "_base_loss", None))
+    if loss is not None:
+        state["loss"] = {
+            "class": type(loss).__name__,
+            "link": type(loss.link).__name__,
+            "n_classes": loss.n_classes,
+        }
+    if hasattr(model, "init_"):
+        initializer = model.init_
+        if isinstance(initializer, BaseEstimator):
+            state["initializer"] = {
+                "class": type(initializer).__name__,
+                "parameters": parameters(initializer),
+                "state_sha256": validate_payload(sio.dumps(initializer)),
+            }
+            if isinstance(initializer, (DummyClassifier, DummyRegressor)):
+                state["initializer"]["values"] = {
+                    name: getattr(initializer, name)
+                    for name in (
+                        "classes_",
+                        "class_prior_",
+                        "constant_",
+                        "n_classes_",
+                        "n_outputs_",
+                    )
+                    if hasattr(initializer, name)
+                }
+        else:
+            state["initializer"] = initializer
+        state["learning_rate"] = model.learning_rate
+    if hasattr(model, "_probA"):
+        state["probability_calibration"] = {"A": model._probA, "B": model._probB}
+    return state

mlbundle/_sklearn/references.py

@@ -0,0 +1,110 @@
+"""Bound skops' recursive audit work before it builds or audits shared nodes."""
+
+from typing import Any
+
+from mlbundle.canonical import canonical_dumps
+from mlbundle.errors import InvalidModelError
+
+MAX_NODES = 100_000
+MAX_REFERENCES = 100_000
+MAX_VISITS = 100_000
+MAX_DEPTH = 32
+
+
+def _without_ids(value: Any) -> Any:
+    if isinstance(value, dict):
+        return {
+            key: _without_ids(child) for key, child in value.items() if key != "__id__"
+        }
+    if isinstance(value, list):
+        return [_without_ids(child) for child in value]
+    return value
+
+
+class References:
+    def __init__(self, schema: Any) -> None:
+        self.nodes: list[dict[str, Any]] = []
+        self.memo: dict[int, dict[str, Any]] = {}
+        pending = [schema]
+        self.reference_count = 0
+        while pending:
+            value = pending.pop()
+            if isinstance(value, dict):
+                if "__loader__" in value:
+                    self.nodes.append(value)
+                    if len(self.nodes) > MAX_NODES:
+                        raise InvalidModelError("Skops node count exceeds the limit.")
+                    self._record(value)
+                pending.extend(reversed(list(value.values())))
+            elif isinstance(value, list):
+                pending.extend(reversed(value))
+        self._work(schema)
+
+    def _record(self, value: dict[str, Any]) -> None:
+        identity = value.get("__id__")
+        if identity is not None and (type(identity) is not int or identity <= 0):
+            raise InvalidModelError("Invalid skops object identity.")
+        if value["__loader__"] == "CachedNode":
+            self.reference_count += 1
+            if self.reference_count > MAX_REFERENCES:
+                raise InvalidModelError("Skops reference count exceeds the limit.")
+        elif identity is not None:
+            previous = self.memo.get(identity)
+            if previous is not None and (
+                previous["__loader__"] != value["__loader__"]
+                or (
+                    value["__loader__"] in ("JsonNode", "NdArrayNode")
+                    and canonical_dumps(_without_ids(previous))
+                    != canonical_dumps(_without_ids(value))
+                )
+            ):
+                raise InvalidModelError("Conflicting skops object references.")
+            self.memo.setdefault(identity, value)
+
+    def resolve(self, value: Any) -> dict[str, Any]:
+        if not isinstance(value, dict):
+            raise InvalidModelError("Invalid skops node reference.")
+        if value.get("__loader__") == "CachedNode":
+            identity = value.get("__id__")
+            if type(identity) is not int:
+                raise InvalidModelError("Invalid skops object identity.")
+            target = self.memo.get(identity)
+            if target is None:
+                raise InvalidModelError("Unknown skops object reference.")
+            if (target.get("__module__"), target.get("__class__")) != (
+                value.get("__module__"),
+                value.get("__class__"),
+            ):
+                raise InvalidModelError("Conflicting skops reference type.")
+            return target
+        return value
+
+    def _work(self, schema: Any) -> None:
+        memo: dict[int, tuple[int, int]] = {}
+        active: set[int] = set()
+
+        def cost(value):
+            if not isinstance(value, (dict, list)):
+                return 0, 0
+            value = self.resolve(value) if isinstance(value, dict) else value
+            identity = id(value)
+            if identity in active:
+                raise InvalidModelError("Cyclic skops object references.")
+            if identity in memo:
+                return memo[identity]
+            active.add(identity)
+            visits, depth = 1, 1
+            children = value.values() if isinstance(value, dict) else value
+            for child in children:
+                child_visits, child_depth = cost(child)
+                visits += child_visits
+                depth = max(depth, child_depth + 1)
+                if visits > MAX_VISITS or depth > MAX_DEPTH:
+                    raise InvalidModelError(
+                        "Skops expanded node visits or depth exceed the limit."
+                    )
+            active.remove(identity)
+            memo[identity] = visits, depth
+            return visits, depth
+
+        cost(schema)

mlbundle/_sklearn/serialization.py

@@ -0,0 +1,80 @@
+"""One owned skops byte boundary for estimators and transformer components."""
+
+from collections.abc import Callable
+
+import skops.io as sio
+
+from mlbundle.errors import InvalidModelError, MLBundleError, UnsupportedModelError
+
+from .payload import validate_payload
+from .trusted import TRUSTED_TYPES
+
+
+def audit_types(data: bytes) -> None:
+    unsupported = tuple(
+        sorted(set(sio.get_untrusted_types(data=data)) - set(TRUSTED_TYPES))
+    )
+    if unsupported:
+        raise UnsupportedModelError(unsupported)
+
+
+class SkopsIO:
+    """Reuse checked snapshots inside one operation; never across public calls."""
+
+    def __init__(
+        self,
+        validate: Callable[[object], None],
+        payload: Callable[[bytes], str] = validate_payload,
+    ) -> None:
+        self.validate = validate
+        self.payload = payload
+        self._states: list[tuple[object, str]] = []
+        self._payload: bytes | None = None
+        self._digest: str | None = None
+
+    def state(self, model: object) -> str:
+        for native, digest in self._states:
+            if native is model:
+                return digest
+        return self.payload(sio.dumps(model))
+
+    def serialize(self, model: object) -> bytes:
+        try:
+            self._states = []
+            self._payload, self._digest = None, None
+            data = sio.dumps(model)
+            try:
+                digest = self.payload(data)
+            except UnsupportedModelError:
+                audit_types(data)
+                raise
+            audit_types(data)
+            self.validate(model)
+            self._states = [(model, digest)]
+            self._payload, self._digest = data, digest
+            return data
+        except MLBundleError:
+            raise
+        except Exception:
+            raise UnsupportedModelError(
+                reason="Model cannot be serialized safely with skops."
+            ) from None
+
+    def deserialize(self, data: bytes) -> object:
+        try:
+            expected = self._digest if data is self._payload else self.payload(data)
+            self._payload, self._digest = None, None
+            audit_types(data)
+            model = sio.loads(data, trusted=list(TRUSTED_TYPES))
+            self.validate(model)
+            actual = self.payload(sio.dumps(model))
+            if expected != actual:
+                raise InvalidModelError(
+                    "Skops state does not survive native construction."
+                )
+            self._states = [*self._states[-1:], (model, actual)]
+            return model
+        except MLBundleError:
+            raise
+        except Exception:
+            raise InvalidModelError("Invalid skops model payload.") from None

mlbundle/_sklearn/state.py

@@ -0,0 +1,144 @@
+"""Closed sklearn state keys and a content digest independent of skops object IDs."""
+
+import hashlib
+import json
+from importlib.resources import files
+from io import BytesIO
+from typing import Any
+from zipfile import ZipFile
+
+import numpy as np
+
+from mlbundle.canonical import canonical_dumps
+from mlbundle.errors import InvalidModelError, LimitExceededError, UnsupportedModelError
+from mlbundle.limits import MANIFEST_BYTES
+
+FIELDS = json.loads(files(__package__).joinpath("state_fields.json").read_text())
+
+
+def _buffer_digest(data: bytes) -> str:
+    if data.startswith(b"PK"):
+        with ZipFile(BytesIO(data)) as archive:
+            record = {
+                name: _buffer_digest(archive.read(name)) for name in archive.namelist()
+            }
+        return hashlib.sha256(canonical_dumps(record)).hexdigest()
+    array = np.load(BytesIO(data), allow_pickle=False)
+    if array.dtype.names:
+        # Native Tree records have uninitialized padding; only named fields
+        # participate in inference and survive the native round trip.
+        record = {name: _numeric_digest(array[name]) for name in array.dtype.names}
+        return hashlib.sha256(canonical_dumps(record)).hexdigest()
+    return _numeric_digest(array)
+
+
+def _numeric_digest(array: np.ndarray) -> str:
+    if array.dtype.kind == "f" and np.isnan(array).any():
+        array = array.copy()
+        array[np.isnan(array)] = np.nan
+    digest = hashlib.sha256(canonical_dumps([array.dtype.str, list(array.shape)]))
+    digest.update(np.ascontiguousarray(array).tobytes())
+    return digest.hexdigest()
+
+
+def _hash(domain: bytes, value: Any) -> str:
+    if domain == b"scalar:" or (isinstance(value, dict) and "$float" in value):
+        encoded = canonical_dumps(value)
+    else:
+        # These records contain only checked string keys and fixed-width hex digests.
+        size = (
+            sum(len(key) + 70 for key in value)
+            if isinstance(value, dict)
+            else len(value) * 67
+            if isinstance(value, list)
+            else 66
+        ) + 1
+        if size > MANIFEST_BYTES:
+            raise LimitExceededError("JSON exceeds the byte limit.")
+        encoded = json.dumps(
+            value,
+            sort_keys=True,
+            ensure_ascii=False,
+            allow_nan=False,
+            separators=(",", ":"),
+        ).encode("utf-8")
+        if len(encoded) > MANIFEST_BYTES:
+            raise LimitExceededError("JSON exceeds the byte limit.")
+    return hashlib.sha256(domain + encoded).hexdigest()
+
+
+def validate_state(nodes: list[dict[str, Any]], resolve) -> None:
+    for node in nodes:
+        if node.get("__loader__") != "ObjectNode":
+            continue
+        name = node.get("__module__", "") + "." + node.get("__class__", "")
+        if name not in FIELDS:
+            raise UnsupportedModelError((name,))
+        state = resolve(node.get("content"))
+        if state.get("__loader__") == "JsonNode" and state.get("content") == "null":
+            continue
+        if state.get("__loader__") != "DictNode":
+            raise InvalidModelError("Invalid sklearn object state.")
+        attrs = state.get("content")
+        if not isinstance(attrs, dict) or set(attrs) - set(FIELDS[name]):
+            raise InvalidModelError("Unknown sklearn estimator state.")
+        # DictNode otherwise silently truncates attributes through zip().
+        keys = resolve(state.get("key_types"))
+        if keys.get("__loader__") != "ListNode" or len(keys["content"]) != len(attrs):
+            raise InvalidModelError("Invalid sklearn state key types.")
+        for raw_key in keys["content"]:
+            key = resolve(raw_key)
+            if key.get("__loader__") != "TypeNode" or (
+                key.get("__module__"),
+                key.get("__class__"),
+            ) != ("builtins", "str"):
+                raise InvalidModelError("Sklearn state keys must be strings.")
+
+
+def _memoized_hash(cache, key, domain: bytes, value) -> str:
+    if key not in cache:
+        cache[key] = _hash(domain, value)
+    return cache[key]
+
+
+def _record_key(value):
+    if isinstance(value, dict):
+        return tuple(sorted(value.items()))
+    return tuple(value) if isinstance(value, list) else value
+
+
+def state_digest(schema: Any, archive, resolve) -> str:
+    """Merkle hash all state and buffers, without expanding shared references."""
+    memo: dict[int, str] = {}
+    buffers: dict[str, str] = {}
+    scalars: dict[tuple[type, Any], str] = {}
+    records: dict[tuple, str] = {}
+
+    def record(domain: bytes, value) -> str:
+        key = (domain, _record_key(value))
+        return _memoized_hash(records, key, domain, value)
+
+    def digest(value: Any) -> str:
+        if isinstance(value, dict):
+            node = resolve(value) if "__loader__" in value else value
+            if id(node) in memo:
+                return memo[id(node)]
+            content = {}
+            for key, child in node.items():
+                if key == "__id__":
+                    continue
+                if key == "file":
+                    if child not in buffers:
+                        buffers[child] = _buffer_digest(archive.read(child))
+                    content[key] = record(b"file:", buffers[child])
+                else:
+                    content[key] = digest(child)
+            result = record(b"dict:", content)
+            memo[id(node)] = result
+            return result
+        if isinstance(value, list):
+            return record(b"list:", [digest(child) for child in value])
+        key = (type(value), value.hex() if type(value) is float else value)
+        return _memoized_hash(scalars, key, b"scalar:", value)
+
+    return digest(schema)