mlbundle 0.1.0a1__py3-none-any.whl

mlbundle/_catboost/native.py

@@ -0,0 +1,244 @@
+"""Numeric CBM preflight before CatBoost's native zero-copy loader sees bytes."""
+
+import math
+import struct
+from dataclasses import dataclass
+from typing import Any, cast
+
+import numpy as np
+
+from mlbundle.canonical import canonical_loads
+from mlbundle.errors import InvalidModelError
+from mlbundle.limits import (
+    HYPERPARAMETER_BYTES,
+    MANIFEST_BYTES,
+    MAX_CLASSES,
+    MAX_FEATURES,
+    ArchiveLimits,
+)
+
+from .budget import structure_budget
+from .flatbuffer import Buffer, Table
+from .scope import CLASSIFIERS, MAX_DEPTH, MAX_TREES, REGRESSORS
+
+
+@dataclass
+class Snapshot:
+    native: dict[str, Any]
+    metadata: dict[str, str]
+    loss: str
+    width: int
+    dimension: int
+
+
+def _features(tree: Table) -> list[dict[str, Any]]:
+    records = []
+    remaining = MAX_TREES * MAX_DEPTH
+    for feature in tree.tables(5, 6, MAX_FEATURES):
+        has_nans = feature.scalar(0, "B")
+        index, flat = (feature.scalar(i, "i", -1) for i in (1, 2))
+        treatment = feature.scalar(5, "b")
+        borders = feature.vector(3, "<f4", MAX_TREES * MAX_DEPTH)
+        remaining -= len(borders)
+        if (
+            remaining < 0
+            or index != len(records)
+            or flat != len(records)
+            or has_nans not in (0, 1)
+            or treatment not in (0, 1, 2)
+            or not 0 <= index < MAX_FEATURES
+            or not 0 <= flat < MAX_FEATURES
+            or not np.isfinite(borders).all()
+            or np.any(borders[1:] <= borders[:-1])
+        ):
+            raise InvalidModelError("Invalid CatBoost numeric feature record.")
+        name = feature.string(4, 4096)
+        if len(name) > 1024:
+            raise InvalidModelError("CatBoost feature name exceeds the limit.")
+        records.append(
+            {
+                "index": index,
+                "flat_index": flat,
+                "name": name,
+                "has_nans": bool(has_nans),
+                "nan_value_treatment": treatment,
+                "borders": borders,
+            }
+        )
+    if (
+        not records
+        or [r["index"] for r in records] != list(range(len(records)))
+        or [r["flat_index"] for r in records] != list(range(len(records)))
+    ):
+        raise InvalidModelError("Invalid CatBoost numeric feature order.")
+    return records
+
+
+def _metadata(core: Table) -> dict[str, str]:
+    result = {}
+    total = 0
+    for item in core.tables(2, 2, 4096):
+        key, value = item.string(0, 1024), item.string(1, MANIFEST_BYTES)
+        total += len(key.encode()) + len(value.encode())
+        if not key or key in result or total > MANIFEST_BYTES:
+            raise InvalidModelError("Invalid or oversized CatBoost metadata.")
+        result[key] = value
+    for key in ("training", "output_options"):
+        if key in result:
+            canonical_loads(result[key].encode(), max_bytes=MANIFEST_BYTES)
+    return result
+
+
+def _classes(metadata: dict[str, str], count: int) -> None:
+    classes = canonical_loads(
+        metadata["class_params"].encode(), max_bytes=HYPERPARAMETER_BYTES
+    )
+    if not isinstance(classes, dict):
+        raise InvalidModelError("Invalid CatBoost native class record.")
+    mapping, names = classes["class_to_label"], classes["class_names"]
+    if (
+        not isinstance(mapping, list)
+        or len(mapping) != count
+        or not isinstance(names, list)
+        or len(names) not in (0, count)
+    ):
+        raise InvalidModelError("Invalid CatBoost native class count.")
+    declared = classes["classes_count"]
+    if type(declared) is not int or declared not in (0, count):
+        raise InvalidModelError("Invalid CatBoost declared class count.")
+    if classes.get("class_label_type") not in (
+        "Integer",
+        "Float",
+        "String",
+        "Boolean",
+    ) or any(
+        type(label) not in (int, float) or not math.isfinite(cast(float, label))
+        for label in mapping
+    ):
+        raise InvalidModelError("Invalid CatBoost native class mapping.")
+
+
+def _loss(metadata: dict[str, str], dimension: int) -> str:
+    params = canonical_loads(
+        metadata["params"].encode(), max_bytes=HYPERPARAMETER_BYTES, max_depth=16
+    )
+    if not isinstance(params, dict):
+        raise InvalidModelError("Invalid CatBoost native parameters.")
+    params = cast(dict[str, Any], params)
+    loss = params["loss_function"]["type"]
+    if (
+        loss not in CLASSIFIERS | REGRESSORS
+        or params["tree_learner_options"]["grow_policy"] != "SymmetricTree"
+    ):
+        raise InvalidModelError("Unsupported CatBoost native loss or tree policy.")
+    if loss == "MultiClass":
+        if not 2 <= dimension <= MAX_CLASSES:
+            raise InvalidModelError("Invalid CatBoost multiclass dimension.")
+    elif dimension != 1:
+        raise InvalidModelError("CatBoost requires a single target.")
+    for metric in [
+        params["metrics"]["eval_metric"],
+        *params["metrics"]["custom_metrics"],
+    ]:
+        if metric["type"].startswith("Python"):
+            raise InvalidModelError("Custom Python CatBoost metrics are unsupported.")
+    if loss in CLASSIFIERS:
+        _classes(metadata, dimension if loss == "MultiClass" else 2)
+    return loss
+
+
+def _trees(tree: Table) -> dict[str, Any]:
+    dimension = tree.scalar(0, "i")
+    if not 1 <= dimension <= MAX_CLASSES:
+        raise InvalidModelError("Invalid CatBoost output dimension.")
+    depths = tree.vector(2, "<i4", MAX_TREES)
+    offsets = tree.vector(3, "<i4", MAX_TREES)
+    splits = tree.vector(1, "<i4", MAX_TREES * MAX_DEPTH)
+    if (
+        not len(depths)
+        or np.any(depths < 0)
+        or np.any(depths > MAX_DEPTH)
+        or len(offsets) != len(depths)
+    ):
+        raise InvalidModelError("Invalid CatBoost tree count or depth.")
+    expected = np.concatenate(([0], np.cumsum(depths[:-1], dtype=np.int64)))
+    if not np.array_equal(offsets, expected) or len(splits) != int(depths.sum()):
+        raise InvalidModelError("Invalid CatBoost tree split offsets.")
+    leaves = int(np.left_shift(np.int64(1), depths).sum())
+    values = tree.vector(8, "<f8", ArchiveLimits().max_member_bytes // 8)
+    weights = tree.vector(9, "<f8", ArchiveLimits().max_member_bytes // 8)
+    if (
+        len(values) != leaves * dimension
+        or len(weights) != leaves
+        or not np.isfinite(values).all()
+        or not np.isfinite(weights).all()
+        or np.any(weights < 0)
+    ):
+        raise InvalidModelError("Invalid CatBoost leaf values or weights.")
+    features = _features(tree)
+    border_count = sum(len(feature["borders"]) for feature in features)
+    if np.any(splits < 0) or np.any(splits >= border_count):
+        raise InvalidModelError("Invalid CatBoost split feature indices.")
+    scale, legacy_bias = tree.scalar(14, "d", 1.0), tree.scalar(15, "d", 0.0)
+    bias = tree.vector(16, "<f8", MAX_CLASSES)
+    if (
+        not math.isfinite(scale)
+        or not math.isfinite(legacy_bias)
+        or len(bias) not in (0, dimension)
+        or not np.isfinite(bias).all()
+    ):
+        raise InvalidModelError("Invalid CatBoost scale or bias.")
+    if len(bias) and legacy_bias != 0.0:
+        raise InvalidModelError("Ambiguous CatBoost bias records.")
+    repacked = tree.vector(17, "<u4", MAX_TREES * MAX_DEPTH)
+    if len(repacked) not in (0, len(splits)):
+        raise InvalidModelError("Invalid CatBoost repacked split count.")
+    return {
+        "approx_dimension": dimension,
+        "tree_depths": depths,
+        "tree_split_offsets": offsets,
+        "tree_splits": splits,
+        "leaf_values": values,
+        "leaf_weights": weights,
+        "float_features": features,
+        "scale": scale,
+        "bias": bias if len(bias) else np.full(dimension, legacy_bias),
+        "repacked_bins": repacked,
+    }
+
+
+def decode(data: bytes) -> Snapshot:
+    if (
+        type(data) is not bytes
+        or len(data) > ArchiveLimits().max_member_bytes
+        or len(data) < 12
+        or data[:4] != b"CBM1"
+    ):
+        raise InvalidModelError("Expected a bounded native CatBoost CBM model.")
+    size = struct.unpack_from("<I", data, 4)[0]
+    if size != len(data) - 8:
+        raise InvalidModelError(
+            "Truncated CBM core or unsupported CatBoost model parts."
+        )
+    buffer = Buffer(memoryview(data)[8:])
+    core = buffer.table(buffer.indirect(0), 4)
+    if core.string(0, 256) != "FlabuffersModel_v1" or len(core.vector(3, "<u4", 0)):
+        raise InvalidModelError("Unsupported CatBoost core version or model parts.")
+    tree = core.child(1, 19)
+    # No CTR parser, feature calcer, or non-oblivious traversal reaches native code.
+    for field in (4, 6, 7, 10, 11, 12, 13, 18):
+        if len(tree.vector(field, "<u4", 0)):
+            raise InvalidModelError(
+                "Unsupported CatBoost feature or non-symmetric tree state."
+            )
+    native = _trees(tree)
+    structure_budget(native)
+    metadata = _metadata(core)
+    loss = _loss(metadata, native["approx_dimension"])
+    return Snapshot(
+        native,
+        metadata,
+        loss,
+        len(native["float_features"]),
+        native["approx_dimension"],
+    )

mlbundle/_catboost/prediction.py

@@ -0,0 +1,67 @@
+"""Numeric CatBoost input and output validation without native exception contents."""
+
+from typing import Any, cast
+
+import numpy as np
+
+from mlbundle._prediction_input import tabular
+from mlbundle.adapters import ProbabilityOutput, RegressionOutput
+from mlbundle.claims import ModelClaims
+from mlbundle.contract import ModelKind
+from mlbundle.errors import InvalidModelError, MLBundleError, PredictionInputError
+
+
+class Predictor:
+    def __init__(self, model: Any, claims: ModelClaims) -> None:
+        self.model, self.claims = model, claims
+
+    def _call(self, data: object, *, probabilities: bool) -> tuple[np.ndarray, int]:
+        try:
+            inputs, rows = tabular(data, self.claims.n_features)
+            if rows < 1:
+                raise PredictionInputError(
+                    "CatBoost prediction requires at least one row."
+                )
+            if (
+                hasattr(inputs, "columns")
+                and list(inputs.columns) != self.claims.feature_names
+            ):
+                raise PredictionInputError(
+                    "CatBoost prediction columns must match native feature order."
+                )
+            raw = (
+                self.model.predict_proba(inputs, thread_count=1)
+                if probabilities
+                else self.model.predict(inputs, thread_count=1)
+            )
+            return np.asarray(raw, dtype=float), rows
+        except MLBundleError:
+            raise
+        except Exception:
+            raise InvalidModelError("CatBoost prediction failed.") from None
+
+    def predict(self, data: object) -> RegressionOutput:
+        if self.claims.kind != ModelKind.REGRESSOR:
+            raise ValueError("predict requires a regressor.")
+        values, rows = self._call(data, probabilities=False)
+        if values.shape != (rows,) or not np.isfinite(values).all():
+            raise InvalidModelError(
+                "Invalid regression prediction dimensions or values."
+            )
+        return cast(RegressionOutput, values.tolist())
+
+    def predict_proba(self, data: object) -> ProbabilityOutput:
+        if self.claims.kind != ModelKind.CLASSIFIER:
+            raise ValueError("predict_proba requires a classifier.")
+        values, rows = self._call(data, probabilities=True)
+        if (
+            values.shape != (rows, (self.claims.n_classes or [0])[0])
+            or not np.isfinite(values).all()
+            or np.any(values < 0)
+            or np.any(values > 1)
+            or not np.allclose(values.sum(axis=1), 1.0, rtol=1e-6, atol=1e-7)
+        ):
+            raise InvalidModelError(
+                "Invalid probability dimensions, values or row sums."
+            )
+        return cast(ProbabilityOutput, values.tolist())

mlbundle/_catboost/scope.py

@@ -0,0 +1,109 @@
+"""Exact wrappers, verified losses and numeric symmetric trees only."""
+
+from typing import Any
+
+import catboost as cb
+
+from mlbundle.canonical import canonical_dumps
+from mlbundle.errors import InvalidJSONError, UnsupportedModelError
+from mlbundle.limits import HYPERPARAMETER_BYTES
+
+CLASSIFIERS = frozenset({"Logloss", "CrossEntropy", "MultiClass"})
+REGRESSORS = frozenset({"RMSE", "MAE", "Quantile", "Huber", "Poisson", "MAPE"})
+CLASSES = (cb.CatBoostClassifier, cb.CatBoostRegressor)
+NATIVE_CLASS = type(vars(cb.CatBoostClassifier())["_object"])
+MAX_TREES = 100_000
+MAX_DEPTH = 16
+WRAPPER_FIELDS = frozenset(
+    {
+        "_init_params",
+        "_canonized_params",
+        "_object",
+        "_is_fitted_",
+        "_random_seed",
+        "_learning_rate",
+        "_tree_count",
+        "_n_features_in",
+        "_prediction_values_change",
+        "_loss_value_change",
+    }
+)
+
+
+def _parameters(params: dict[str, Any]) -> None:
+    for key in ("loss_function", "objective", "eval_metric"):
+        if params.get(key) is not None and not isinstance(params[key], str):
+            raise UnsupportedModelError(
+                reason="Custom Python CatBoost objectives and metrics are unsupported."
+            )
+    metrics = params.get("custom_metric", [])
+    if isinstance(metrics, str):
+        metrics = [metrics]
+    if metrics is not None and any(not isinstance(metric, str) for metric in metrics):
+        raise UnsupportedModelError(
+            reason="Custom Python CatBoost objectives and metrics are unsupported."
+        )
+    try:
+        canonical_dumps(params, max_bytes=HYPERPARAMETER_BYTES)
+    except InvalidJSONError:
+        raise UnsupportedModelError(
+            reason="CatBoost parameters must contain only JSON values."
+        ) from None
+
+
+def _wrapper(model: Any, loss: str) -> None:
+    cached = model._canonized_params
+    params = model.get_params() if cached is None else cached
+    if not isinstance(params, dict):
+        raise UnsupportedModelError(reason="Invalid CatBoost wrapper parameter cache.")
+    _parameters(params)
+    if type(model) is cb.CatBoostRegressor:
+        declared = params.get("loss_function", params.get("objective"))
+        mode = "RawFormulaVal"
+        if isinstance(declared, str):
+            if declared.startswith(("Poisson", "Tweedie")):
+                mode = "Exponent"
+            elif declared == "RMSEWithUncertainty":
+                mode = "RMSEWithUncertainty"
+        if mode != ("Exponent" if loss == "Poisson" else "RawFormulaVal"):
+            raise UnsupportedModelError(
+                reason="CatBoost wrapper default prediction does not survive CBM loading."
+            )
+
+
+def validate(model: Any) -> None:
+    if type(model) not in CLASSES:
+        raise UnsupportedModelError(
+            reason="Only exact CatBoostClassifier and CatBoostRegressor are supported."
+        )
+    if set(vars(model)) - WRAPPER_FIELDS:
+        raise UnsupportedModelError(
+            reason="Unknown CatBoost wrapper attributes are unsupported."
+        )
+    if type(vars(model).get("_object")) is not NATIVE_CLASS:
+        raise UnsupportedModelError(
+            reason="Only the native CatBoost model object is supported."
+        )
+    if not model.is_fitted():
+        raise UnsupportedModelError(reason="CatBoost estimator must be fitted.")
+    _parameters(model.get_params())
+    if model.get_cat_feature_indices():
+        raise UnsupportedModelError(
+            reason="CatBoost categorical features are unsupported."
+        )
+    if model.get_text_feature_indices() or model.get_embedding_feature_indices():
+        raise UnsupportedModelError(
+            reason="CatBoost text and embedding features are unsupported."
+        )
+    native = model.get_all_params()
+    loss = native["loss_function"].split(":", 1)[0]
+    allowed = CLASSIFIERS if type(model) is cb.CatBoostClassifier else REGRESSORS
+    if loss not in allowed:
+        raise UnsupportedModelError(
+            reason="Unsupported CatBoost loss; only closed single-target losses are supported."
+        )
+    if native.get("grow_policy") != "SymmetricTree":
+        raise UnsupportedModelError(
+            reason="Only CatBoost SymmetricTree is supported; Depthwise/Lossguide are unsupported."
+        )
+    _wrapper(model, loss)

mlbundle/_catboost/state.py

@@ -0,0 +1,62 @@
+"""Recomputed native snapshots and commitments independent of CBM buffer layout."""
+
+from hashlib import sha256
+from typing import Any
+
+import numpy as np
+
+from mlbundle.canonical import canonical_dumps
+from mlbundle.errors import InvalidModelError
+from mlbundle.limits import MANIFEST_BYTES
+
+from .native import Snapshot, decode
+
+
+def snapshot(model: Any) -> Snapshot:
+    # CatBoost has no public in-memory save. This audited native CBM serializer
+    # lets extraction recompute loaded state without reader-side disk writes.
+    return decode(model._serialize_model())
+
+
+def commitment(state: Snapshot) -> str:
+    def digest(value: Any) -> str:
+        if isinstance(value, np.ndarray):
+            header = canonical_dumps([value.dtype.str, list(value.shape)])
+            result = sha256(b"array:")
+            result.update(header)
+            result.update(memoryview(value))
+            return result.hexdigest()
+        if isinstance(value, dict):
+            record = {key: digest(child) for key, child in value.items()}
+            return sha256(b"dict:" + canonical_dumps(record)).hexdigest()
+        if isinstance(value, list):
+            return sha256(
+                b"list:" + canonical_dumps([digest(child) for child in value])
+            ).hexdigest()
+        return sha256(
+            b"scalar:" + canonical_dumps(value, max_bytes=MANIFEST_BYTES)
+        ).hexdigest()
+
+    return digest({"native_model": state.native, "metadata": state.metadata})
+
+
+def json_value(value: Any) -> Any:
+    if isinstance(value, np.ndarray):
+        return value.tolist()
+    if isinstance(value, dict):
+        return {key: json_value(child) for key, child in value.items()}
+    if isinstance(value, list):
+        return [json_value(child) for child in value]
+    return value
+
+
+def feature_importances(model: Any, width: int) -> list[float]:
+    values = np.asarray(
+        model.copy().get_feature_importance(
+            type="PredictionValuesChange", thread_count=1
+        ),
+        dtype=float,
+    )
+    if values.shape != (width,) or not np.isfinite(values).all() or np.any(values < 0):
+        raise InvalidModelError("Invalid CatBoost feature importances.")
+    return values.tolist()

mlbundle/_composite/__init__.py

@@ -0,0 +1 @@
+"""Separate checked components, closed sklearn reconstruction."""

mlbundle/_composite/claims.py

@@ -0,0 +1,133 @@
+"""One composite extraction for export and load."""
+
+from typing import Any
+
+from pydantic import JsonValue
+from sklearn.pipeline import Pipeline
+
+from mlbundle._pipeline_contract import (
+    NativeComponentInfo,
+    PipelineInfo,
+    TransformedFeatures,
+)
+from mlbundle._sklearn import transformers
+from mlbundle._sklearn.pipeline_scope import validate_components
+from mlbundle._sklearn.transform_names import input_boundary, output_names
+from mlbundle._sklearn.transform_width import output_width
+from mlbundle.canonical import canonical_dumps
+from mlbundle.claims import ModelClaims
+from mlbundle.contract import MODEL_MEMBERS, Library
+from mlbundle.errors import ClaimMismatchError, InvalidModelError, MLBundleError
+
+
+def outer_claims(
+    prefix: Pipeline | None, leaf: ModelClaims, verbose: bool
+) -> ModelClaims:
+    count, names = input_boundary(prefix, leaf)
+    return ModelClaims(
+        estimator_class="Pipeline",
+        kind=leaf.kind,
+        hyperparameters={"memory": None, "verbose": verbose, "transform_input": None},
+        n_features=count,
+        n_classes=leaf.n_classes,
+        n_outputs=leaf.n_outputs,
+        feature_names=names,
+        feature_dtypes=leaf.feature_dtypes if prefix is None else None,
+        class_labels=leaf.class_labels,
+    )
+
+
+def extract(
+    prefix: Pipeline | None,
+    leaf: ModelClaims,
+    library: Library,
+    step_names: list[str],
+    verbose: bool,
+    *,
+    names: list[str],
+    transformer_adapter: transformers.Adapter,
+) -> PipelineInfo:
+    try:
+        if library not in (Library.XGBOOST, Library.CATBOOST):
+            raise InvalidModelError("Invalid composite Pipeline state.")
+        outer = outer_claims(prefix, leaf, verbose)
+        count = (
+            outer.n_features
+            if prefix is None
+            else output_width(prefix, outer.n_features)
+        )
+        if count != leaf.n_features:
+            raise InvalidModelError(
+                "Transformer output width disagrees with final estimator."
+            )
+        transformed = names if prefix is None else output_names(prefix, names)
+        origin = "input" if prefix is None else "transformer"
+        if transformed is None:
+            transformed = leaf.feature_names
+            origin = "model"
+        if transformed is None:
+            transformed = [f"transformed_feature_{i}" for i in range(1, count + 1)]
+            origin = "generated"
+        if len(transformed) != count:
+            raise InvalidModelError("Transformed names and width disagree.")
+        if prefix is not None:
+            validate_components(prefix)
+        return PipelineInfo(
+            step_names=step_names,
+            transformers=None if prefix is None else transformer_adapter.claims(prefix),
+            estimator=NativeComponentInfo(
+                library=library,
+                serialization=MODEL_MEMBERS[library][0],
+                member=MODEL_MEMBERS[library][1],
+                claims=leaf,
+            ),
+            transformed_features=TransformedFeatures(
+                n_features=count, names=transformed, origin=origin
+            ),
+        )
+    except MLBundleError:
+        raise
+    except Exception:
+        raise InvalidModelError("Invalid composite Pipeline state.") from None
+
+
+def verify(expected: PipelineInfo, actual: PipelineInfo) -> None:
+    for field in ("step_names", "transformers", "transformed_features"):
+        if canonical_dumps(expected.model_dump(mode="json")[field]) != canonical_dumps(
+            actual.model_dump(mode="json")[field]
+        ):
+            raise ClaimMismatchError("pipeline." + field)
+    left = expected.estimator.model_dump(mode="json")
+    right = actual.estimator.model_dump(mode="json")
+    for field, value in left["claims"].items():
+        if canonical_dumps(value) != canonical_dumps(right["claims"][field]):
+            raise ClaimMismatchError("pipeline.estimator." + field)
+    if canonical_dumps(left) != canonical_dumps(right):
+        raise ClaimMismatchError("pipeline.estimator")
+
+
+def structure(
+    native: dict[str, JsonValue] | None, recipe: PipelineInfo
+) -> dict[str, JsonValue]:
+    if native is None:
+        raise InvalidModelError("Invalid composite Pipeline state.")
+    return {
+        **native,
+        "pipeline": {
+            "step_name": recipe.step_names[-1],
+            "features": recipe.transformed_features.model_dump(mode="json"),
+        },
+    }
+
+
+def rebuild(
+    prefix: Pipeline | None, leaf: Any, step_names: list[str], verbose: bool
+) -> Pipeline:
+    if prefix is not None and [name for name, _ in prefix.steps] != step_names[:-1]:
+        raise ClaimMismatchError("pipeline.transformers")
+    return Pipeline(
+        [*([] if prefix is None else prefix.steps), (step_names[-1], leaf)],
+        memory=None,
+        verbose=verbose,
+        transform_input=None,
+    )