ml-reliability-toolkit 0.1.0__py3-none-any.whl

ml_reliability/checks/regression_metrics.py

@@ -0,0 +1,202 @@
+import math
+from math import sqrt
+from typing import Any
+
+from ml_reliability.core.result import CheckResult
+
+
+def check_regression_metrics(
+    model: Any,
+    X_test: Any,
+    y_test: Any,
+    max_acceptable_mae: float | None = None,
+    max_scoring_mae: float | None = None,
+) -> CheckResult:
+    """Calculate regression MAE and RMSE and optionally assess quality."""
+
+    check_id = "model.regression_metrics"
+    title = "Regression Metrics"
+    category = "performance"
+
+    def configuration_error(message: str) -> CheckResult:
+        return CheckResult(
+            check_id=check_id,
+            title=title,
+            category=category,
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message=message,
+            recommendation="Correct the regression metric configuration.",
+        )
+
+    for name, value in (
+        ("max_acceptable_mae", max_acceptable_mae),
+        ("max_scoring_mae", max_scoring_mae),
+    ):
+        if value is None:
+            continue
+
+        if (
+            isinstance(value, bool)
+            or not isinstance(value, (int, float))
+            or not math.isfinite(value)
+            or value < 0
+            or (name == "max_scoring_mae" and value == 0)
+        ):
+            requirement = (
+                "a finite, strictly positive number"
+                if name == "max_scoring_mae"
+                else "a finite, non-negative number"
+            )
+            return configuration_error(f"{name} must be {requirement}.")
+
+    if model is None:
+        return CheckResult(
+            check_id=check_id,
+            title=title,
+            category=category,
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="A trained model is required.",
+            recommendation="Provide a trained model with a predict() method.",
+        )
+
+    if X_test is None or y_test is None:
+        return CheckResult(
+            check_id=check_id,
+            title=title,
+            category=category,
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="X_test and y_test are required.",
+            recommendation="Provide both X_test and y_test.",
+        )
+
+    if not callable(getattr(model, "predict", None)):
+        return CheckResult(
+            check_id=check_id,
+            title=title,
+            category=category,
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="The model must provide a callable predict() method.",
+            recommendation="Provide a trained model implementing predict().",
+        )
+
+    try:
+        predictions = list(model.predict(X_test))
+        actual = list(y_test)
+
+        for name, values in (
+            ("predictions", predictions),
+            ("targets", actual),
+        ):
+            for index, value in enumerate(values):
+                if (
+                    isinstance(value, bool)
+                    or not isinstance(value, (int, float))
+                    or not math.isfinite(value)
+                ):
+                    raise ValueError(
+                        f"Non-finite or invalid numeric value in "
+                        f"{name} at index {index}: {value!r}."
+                    )
+
+        if not actual:
+            raise ValueError("y_test must contain at least one sample.")
+
+        if len(actual) != len(predictions):
+            raise ValueError("Predictions and y_test must have the same length.")
+
+        absolute_errors = [
+            abs(prediction - target) for prediction, target in zip(predictions, actual)
+        ]
+        mae = sum(absolute_errors) / len(actual)
+
+        squared_errors = [
+            (prediction - target) ** 2
+            for prediction, target in zip(predictions, actual)
+        ]
+        rmse = sqrt(sum(squared_errors) / len(actual))
+
+        if not math.isfinite(mae) or not math.isfinite(rmse):
+            raise ValueError(
+                "Calculated MAE and RMSE must be finite; "
+                f"received mae={mae!r}, rmse={rmse!r}."
+            )
+
+        passed = True if max_acceptable_mae is None else mae <= max_acceptable_mae
+
+        if max_scoring_mae is not None:
+            score = max(
+                0.0,
+                min(100.0, (1.0 - mae / max_scoring_mae) * 100.0),
+            )
+        elif max_acceptable_mae is not None:
+            score = 100.0 if passed else 0.0
+        else:
+            score = None
+
+        severity = "info" if max_acceptable_mae is None or passed else "warning"
+
+        message = (
+            f"Regression metrics calculated successfully: "
+            f"mae={mae:.4f}, rmse={rmse:.4f}."
+        )
+
+        if max_acceptable_mae is None:
+            message += (
+                " No MAE tolerance configured; "
+                "model quality was not assessed against a pass/fail threshold."
+            )
+        else:
+            message += (
+                f" MAE tolerance={max_acceptable_mae:.4f}; "
+                f"tolerance {'met' if passed else 'exceeded'}."
+            )
+
+        if max_scoring_mae is not None:
+            message += f" Scoring scale={max_scoring_mae:.4f}; score={score:.2f}/100."
+
+        recommendation = (
+            ""
+            if passed
+            else (
+                "Investigate prediction errors or revise the configured MAE tolerance."
+            )
+        )
+
+        return CheckResult(
+            check_id=check_id,
+            title=title,
+            category=category,
+            passed=passed,
+            severity=severity,
+            score=score,
+            message=message,
+            evidence={
+                "metric": "mae",
+                "mae": round(float(mae), 4),
+                "rmse": round(float(rmse), 4),
+                "test_samples": len(actual),
+                "max_acceptable_mae": max_acceptable_mae,
+                "max_scoring_mae": max_scoring_mae,
+            },
+            recommendation=recommendation,
+        )
+
+    except Exception as exc:  # noqa: BLE001 - Convert evaluation failures to check results.
+        return CheckResult(
+            check_id=check_id,
+            title=title,
+            category=category,
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message=f"Regression metric calculation failed: {exc}",
+            recommendation=("Verify that predictions and test targets are valid."),
+        )

ml_reliability/checks/schema.py

@@ -0,0 +1,94 @@
+from typing import Any
+
+from ml_reliability.core.result import CheckResult
+
+
+def check_schema(X: Any) -> CheckResult:
+    """
+    Validate the basic structure of a tabular dataset.
+    """
+
+    if X is None:
+        return CheckResult(
+            check_id="data.schema",
+            title="Dataset Schema",
+            category="data_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Input dataset is missing.",
+            recommendation="Provide a valid pandas DataFrame.",
+        )
+
+    if not hasattr(X, "shape") or not hasattr(X, "columns") or not hasattr(X, "dtypes"):
+        return CheckResult(
+            check_id="data.schema",
+            title="Dataset Schema",
+            category="data_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Input must be a pandas DataFrame.",
+            recommendation="Provide the dataset as a pandas DataFrame.",
+        )
+
+    rows = int(X.shape[0])
+    columns = int(X.shape[1])
+
+    if rows == 0:
+        return CheckResult(
+            check_id="data.schema",
+            title="Dataset Schema",
+            category="data_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Dataset contains zero rows.",
+            evidence={
+                "rows": rows,
+                "columns": columns,
+                "column_names": [str(column) for column in X.columns],
+                "dtypes": {
+                    str(column): str(dtype) for column, dtype in X.dtypes.items()
+                },
+            },
+            recommendation="Provide a dataset containing at least one row.",
+        )
+
+    if columns == 0:
+        return CheckResult(
+            check_id="data.schema",
+            title="Dataset Schema",
+            category="data_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Dataset contains zero columns.",
+            evidence={
+                "rows": rows,
+                "columns": columns,
+                "column_names": [],
+                "dtypes": {},
+            },
+            recommendation="Provide a dataset containing at least one column.",
+        )
+
+    column_names = [str(column) for column in X.columns]
+
+    dtypes = {str(column): str(dtype) for column, dtype in X.dtypes.items()}
+
+    return CheckResult(
+        check_id="data.schema",
+        title="Dataset Schema",
+        category="data_quality",
+        passed=True,
+        severity="info",
+        score=100.0,
+        message="Dataset schema is valid.",
+        evidence={
+            "rows": rows,
+            "columns": columns,
+            "column_names": column_names,
+            "dtypes": dtypes,
+        },
+    )

ml_reliability/checks/slice_analysis.py

@@ -0,0 +1,155 @@
+from typing import Any
+
+from ml_reliability.core.result import CheckResult
+
+
+def check_slice_analysis(
+    model: Any,
+    X_test: Any,
+    y_test: Any,
+    slices: Any,
+    task: str,
+) -> CheckResult:
+    """
+    Calculate classification accuracy separately for each slice.
+
+    The current implementation supports classification only.
+    """
+
+    if model is None:
+        return CheckResult(
+            check_id="model.slice_analysis",
+            title="Slice Analysis",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="A trained model is required.",
+            recommendation="Provide a trained model with a predict() method.",
+        )
+
+    if X_test is None or y_test is None or slices is None:
+        return CheckResult(
+            check_id="model.slice_analysis",
+            title="Slice Analysis",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="X_test, y_test, and slices are required.",
+            recommendation="Provide test data, targets, and slice labels.",
+        )
+
+    if task != "classification":
+        return CheckResult(
+            check_id="model.slice_analysis",
+            title="Slice Analysis",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Slice analysis currently supports classification only.",
+            recommendation="Use task='classification'.",
+        )
+
+    if not hasattr(model, "predict") or not callable(model.predict):
+        return CheckResult(
+            check_id="model.slice_analysis",
+            title="Slice Analysis",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="The model must provide a callable predict() method.",
+            recommendation="Provide a trained model implementing predict().",
+        )
+
+    try:
+        predictions = list(model.predict(X_test))
+        actual = list(y_test)
+        slice_labels = list(slices)
+
+        if not actual:
+            raise ValueError("y_test must contain at least one sample.")
+
+        if len(actual) != len(predictions):
+            raise ValueError("Predictions and y_test must have the same length.")
+
+        if len(actual) != len(slice_labels):
+            raise ValueError("Slices and y_test must have the same length.")
+
+        grouped_results = {}
+
+        for prediction, target, slice_label in zip(
+            predictions,
+            actual,
+            slice_labels,
+        ):
+            if slice_label not in grouped_results:
+                grouped_results[slice_label] = {
+                    "correct": 0,
+                    "total": 0,
+                }
+
+            grouped_results[slice_label]["total"] += 1
+
+            if prediction == target:
+                grouped_results[slice_label]["correct"] += 1
+
+        slice_metrics = {}
+
+        for slice_label, values in grouped_results.items():
+            accuracy = values["correct"] / values["total"]
+
+            slice_metrics[str(slice_label)] = {
+                "accuracy": round(float(accuracy), 4),
+                "samples": values["total"],
+            }
+
+        accuracies = [metrics["accuracy"] for metrics in slice_metrics.values()]
+
+        slice_accuracy_gap = max(accuracies) - min(accuracies)
+
+        score = round(
+            max(0.0, min(100.0, (1.0 - slice_accuracy_gap) * 100.0)),
+            2,
+        )
+
+        return CheckResult(
+            check_id="model.slice_analysis",
+            title="Slice Analysis",
+            category="performance",
+            passed=True,
+            severity="info",
+            score=score,
+            message=(
+                "Classification accuracy calculated for each slice. "
+                f"Accuracy gap: {slice_accuracy_gap:.4f}. "
+                f"Slice consistency score: {score:.2f}/100."
+            ),
+            evidence={
+                "task": "classification",
+                "metric": "accuracy",
+                "slices": slice_metrics,
+                "slice_accuracy_gap": round(
+                    float(slice_accuracy_gap),
+                    4,
+                ),
+                "test_samples": len(actual),
+            },
+            recommendation="",
+        )
+
+    except Exception as exc:  # noqa: BLE001 - Convert evaluation failures to check results.
+        return CheckResult(
+            check_id="model.slice_analysis",
+            title="Slice Analysis",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message=f"Slice analysis failed: {exc}",
+            recommendation=(
+                "Verify that predictions, targets, and slice labels are valid."
+            ),
+        )

ml_reliability/checks/stress/__init__.py

@@ -0,0 +1,41 @@
+from .base import (
+    StressTestOutcome,
+    calculate_degradation,
+    validate_rate,
+)
+from .categorical import (
+    evaluate_categorical_corruption_stress,
+    inject_categorical_corruption,
+)
+from .distribution import (
+    evaluate_distribution_perturbation_stress,
+    perturb_feature_distribution,
+)
+from .feature_dropout import (
+    apply_feature_dropout,
+    evaluate_feature_dropout_stress,
+)
+from .missing_values import (
+    evaluate_missing_value_stress,
+    inject_missing_values,
+)
+from .noise import (
+    evaluate_noise_stress,
+    inject_feature_noise,
+)
+
+__all__ = [
+    "StressTestOutcome",
+    "apply_feature_dropout",
+    "calculate_degradation",
+    "evaluate_categorical_corruption_stress",
+    "evaluate_distribution_perturbation_stress",
+    "evaluate_feature_dropout_stress",
+    "evaluate_missing_value_stress",
+    "evaluate_noise_stress",
+    "inject_categorical_corruption",
+    "inject_feature_noise",
+    "inject_missing_values",
+    "perturb_feature_distribution",
+    "validate_rate",
+]

ml_reliability/checks/stress/base.py

@@ -0,0 +1,74 @@
+import math
+from dataclasses import dataclass, field
+from typing import Any
+
+import numpy as np
+
+
+def _is_finite_number(value: Any) -> bool:
+    """Return whether a supported numeric value is finite."""
+    try:
+        return math.isfinite(value)
+    except OverflowError:
+        return False
+
+
+@dataclass
+class StressTestOutcome:
+    """Standard result produced by a stress test.
+
+    The ``passed`` field indicates whether the stress evaluation
+    completed successfully. It does not indicate that the model
+    was robust to the applied stress.
+
+    Metric validation is handled by the calculation and reporting
+    layers so failed evaluations can preserve diagnostic results.
+    """
+
+    stress_type: str
+    baseline_metric: float
+    stressed_metric: float
+    degradation: float
+    passed: bool
+    evidence: dict[str, Any] = field(default_factory=dict)
+
+
+def calculate_degradation(
+    baseline_metric: float,
+    stressed_metric: float,
+) -> float:
+    """Calculate degradation from two finite numeric metrics."""
+
+    metrics = (baseline_metric, stressed_metric)
+
+    if any(
+        isinstance(metric, (bool, np.bool_))
+        or not isinstance(
+            metric,
+            (int, float, np.integer, np.floating),
+        )
+        or not _is_finite_number(metric)
+        for metric in metrics
+    ):
+        raise ValueError("baseline_metric and stressed_metric must be finite numbers")
+
+    degradation = float(baseline_metric) - float(stressed_metric)
+
+    if not math.isfinite(degradation):
+        raise ValueError("degradation must be finite")
+
+    return round(degradation, 10)
+
+
+def validate_rate(rate: float) -> float:
+    """Validate that a stress rate is finite and between 0 and 1."""
+
+    if (
+        isinstance(rate, (bool, np.bool_))
+        or not isinstance(rate, (int, float, np.integer, np.floating))
+        or not _is_finite_number(rate)
+        or not 0.0 <= rate <= 1.0
+    ):
+        raise ValueError("rate must be between 0.0 and 1.0")
+
+    return rate