ml-reliability-toolkit 0.1.0__py3-none-any.whl

ml_reliability/__init__.py

@@ -0,0 +1,3 @@
+from ml_reliability.suite import ReliabilityReport, ReliabilitySuite
+
+__all__ = ["ReliabilityReport", "ReliabilitySuite"]

ml_reliability/checks/calibration.py

@@ -0,0 +1,152 @@
+import math
+from typing import Any
+
+from ml_reliability.core.result import CheckResult
+
+
+def check_calibration(
+    model: Any,
+    X_test: Any,
+    y_test: Any,
+) -> CheckResult:
+    """
+    Calculate mean absolute probability error for binary classification.
+
+    The model must provide predict_proba().
+    """
+
+    if model is None:
+        return CheckResult(
+            check_id="model.calibration",
+            title="Classification Calibration",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="A trained model is required.",
+            recommendation="Provide a trained classification model.",
+        )
+
+    if X_test is None or y_test is None:
+        return CheckResult(
+            check_id="model.calibration",
+            title="Classification Calibration",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="X_test and y_test are required.",
+            recommendation="Provide both X_test and y_test.",
+        )
+
+    if not hasattr(model, "predict_proba") or not callable(model.predict_proba):
+        return CheckResult(
+            check_id="model.calibration",
+            title="Classification Calibration",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="The model must provide a callable predict_proba() method.",
+            recommendation=(
+                "Provide a classification model implementing predict_proba()."
+            ),
+        )
+
+    try:
+        probabilities = list(model.predict_proba(X_test))
+        actual = list(y_test)
+
+        if not actual:
+            raise ValueError("y_test must contain at least one sample.")
+
+        if len(actual) != len(probabilities):
+            raise ValueError(
+                "Predicted probabilities and y_test must have the same length."
+            )
+
+        positive_probabilities = []
+
+        for probability in probabilities:
+            if len(probability) != 2:
+                raise ValueError(
+                    "Binary classification probabilities must contain exactly two class probabilities."
+                )
+
+            try:
+                class_probabilities = [
+                    float(probability[0]),
+                    float(probability[1]),
+                ]
+            except (TypeError, ValueError, OverflowError) as exc:
+                raise ValueError(
+                    "Class probabilities must be valid finite numbers."
+                ) from exc
+
+            if not all(math.isfinite(value) for value in class_probabilities):
+                raise ValueError("Class probabilities must be finite numbers.")
+
+            if any(value < 0.0 or value > 1.0 for value in class_probabilities):
+                raise ValueError("Class probabilities must be between 0 and 1.")
+
+            if not math.isclose(
+                sum(class_probabilities),
+                1.0,
+                rel_tol=0.0,
+                abs_tol=1e-6,
+            ):
+                raise ValueError("Binary class probabilities must sum to 1.")
+
+            positive_probabilities.append(class_probabilities[1])
+
+        absolute_errors = [
+            abs(probability - target)
+            for probability, target in zip(
+                positive_probabilities,
+                actual,
+            )
+        ]
+
+        calibration_error = sum(absolute_errors) / len(actual)
+
+        score = round(
+            max(0.0, min(100.0, (1.0 - calibration_error) * 100.0)),
+            2,
+        )
+
+        return CheckResult(
+            check_id="model.calibration",
+            title="Classification Calibration",
+            category="performance",
+            passed=True,
+            severity="info",
+            score=score,
+            message=(
+                "Classification probability calibration error "
+                f"calculated successfully: {calibration_error:.4f}. "
+                f"Calibration score: {score:.2f}/100."
+            ),
+            evidence={
+                "metric": "mean_absolute_probability_error",
+                "calibration_error": round(
+                    float(calibration_error),
+                    4,
+                ),
+                "test_samples": len(actual),
+            },
+            recommendation="",
+        )
+
+    except Exception as exc:  # noqa: BLE001 - Convert evaluation failures to check results.
+        return CheckResult(
+            check_id="model.calibration",
+            title="Classification Calibration",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message=f"Calibration calculation failed: {exc}",
+            recommendation=(
+                "Verify that predicted probabilities and test targets are valid."
+            ),
+        )

ml_reliability/checks/cardinality.py

@@ -0,0 +1,155 @@
+from typing import Any
+
+import pandas as pd
+
+from ml_reliability.core.result import CheckResult
+
+
+def check_cardinality(X: Any) -> CheckResult:
+    """
+    Detect high-cardinality and potentially identifier-like columns.
+    """
+
+    if X is None:
+        return CheckResult(
+            check_id="data.cardinality",
+            title="Cardinality / ID Detection",
+            category="data_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Input data is missing.",
+            recommendation="Provide X_train before running cardinality checks.",
+        )
+
+    if not isinstance(X, pd.DataFrame):
+        return CheckResult(
+            check_id="data.cardinality",
+            title="Cardinality / ID Detection",
+            category="data_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Input must be a pandas DataFrame.",
+            recommendation="Provide tabular data as a pandas DataFrame.",
+        )
+
+    if X.empty:
+        return CheckResult(
+            check_id="data.cardinality",
+            title="Cardinality / ID Detection",
+            category="data_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Input DataFrame contains zero rows.",
+            evidence={
+                "rows": 0,
+                "columns": int(X.shape[1]),
+            },
+            recommendation="Provide a DataFrame containing at least one row.",
+        )
+
+    if X.shape[1] == 0:
+        return CheckResult(
+            check_id="data.cardinality",
+            title="Cardinality / ID Detection",
+            category="data_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Input DataFrame contains zero columns.",
+            evidence={
+                "rows": int(X.shape[0]),
+                "columns": 0,
+            },
+            recommendation="Provide a DataFrame containing at least one feature column.",
+        )
+
+    column_analysis = {}
+    high_cardinality_columns = []
+    identifier_like_columns = []
+    # Cardinality signals are unreliable on very small datasets.
+    # We still report the evidence, but avoid penalizing the dataset.
+    small_dataset = len(X) < 10
+
+    for column in X.columns:
+        total_values = int(X[column].notna().sum())
+        unique_values = int(X[column].nunique(dropna=True))
+
+        if total_values == 0:
+            uniqueness_percentage = 0.0
+        else:
+            uniqueness_percentage = (unique_values / total_values) * 100
+
+        uniqueness_percentage = round(uniqueness_percentage, 2)
+
+        column_analysis[str(column)] = {
+            "total_values": total_values,
+            "unique_values": unique_values,
+            "uniqueness_percentage": uniqueness_percentage,
+        }
+
+        if not small_dataset:
+            if uniqueness_percentage >= 80:
+                high_cardinality_columns.append(str(column))
+
+            if uniqueness_percentage > 95:
+                identifier_like_columns.append(str(column))
+
+    if small_dataset:
+        severity = "info"
+        score = 100.0
+        passed = True
+        message = (
+            "Dataset is too small for reliable cardinality-based identifier detection."
+        )
+    elif identifier_like_columns:
+        severity = "warning"
+        score = 50.0
+        passed = True
+        message = (
+            f"Detected {len(identifier_like_columns)} potentially "
+            "identifier-like column(s)."
+        )
+    elif high_cardinality_columns:
+        severity = "warning"
+        score = 70.0
+        passed = True
+        message = (
+            f"Detected {len(high_cardinality_columns)} high-cardinality column(s)."
+        )
+    else:
+        severity = "info"
+        score = 100.0
+        passed = True
+        message = "No high-cardinality columns were detected."
+
+    return CheckResult(
+        check_id="data.cardinality",
+        title="Cardinality / ID Detection",
+        category="data_quality",
+        passed=passed,
+        severity=severity,
+        score=score,
+        message=message,
+        evidence={
+            "total_columns": int(X.shape[1]),
+            "high_cardinality_columns": high_cardinality_columns,
+            "identifier_like_columns": identifier_like_columns,
+            "columns": column_analysis,
+            "small_dataset": small_dataset,
+            "minimum_rows_for_detection": 10,
+        },
+        recommendation=(
+            "Review identifier-like columns and determine whether they should "
+            "be excluded from model features."
+            if identifier_like_columns
+            else (
+                "Review high-cardinality columns for potential identifiers "
+                "or leakage before model training."
+                if high_cardinality_columns
+                else ""
+            )
+        ),
+    )

ml_reliability/checks/classification_metrics.py

@@ -0,0 +1,151 @@
+from typing import Any
+
+import numpy as np
+
+from ml_reliability.core.result import CheckResult
+
+
+def check_classification_metrics(
+    model: Any,
+    X_test: Any,
+    y_test: Any,
+) -> CheckResult:
+    """
+    Calculate classification precision and recall.
+
+    This check currently supports binary classification.
+    """
+
+    if model is None:
+        return CheckResult(
+            check_id="model.classification_metrics",
+            title="Classification Metrics",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="A trained model is required.",
+            recommendation="Provide a trained model with a predict() method.",
+        )
+
+    if X_test is None or y_test is None:
+        return CheckResult(
+            check_id="model.classification_metrics",
+            title="Classification Metrics",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="X_test and y_test are required.",
+            recommendation="Provide both X_test and y_test.",
+        )
+
+    if not hasattr(model, "predict") or not callable(model.predict):
+        return CheckResult(
+            check_id="model.classification_metrics",
+            title="Classification Metrics",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="The model must provide a callable predict() method.",
+            recommendation="Provide a trained model implementing predict().",
+        )
+
+    try:
+        predictions = list(model.predict(X_test))
+        actual = list(y_test)
+
+        valid_labels = {0, 1}
+
+        for name, values in (
+            ("predictions", predictions),
+            ("targets", actual),
+        ):
+            for index, value in enumerate(values):
+                if (
+                    isinstance(value, (bool, np.bool_))
+                    or not isinstance(
+                        value,
+                        (int, float, np.integer, np.floating),
+                    )
+                    or value not in valid_labels
+                ):
+                    raise ValueError(
+                        f"Invalid binary label in {name} at index "
+                        f"{index}: {value!r}. Labels must be 0 or 1."
+                    )
+
+        if not actual:
+            raise ValueError("y_test must contain at least one sample.")
+
+        if len(actual) != len(predictions):
+            raise ValueError("Predictions and y_test must have the same length.")
+
+        true_positive = sum(
+            prediction == 1 and target == 1
+            for prediction, target in zip(predictions, actual)
+        )
+
+        false_positive = sum(
+            prediction == 1 and target != 1
+            for prediction, target in zip(predictions, actual)
+        )
+
+        false_negative = sum(
+            prediction != 1 and target == 1
+            for prediction, target in zip(predictions, actual)
+        )
+
+        if true_positive + false_positive == 0:
+            raise ValueError(
+                "Precision cannot be calculated because the model "
+                "predicted no positive samples."
+            )
+
+        precision = true_positive / (true_positive + false_positive)
+
+        if true_positive + false_negative == 0:
+            raise ValueError(
+                "Recall cannot be calculated because there are no "
+                "positive samples in y_test."
+            )
+
+        recall = true_positive / (true_positive + false_negative)
+
+        f1 = 2 * precision * recall / (precision + recall)
+
+        return CheckResult(
+            check_id="model.classification_metrics",
+            title="Classification Metrics",
+            category="performance",
+            passed=True,
+            severity="info",
+            score=round(float(f1 * 100.0), 2),
+            message=(
+                f"Classification metrics calculated successfully: "
+                f"precision={precision:.4f}, "
+                f"recall={recall:.4f}, "
+                f"f1={f1:.4f}."
+            ),
+            evidence={
+                "metric": "classification_metrics",
+                "precision": round(float(precision), 4),
+                "recall": round(float(recall), 4),
+                "f1": round(float(f1), 4),
+                "test_samples": len(actual),
+            },
+            recommendation="",
+        )
+
+    except Exception as exc:  # noqa: BLE001 - Convert evaluation failures to check results.
+        return CheckResult(
+            check_id="model.classification_metrics",
+            title="Classification Metrics",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message=f"Classification metric calculation failed: {exc}",
+            recommendation=("Verify that predictions and test targets are valid."),
+        )

ml_reliability/checks/distribution_shift.py

@@ -0,0 +1,172 @@
+from typing import Any
+
+import pandas as pd
+
+from ml_reliability.core.result import CheckResult
+
+
+def check_distribution_shift(
+    X_train: Any,
+    X_test: Any,
+) -> CheckResult:
+    """
+    Compare numeric feature distributions between training and test data.
+    """
+
+    if X_train is None or X_test is None:
+        return CheckResult(
+            check_id="data.distribution_shift",
+            title="Train/Test Distribution Shift",
+            category="distribution",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Both training and test data are required.",
+            recommendation="Provide X_train and X_test before running distribution shift checks.",
+        )
+
+    if not isinstance(X_train, pd.DataFrame) or not isinstance(X_test, pd.DataFrame):
+        return CheckResult(
+            check_id="data.distribution_shift",
+            title="Train/Test Distribution Shift",
+            category="distribution",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Training and test data must both be pandas DataFrames.",
+            recommendation="Provide X_train and X_test as pandas DataFrames.",
+        )
+
+    if X_train.empty or X_test.empty:
+        return CheckResult(
+            check_id="data.distribution_shift",
+            title="Train/Test Distribution Shift",
+            category="distribution",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Training and test data must both contain at least one row.",
+            evidence={
+                "train_rows": len(X_train),
+                "test_rows": len(X_test),
+            },
+            recommendation="Provide non-empty training and test datasets.",
+        )
+
+    shared_columns = [column for column in X_train.columns if column in X_test.columns]
+
+    numeric_columns = [
+        column
+        for column in shared_columns
+        if pd.api.types.is_numeric_dtype(X_train[column])
+        and pd.api.types.is_numeric_dtype(X_test[column])
+    ]
+
+    if not numeric_columns:
+        return CheckResult(
+            check_id="data.distribution_shift",
+            title="Train/Test Distribution Shift",
+            category="distribution",
+            passed=True,
+            severity="info",
+            score=100.0,
+            message="No shared numeric features were available for distribution comparison.",
+            evidence={
+                "shared_columns": [str(column) for column in shared_columns],
+                "numeric_features_compared": 0,
+                "shifted_features": [],
+            },
+            recommendation="Add numeric features or implement categorical distribution checks for categorical data.",
+        )
+
+    feature_statistics = {}
+    moderate_shift_features = []
+    significant_shift_features = []
+
+    for column in numeric_columns:
+        train_series = pd.to_numeric(X_train[column], errors="coerce").dropna()
+        test_series = pd.to_numeric(X_test[column], errors="coerce").dropna()
+
+        if train_series.empty or test_series.empty:
+            continue
+
+        train_mean = float(train_series.mean())
+        test_mean = float(test_series.mean())
+
+        absolute_difference = abs(test_mean - train_mean)
+
+        if train_mean != 0:
+            relative_shift = (absolute_difference / abs(train_mean)) * 100
+        else:
+            relative_shift = 0.0 if test_mean == 0 else float("inf")
+
+        if relative_shift < 10:
+            status = "stable"
+        elif relative_shift <= 20:
+            status = "moderate"
+            moderate_shift_features.append(str(column))
+        else:
+            status = "significant"
+            significant_shift_features.append(str(column))
+
+        feature_statistics[str(column)] = {
+            "train_mean": round(train_mean, 4),
+            "test_mean": round(test_mean, 4),
+            "absolute_difference": round(absolute_difference, 4),
+            "relative_shift_percentage": (
+                round(relative_shift, 2) if relative_shift != float("inf") else None
+            ),
+            "status": status,
+        }
+
+    if significant_shift_features:
+        severity = "warning"
+        score = 50.0
+        passed = True
+        message = (
+            f"Detected significant distribution shift in "
+            f"{len(significant_shift_features)} feature(s)."
+        )
+    elif moderate_shift_features:
+        severity = "warning"
+        score = 80.0
+        passed = True
+        message = (
+            f"Detected moderate distribution shift in "
+            f"{len(moderate_shift_features)} feature(s)."
+        )
+    else:
+        severity = "info"
+        score = 100.0
+        passed = True
+        message = "No significant numeric distribution shift was detected."
+
+    return CheckResult(
+        check_id="data.distribution_shift",
+        title="Train/Test Distribution Shift",
+        category="distribution",
+        passed=passed,
+        severity=severity,
+        score=score,
+        message=message,
+        evidence={
+            "train_rows": len(X_train),
+            "test_rows": len(X_test),
+            "shared_columns": [str(column) for column in shared_columns],
+            "numeric_features_compared": len(numeric_columns),
+            "moderate_shift_features": moderate_shift_features,
+            "significant_shift_features": significant_shift_features,
+            "feature_statistics": feature_statistics,
+        },
+        recommendation=(
+            "Investigate significant train/test distribution differences "
+            "before relying on model evaluation results."
+            if significant_shift_features
+            else (
+                "Monitor moderate distribution differences and verify that "
+                "they are expected."
+                if moderate_shift_features
+                else ""
+            )
+        ),
+    )