ml-reliability-toolkit 0.1.0__py3-none-any.whl

ml_reliability/checks/duplicates.py

@@ -0,0 +1,104 @@
+from typing import Any
+
+from ml_reliability.core.result import CheckResult
+
+
+def check_duplicates(X: Any) -> CheckResult:
+    """
+    Check a dataset for duplicate rows.
+
+    Duplicate rows are counted after the first occurrence.
+    """
+
+    if X is None:
+        return CheckResult(
+            check_id="data.duplicates",
+            title="Duplicate Rows",
+            category="data_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Input dataset is missing.",
+            recommendation="Provide a valid dataset before running reliability checks.",
+        )
+
+    if not hasattr(X, "duplicated") or not hasattr(X, "shape"):
+        return CheckResult(
+            check_id="data.duplicates",
+            title="Duplicate Rows",
+            category="data_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Input must be a pandas DataFrame.",
+            recommendation="Provide the dataset as a pandas DataFrame.",
+        )
+
+    total_rows = int(X.shape[0])
+
+    if total_rows == 0:
+        return CheckResult(
+            check_id="data.duplicates",
+            title="Duplicate Rows",
+            category="data_quality",
+            passed=True,
+            severity="info",
+            score=100.0,
+            message="Dataset is empty; no duplicate rows were detected.",
+            evidence={
+                "total_rows": 0,
+                "duplicate_rows": 0,
+                "duplicate_percentage": 0.0,
+            },
+        )
+
+    duplicate_rows = int(X.duplicated().sum())
+    duplicate_percentage = (duplicate_rows / total_rows) * 100
+
+    if duplicate_rows == 0:
+        return CheckResult(
+            check_id="data.duplicates",
+            title="Duplicate Rows",
+            category="data_quality",
+            passed=True,
+            severity="info",
+            score=100.0,
+            message="No duplicate rows were detected.",
+            evidence={
+                "total_rows": total_rows,
+                "duplicate_rows": 0,
+                "duplicate_percentage": 0.0,
+            },
+        )
+
+    if duplicate_percentage > 20:
+        severity = "critical"
+        score = 0.0
+    elif duplicate_percentage >= 5:
+        severity = "warning"
+        score = 50.0
+    else:
+        severity = "info"
+        score = 80.0
+
+    return CheckResult(
+        check_id="data.duplicates",
+        title="Duplicate Rows",
+        category="data_quality",
+        passed=False,
+        severity=severity,
+        score=score,
+        message=(
+            f"Detected {duplicate_rows} duplicate rows "
+            f"({duplicate_percentage:.2f}% of all rows)."
+        ),
+        evidence={
+            "total_rows": total_rows,
+            "duplicate_rows": duplicate_rows,
+            "duplicate_percentage": round(duplicate_percentage, 2),
+        },
+        recommendation=(
+            "Investigate duplicate records and remove or handle them "
+            "appropriately before model training."
+        ),
+    )

ml_reliability/checks/generalization_gap.py

@@ -0,0 +1,217 @@
+import math
+from numbers import Real
+from typing import Any
+
+from ml_reliability.core.result import CheckResult
+
+
+def check_generalization_gap(
+    model: Any,
+    X_train: Any,
+    y_train: Any,
+    X_test: Any,
+    y_test: Any,
+    task: str,
+    threshold: float = 0.10,
+) -> CheckResult:
+    """
+    Measure the performance gap between training and test data.
+
+    Supported tasks:
+    - classification: accuracy
+    - regression: R²
+    """
+
+    if model is None:
+        return CheckResult(
+            check_id="model.generalization_gap",
+            title="Generalization Gap",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="A trained model is required.",
+            recommendation="Provide a trained model with a predict() method.",
+        )
+
+    if (
+        isinstance(threshold, bool)
+        or not isinstance(threshold, Real)
+        or not math.isfinite(threshold)
+        or threshold < 0
+    ):
+        return CheckResult(
+            check_id="model.generalization_gap",
+            title="Generalization Gap",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message=(
+                "Invalid threshold: threshold must be a finite, non-negative number."
+            ),
+            evidence={"threshold": repr(threshold)},
+            recommendation=(
+                "Provide a finite, non-negative threshold for the generalization gap."
+            ),
+        )
+
+    if X_train is None or y_train is None or X_test is None or y_test is None:
+        return CheckResult(
+            check_id="model.generalization_gap",
+            title="Generalization Gap",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Training and test data are required.",
+            recommendation=("Provide X_train, y_train, X_test, and y_test."),
+        )
+
+    if task not in {"classification", "regression"}:
+        return CheckResult(
+            check_id="model.generalization_gap",
+            title="Generalization Gap",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message=f"Unsupported task type: {task}.",
+            recommendation=("Use task='classification' or task='regression'."),
+        )
+
+    if not hasattr(model, "predict") or not callable(model.predict):
+        return CheckResult(
+            check_id="model.generalization_gap",
+            title="Generalization Gap",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="The model must provide a callable predict() method.",
+            recommendation="Provide a trained model implementing predict().",
+        )
+
+    try:
+        train_predictions = model.predict(X_train)
+        test_predictions = model.predict(X_test)
+
+        train_actual = list(y_train)
+        test_actual = list(y_test)
+
+        train_predicted = list(train_predictions)
+        test_predicted = list(test_predictions)
+
+        if not train_actual or not test_actual:
+            raise ValueError(
+                "Training and test targets must contain at least one sample."
+            )
+
+        if len(train_actual) != len(train_predicted):
+            raise ValueError(
+                "Training predictions and y_train must have the same length."
+            )
+
+        if len(test_actual) != len(test_predicted):
+            raise ValueError("Test predictions and y_test must have the same length.")
+
+        if task == "classification":
+            train_correct = sum(
+                prediction == target
+                for prediction, target in zip(
+                    train_predicted,
+                    train_actual,
+                )
+            )
+
+            test_correct = sum(
+                prediction == target
+                for prediction, target in zip(
+                    test_predicted,
+                    test_actual,
+                )
+            )
+
+            train_metric = train_correct / len(train_actual)
+            test_metric = test_correct / len(test_actual)
+
+            metric_name = "accuracy"
+
+        else:
+            train_mean = sum(train_actual) / len(train_actual)
+            test_mean = sum(test_actual) / len(test_actual)
+
+            train_total = sum((target - train_mean) ** 2 for target in train_actual)
+
+            test_total = sum((target - test_mean) ** 2 for target in test_actual)
+
+            if train_total == 0 or test_total == 0:
+                raise ValueError(
+                    "R² cannot be calculated when target variance is zero."
+                )
+
+            train_residual = sum(
+                (target - prediction) ** 2
+                for target, prediction in zip(
+                    train_actual,
+                    train_predicted,
+                )
+            )
+
+            test_residual = sum(
+                (target - prediction) ** 2
+                for target, prediction in zip(
+                    test_actual,
+                    test_predicted,
+                )
+            )
+
+            train_metric = 1 - train_residual / train_total
+            test_metric = 1 - test_residual / test_total
+
+            metric_name = "r2"
+
+        generalization_gap = train_metric - test_metric
+
+        passed = generalization_gap <= threshold
+
+        return CheckResult(
+            check_id="model.generalization_gap",
+            title="Generalization Gap",
+            category="performance",
+            passed=passed,
+            severity="info" if passed else "warning",
+            score=100.0 if passed else 0.0,
+            message=(
+                f"Generalization gap calculated successfully: {generalization_gap:.4f}."
+            ),
+            evidence={
+                "task_type": task,
+                "metric": metric_name,
+                "train_metric": round(float(train_metric), 4),
+                "test_metric": round(float(test_metric), 4),
+                "generalization_gap": round(
+                    float(generalization_gap),
+                    4,
+                ),
+                "threshold": threshold,
+            },
+            recommendation=(
+                ""
+                if passed
+                else "Investigate possible overfitting or "
+                "distribution differences between training and test data."
+            ),
+        )
+
+    except Exception as exc:  # noqa: BLE001 - Convert evaluation failures to check results.
+        return CheckResult(
+            check_id="model.generalization_gap",
+            title="Generalization Gap",
+            category="performance",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message=f"Generalization gap calculation failed: {exc}",
+            recommendation=("Verify that the model and training/test data are valid."),
+        )

ml_reliability/checks/missing_values.py

@@ -0,0 +1,103 @@
+from typing import Any
+
+from ml_reliability.core.result import CheckResult
+
+
+def check_missing_values(X: Any) -> CheckResult:
+    """
+    Check a dataset for missing values.
+
+    Returns a CheckResult containing:
+    - total missing values
+    - affected columns
+    - missing percentage
+    """
+
+    if X is None:
+        return CheckResult(
+            check_id="data.missing_values",
+            title="Missing Values",
+            category="data_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Input dataset is missing.",
+            recommendation="Provide a valid dataset before running reliability checks.",
+        )
+
+    # Pandas DataFrame support
+    if not hasattr(X, "isna") or not hasattr(X, "columns"):
+        return CheckResult(
+            check_id="data.missing_values",
+            title="Missing Values",
+            category="data_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Input must be a pandas DataFrame.",
+            recommendation="Provide the dataset as a pandas DataFrame.",
+        )
+
+    missing_by_column = X.isna().sum()
+    missing_by_column = missing_by_column[missing_by_column > 0]
+
+    total_missing = int(missing_by_column.sum())
+    total_cells = int(X.shape[0] * X.shape[1])
+
+    if total_cells == 0:
+        missing_percentage = 0.0
+    else:
+        missing_percentage = (total_missing / total_cells) * 100
+
+    if total_missing == 0:
+        return CheckResult(
+            check_id="data.missing_values",
+            title="Missing Values",
+            category="data_quality",
+            passed=True,
+            severity="info",
+            score=100.0,
+            message="No missing values were detected.",
+            evidence={
+                "total_missing": 0,
+                "missing_percentage": 0.0,
+                "affected_columns": {},
+            },
+        )
+
+    affected_columns = {
+        str(column): int(count) for column, count in missing_by_column.items()
+    }
+
+    # Initial severity thresholds
+    if missing_percentage >= 20:
+        severity = "critical"
+        score = 0.0
+    elif missing_percentage >= 5:
+        severity = "warning"
+        score = 50.0
+    else:
+        severity = "info"
+        score = 80.0
+
+    return CheckResult(
+        check_id="data.missing_values",
+        title="Missing Values",
+        category="data_quality",
+        passed=False,
+        severity=severity,
+        score=score,
+        message=(
+            f"Detected {total_missing} missing values "
+            f"({missing_percentage:.2f}% of all cells)."
+        ),
+        evidence={
+            "total_missing": total_missing,
+            "missing_percentage": round(missing_percentage, 2),
+            "affected_columns": affected_columns,
+        },
+        recommendation=(
+            "Investigate affected columns and apply an appropriate "
+            "missing-value handling strategy."
+        ),
+    )

ml_reliability/checks/model_performance.py

@@ -0,0 +1,187 @@
+import math
+from typing import Any
+
+from ml_reliability.core.result import CheckResult
+
+
+def check_model_performance(
+    model: Any,
+    X_test: Any,
+    y_test: Any,
+    task: str,
+) -> CheckResult:
+    """
+    Evaluate basic model performance on test data.
+
+    Supported tasks:
+    - classification: accuracy
+    - regression: R²
+
+    The reliability score is based on the calculated metric:
+    - Classification: accuracy * 100
+    - Regression: R² * 100, bounded between 0 and 100
+
+    A successful calculation does not necessarily mean the model
+    has acceptable performance. Check the score and metric evidence.
+    """
+
+    check_id = "model.performance"
+    title = "Model Performance"
+    category = "performance"
+
+    if model is None:
+        return CheckResult(
+            check_id=check_id,
+            title=title,
+            category=category,
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="A trained model is required.",
+            recommendation="Provide a trained model with a predict() method.",
+        )
+
+    if X_test is None or y_test is None:
+        return CheckResult(
+            check_id=check_id,
+            title=title,
+            category=category,
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="X_test and y_test are required.",
+            recommendation="Provide both X_test and y_test.",
+        )
+
+    if task not in {"classification", "regression"}:
+        return CheckResult(
+            check_id=check_id,
+            title=title,
+            category=category,
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message=f"Unsupported task type: {task}.",
+            recommendation=("Use task='classification' or task='regression'."),
+        )
+
+    if not callable(getattr(model, "predict", None)):
+        return CheckResult(
+            check_id=check_id,
+            title=title,
+            category=category,
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="The model must provide a callable predict() method.",
+            recommendation="Provide a trained model implementing predict().",
+        )
+
+    try:
+        predictions = model.predict(X_test)
+    except Exception as exc:  # noqa: BLE001 - Convert evaluation failures to check results.
+        return CheckResult(
+            check_id=check_id,
+            title=title,
+            category=category,
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message=f"Model prediction failed: {exc}",
+            recommendation=(
+                "Verify that the model is trained and compatible with X_test."
+            ),
+        )
+
+    try:
+        actual = list(y_test)
+        predicted = list(predictions)
+
+        if not actual:
+            raise ValueError("y_test must contain at least one sample.")
+
+        if len(actual) != len(predicted):
+            raise ValueError("Predictions and y_test must have the same length.")
+
+        def contains_non_finite_numeric_value(values: list[Any]) -> bool:
+            for value in values:
+                if isinstance(value, (int, float)):
+                    if not math.isfinite(value):
+                        return True
+                elif hasattr(value, "item"):
+                    # Handle NumPy scalar values without rejecting strings.
+                    scalar = value.item()
+                    if isinstance(scalar, (int, float)) and not math.isfinite(scalar):
+                        return True
+            return False
+
+        if contains_non_finite_numeric_value(
+            predicted
+        ) or contains_non_finite_numeric_value(actual):
+            raise ValueError(
+                "Predictions and targets must not contain non-finite numeric values."
+            )
+
+        if task == "classification":
+            correct = sum(
+                prediction == target for prediction, target in zip(predicted, actual)
+            )
+
+            metric_name = "accuracy"
+            metric_value = correct / len(actual)
+
+        else:
+            mean_actual = sum(actual) / len(actual)
+
+            total_sum_squares = sum((target - mean_actual) ** 2 for target in actual)
+
+            if total_sum_squares == 0:
+                raise ValueError(
+                    "R² cannot be calculated when y_test has zero variance."
+                )
+
+            residual_sum_squares = sum(
+                (target - prediction) ** 2
+                for target, prediction in zip(actual, predicted)
+            )
+
+            metric_name = "r2"
+            metric_value = 1 - residual_sum_squares / total_sum_squares
+
+        score = max(
+            0.0,
+            min(100.0, metric_value * 100.0),
+        )
+
+    except Exception as exc:  # noqa: BLE001 - Convert evaluation failures to check results.
+        return CheckResult(
+            check_id=check_id,
+            title=title,
+            category=category,
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message=f"Performance calculation failed: {exc}",
+            recommendation=("Verify that predictions and test targets are valid."),
+        )
+
+    return CheckResult(
+        check_id=check_id,
+        title=title,
+        category=category,
+        passed=True,
+        severity="info",
+        score=round(float(score), 2),
+        message=(
+            f"Model {metric_name} calculated successfully: "
+            f"{metric_value:.4f}. "
+            f"Reliability score: {score:.2f}/100."
+        ),
+        evidence={
+            "task_type": task,
+            "metric": metric_name,
+            "metric_value": round(float(metric_value), 4),
+            "test_samples": len(actual),
+        },
+        recommendation="",
+    )