ml-reliability-toolkit 0.1.0__py3-none-any.whl

ml_reliability/checks/stress/noise.py

@@ -0,0 +1,161 @@
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from ml_reliability.checks.model_performance import check_model_performance
+
+from .base import (
+    StressTestOutcome,
+    calculate_degradation,
+    validate_rate,
+)
+
+
+def inject_feature_noise(
+    X: pd.DataFrame,
+    noise_level: float,
+    random_state: int = 42,
+) -> pd.DataFrame:
+    """Return a copy of X with controlled Gaussian noise added to numeric features."""
+
+    validate_rate(noise_level)
+
+    if (
+        isinstance(random_state, (bool, np.bool_))
+        or not isinstance(random_state, (int, np.integer))
+        or random_state < 0
+    ):
+        raise ValueError("random_state must be a non-negative integer")
+
+    stressed_X = X.copy()
+
+    if stressed_X.columns.has_duplicates:
+        raise ValueError("duplicate column names are ambiguous for feature noise")
+
+    numeric_columns = stressed_X.select_dtypes(include=np.number).columns
+
+    if len(numeric_columns) > 0:
+        numeric_values = stressed_X[numeric_columns].to_numpy(dtype=float)
+
+        invalid_values = np.isinf(numeric_values)
+
+        if invalid_values.any():
+            raise ValueError("feature noise produced non-finite values")
+
+    if stressed_X.empty or noise_level == 0.0:
+        return stressed_X
+
+    numeric_columns = stressed_X.select_dtypes(include=np.number).columns
+
+    if len(numeric_columns) == 0:
+        return stressed_X
+
+    rng = np.random.default_rng(random_state)
+
+    for column in numeric_columns:
+        if stressed_X[column].isna().all():
+            continue
+
+        with np.errstate(over="ignore", invalid="ignore"):
+            feature_std = stressed_X[column].std()
+
+        if not np.isfinite(feature_std):
+            raise ValueError("feature noise produced non-finite values")
+
+        noise_scale = feature_std * noise_level
+
+        if not np.isfinite(noise_scale):
+            raise ValueError("feature noise produced non-finite values")
+
+        noise = rng.normal(
+            loc=0.0,
+            scale=noise_scale,
+            size=len(stressed_X),
+        )
+
+        stressed_X[column] = stressed_X[column] + noise
+
+        stressed_values = stressed_X[column].to_numpy(dtype=float)
+
+        if np.isinf(stressed_values).any():
+            raise ValueError("feature noise produced non-finite values")
+
+    return stressed_X
+
+
+def evaluate_noise_stress(
+    model: Any,
+    X_test: pd.DataFrame,
+    y_test: Any,
+    task: str,
+    noise_level: float,
+    random_state: int = 42,
+) -> StressTestOutcome:
+    """Evaluate model performance after adding noise to numeric features."""
+
+    stressed_X = inject_feature_noise(
+        X_test,
+        noise_level=noise_level,
+        random_state=random_state,
+    )
+
+    baseline_result = check_model_performance(
+        model=model,
+        X_test=X_test,
+        y_test=y_test,
+        task=task,
+    )
+
+    if not baseline_result.passed:
+        raise ValueError(
+            f"Baseline model performance could not be calculated: "
+            f"{baseline_result.message}"
+        )
+
+    stressed_result = check_model_performance(
+        model=model,
+        X_test=stressed_X,
+        y_test=y_test,
+        task=task,
+    )
+
+    baseline_metric = float(baseline_result.evidence["metric_value"])
+
+    if not stressed_result.passed:
+        return StressTestOutcome(
+            stress_type="feature_noise",
+            baseline_metric=baseline_metric,
+            stressed_metric=float("nan"),
+            degradation=float("inf"),
+            passed=False,
+            evidence={
+                "noise_level": noise_level,
+                "random_state": random_state,
+                "baseline_metric": baseline_metric,
+                "stress_prediction_failed": True,
+                "error": stressed_result.message,
+            },
+        )
+
+    stressed_metric = float(stressed_result.evidence["metric_value"])
+
+    degradation = calculate_degradation(
+        baseline_metric,
+        stressed_metric,
+    )
+
+    return StressTestOutcome(
+        stress_type="feature_noise",
+        baseline_metric=baseline_metric,
+        stressed_metric=stressed_metric,
+        degradation=degradation,
+        passed=True,
+        evidence={
+            "noise_level": noise_level,
+            "random_state": random_state,
+            "baseline_metric": baseline_metric,
+            "stressed_metric": stressed_metric,
+            "degradation": degradation,
+        },
+    )

ml_reliability/checks/target_distribution.py

@@ -0,0 +1,189 @@
+from typing import Any
+
+import pandas as pd
+
+from ml_reliability.core.result import CheckResult
+
+
+def _infer_task_type(y: Any) -> str:
+    """Infer whether the target is classification or regression."""
+
+    series = pd.Series(y)
+
+    if not pd.api.types.is_numeric_dtype(series):
+        return "classification"
+
+    unique_values = series.nunique(dropna=True)
+
+    if unique_values <= 10:
+        return "classification"
+
+    return "regression"
+
+
+def check_target_distribution(
+    y: Any,
+    task_type: str | None = None,
+) -> CheckResult:
+    """
+    Check the distribution of a machine-learning target.
+
+    Supports classification and regression targets.
+    """
+
+    if y is None:
+        return CheckResult(
+            check_id="target.distribution",
+            title="Target Distribution",
+            category="target_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Target data is missing.",
+            recommendation="Provide y_train before running target distribution checks.",
+        )
+
+    try:
+        series = pd.Series(y)
+    except Exception:  # noqa: BLE001 - Handle invalid target conversion.
+        return CheckResult(
+            check_id="target.distribution",
+            title="Target Distribution",
+            category="target_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Target could not be converted into a valid series.",
+            recommendation="Provide a valid one-dimensional target.",
+        )
+
+    if series.empty:
+        return CheckResult(
+            check_id="target.distribution",
+            title="Target Distribution",
+            category="target_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Target contains zero samples.",
+            evidence={"total_samples": 0},
+            recommendation="Provide a target containing at least one sample.",
+        )
+
+    if series.isna().any():
+        missing_count = int(series.isna().sum())
+
+        return CheckResult(
+            check_id="target.distribution",
+            title="Target Distribution",
+            category="target_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message=f"Target contains {missing_count} missing values.",
+            evidence={
+                "total_samples": len(series),
+                "missing_values": missing_count,
+            },
+            recommendation="Remove or appropriately handle missing target values.",
+        )
+
+    if task_type is None:
+        task_type = _infer_task_type(series)
+
+    if task_type not in {"classification", "regression"}:
+        return CheckResult(
+            check_id="target.distribution",
+            title="Target Distribution",
+            category="target_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message=f"Unsupported task type: {task_type}.",
+            recommendation="Use 'classification' or 'regression'.",
+        )
+
+    if task_type == "classification":
+        counts = series.value_counts()
+        percentages = (counts / len(series)) * 100
+
+        class_counts = {str(label): int(count) for label, count in counts.items()}
+
+        class_percentages = {
+            str(label): round(float(percentage), 2)
+            for label, percentage in percentages.items()
+        }
+
+        minority_percentage = float(percentages.min())
+
+        if minority_percentage >= 20:
+            severity = "info"
+            score = 100.0
+            passed = True
+        elif minority_percentage >= 5:
+            severity = "warning"
+            score = 70.0
+            passed = True
+        else:
+            severity = "critical"
+            score = 30.0
+            passed = False
+
+        return CheckResult(
+            check_id="target.distribution",
+            title="Target Distribution",
+            category="target_quality",
+            passed=passed,
+            severity=severity,
+            score=score,
+            message=(
+                f"Classification target contains {len(counts)} classes. "
+                f"Minority class represents {minority_percentage:.2f}% of samples."
+            ),
+            evidence={
+                "task_type": "classification",
+                "total_samples": len(series),
+                "class_counts": class_counts,
+                "class_percentages": class_percentages,
+                "minority_percentage": round(minority_percentage, 2),
+            },
+            recommendation=(
+                "Consider class-balancing techniques or appropriate evaluation "
+                "metrics if the target distribution is highly imbalanced."
+                if not passed
+                else ""
+            ),
+        )
+
+    # Regression
+    numeric_series = pd.to_numeric(series, errors="coerce")
+
+    if numeric_series.isna().any():
+        return CheckResult(
+            check_id="target.distribution",
+            title="Target Distribution",
+            category="target_quality",
+            passed=False,
+            severity="critical",
+            score=0.0,
+            message="Regression target contains non-numeric values.",
+            recommendation="Provide a numeric target for regression.",
+        )
+
+    return CheckResult(
+        check_id="target.distribution",
+        title="Target Distribution",
+        category="target_quality",
+        passed=True,
+        severity="info",
+        score=100.0,
+        message="Regression target distribution was successfully analyzed.",
+        evidence={
+            "task_type": "regression",
+            "total_samples": len(numeric_series),
+            "mean": float(numeric_series.mean()),
+            "std": float(numeric_series.std()),
+            "min": float(numeric_series.min()),
+            "max": float(numeric_series.max()),
+        },
+    )

ml_reliability/core/result.py

@@ -0,0 +1,15 @@
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(slots=True)
+class CheckResult:
+    check_id: str
+    title: str
+    category: str
+    passed: bool
+    severity: str
+    score: float | None
+    message: str
+    evidence: dict[str, Any] = field(default_factory=dict)
+    recommendation: str = ""