ml-reliability-toolkit 0.1.0__py3-none-any.whl

ml_reliability/checks/stress/categorical.py

@@ -0,0 +1,181 @@
+from collections.abc import Sequence
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from ml_reliability.checks.model_performance import check_model_performance
+
+from .base import (
+    StressTestOutcome,
+    calculate_degradation,
+    validate_rate,
+)
+
+
+def inject_categorical_corruption(
+    X: pd.DataFrame,
+    columns: Sequence[str],
+    rate: float,
+    random_state: int = 42,
+) -> pd.DataFrame:
+    """Return a copy of X with categorical values replaced by other valid values."""
+
+    if isinstance(columns, (str, bytes)) or not isinstance(columns, Sequence):
+        raise TypeError("columns must be a sequence of column names")
+
+    if any(not isinstance(column, str) for column in columns):
+        raise ValueError("columns must contain only column names")
+
+    validate_rate(rate)
+    if (
+        isinstance(random_state, (bool, np.bool_))
+        or not isinstance(random_state, (int, np.integer))
+        or random_state < 0
+    ):
+        raise ValueError("random_state must be a non-negative integer")
+
+    if len(columns) != len(set(columns)):
+        raise ValueError("duplicate requested column names are not allowed")
+
+    stressed_X = X.copy()
+
+    if stressed_X.columns.has_duplicates:
+        duplicated_columns = (
+            stressed_X.columns[stressed_X.columns.duplicated()].unique().tolist()
+        )
+        requested_duplicates = [
+            column for column in columns if column in duplicated_columns
+        ]
+        if requested_duplicates:
+            raise ValueError(
+                f"duplicate column names are ambiguous: {requested_duplicates}"
+            )
+
+    missing_columns = [column for column in columns if column not in stressed_X.columns]
+
+    if missing_columns:
+        raise ValueError(f"Columns not found in X: {missing_columns}")
+
+    if not columns or stressed_X.empty or rate == 0.0:
+        return stressed_X
+
+    rng = np.random.default_rng(random_state)
+
+    for column in columns:
+        unique_values = stressed_X[column].dropna().unique()
+
+        if len(unique_values) < 2:
+            continue
+
+        valid_positions = np.flatnonzero(stressed_X[column].notna().to_numpy())
+
+        values_to_corrupt = int(len(valid_positions) * rate)
+
+        if rate > 0.0 and len(valid_positions) > 0:
+            values_to_corrupt = max(1, values_to_corrupt)
+
+        if values_to_corrupt == 0:
+            continue
+
+        selected_positions = rng.choice(
+            valid_positions,
+            size=values_to_corrupt,
+            replace=False,
+        )
+
+        for position in selected_positions:
+            current_value = stressed_X[column].iloc[position]
+
+            alternatives = [value for value in unique_values if value != current_value]
+
+            if not alternatives:
+                continue
+
+            alternative_index = rng.integers(len(alternatives))
+            stressed_X.iat[position, stressed_X.columns.get_loc(column)] = alternatives[
+                alternative_index
+            ]
+
+    return stressed_X
+
+
+def evaluate_categorical_corruption_stress(
+    model: Any,
+    X_test: pd.DataFrame,
+    y_test: Any,
+    task: str,
+    columns: Sequence[str],
+    rate: float,
+    random_state: int = 42,
+) -> StressTestOutcome:
+    """Evaluate model performance after categorical value corruption."""
+
+    stressed_X = inject_categorical_corruption(
+        X_test,
+        columns=columns,
+        rate=rate,
+        random_state=random_state,
+    )
+
+    baseline_result = check_model_performance(
+        model=model,
+        X_test=X_test,
+        y_test=y_test,
+        task=task,
+    )
+
+    if not baseline_result.passed:
+        raise ValueError(
+            f"Baseline model performance could not be calculated: "
+            f"{baseline_result.message}"
+        )
+
+    stressed_result = check_model_performance(
+        model=model,
+        X_test=stressed_X,
+        y_test=y_test,
+        task=task,
+    )
+
+    baseline_metric = float(baseline_result.evidence["metric_value"])
+
+    if not stressed_result.passed:
+        return StressTestOutcome(
+            stress_type="categorical_corruption",
+            baseline_metric=baseline_metric,
+            stressed_metric=float("nan"),
+            degradation=float("inf"),
+            passed=False,
+            evidence={
+                "columns": list(columns),
+                "rate": rate,
+                "random_state": random_state,
+                "baseline_metric": baseline_metric,
+                "stress_prediction_failed": True,
+                "error": stressed_result.message,
+            },
+        )
+
+    stressed_metric = float(stressed_result.evidence["metric_value"])
+
+    degradation = calculate_degradation(
+        baseline_metric,
+        stressed_metric,
+    )
+
+    return StressTestOutcome(
+        stress_type="categorical_corruption",
+        baseline_metric=baseline_metric,
+        stressed_metric=stressed_metric,
+        degradation=degradation,
+        passed=True,
+        evidence={
+            "columns": list(columns),
+            "rate": rate,
+            "random_state": random_state,
+            "baseline_metric": baseline_metric,
+            "stressed_metric": stressed_metric,
+            "degradation": degradation,
+        },
+    )

ml_reliability/checks/stress/distribution.py

@@ -0,0 +1,154 @@
+import math
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from ml_reliability.checks.model_performance import check_model_performance
+
+from .base import StressTestOutcome, calculate_degradation
+
+
+def perturb_feature_distribution(
+    X: pd.DataFrame,
+    shift: float = 0.0,
+    scale: float = 1.0,
+    random_state: int = 42,
+) -> pd.DataFrame:
+    """Return a copy of X with controlled numeric distribution perturbation."""
+
+    if (
+        isinstance(random_state, (bool, np.bool_))
+        or not isinstance(random_state, (int, np.integer))
+        or random_state < 0
+    ):
+        raise ValueError("random_state must be a non-negative integer")
+
+    for name, value in (("shift", shift), ("scale", scale)):
+        if isinstance(value, (bool, np.bool_)) or not isinstance(
+            value, (int, float, np.integer, np.floating)
+        ):
+            raise TypeError(f"{name} must be a numeric value")
+
+        try:
+            is_finite = math.isfinite(value)
+        except OverflowError:
+            is_finite = False
+
+        if not is_finite:
+            if name == "scale":
+                raise ValueError("scale must be a finite number greater than 0")
+            raise ValueError("shift must be a finite number")
+
+    if scale <= 0:
+        raise ValueError("scale must be a finite number greater than 0")
+    stressed_X = X.copy()
+
+    if stressed_X.empty:
+        return stressed_X
+
+    numeric_columns = stressed_X.select_dtypes(include=np.number).columns
+
+    if len(numeric_columns) == 0:
+        return stressed_X
+
+    with np.errstate(over="ignore", invalid="ignore"):
+        for column in numeric_columns:
+            stressed_X[column] = (stressed_X[column] * scale) + shift
+
+    original_values = X[numeric_columns].to_numpy(dtype=float)
+    stressed_values = stressed_X[numeric_columns].to_numpy(dtype=float)
+
+    original_missing = np.isnan(original_values)
+    stressed_missing = np.isnan(stressed_values)
+
+    new_non_finite = ~np.isfinite(stressed_values) & ~original_missing
+
+    if new_non_finite.any():
+        raise ValueError("distribution perturbation produced non-finite values")
+
+    if np.any(stressed_missing & ~original_missing):
+        raise ValueError("distribution perturbation produced non-finite values")
+
+    return stressed_X
+
+
+def evaluate_distribution_perturbation_stress(
+    model: Any,
+    X_test: pd.DataFrame,
+    y_test: Any,
+    task: str,
+    shift: float = 0.0,
+    scale: float = 1.0,
+    random_state: int = 42,
+) -> StressTestOutcome:
+    """Evaluate model performance after distribution perturbation."""
+
+    stressed_X = perturb_feature_distribution(
+        X_test,
+        shift=shift,
+        scale=scale,
+        random_state=random_state,
+    )
+
+    baseline_result = check_model_performance(
+        model=model,
+        X_test=X_test,
+        y_test=y_test,
+        task=task,
+    )
+
+    if not baseline_result.passed:
+        raise ValueError(
+            "Baseline model performance could not be calculated: "
+            f"{baseline_result.message}"
+        )
+
+    stressed_result = check_model_performance(
+        model=model,
+        X_test=stressed_X,
+        y_test=y_test,
+        task=task,
+    )
+
+    baseline_metric = float(baseline_result.evidence["metric_value"])
+
+    if not stressed_result.passed:
+        return StressTestOutcome(
+            stress_type="distribution_perturbation",
+            baseline_metric=baseline_metric,
+            stressed_metric=float("nan"),
+            degradation=float("inf"),
+            passed=False,
+            evidence={
+                "shift": shift,
+                "scale": scale,
+                "random_state": random_state,
+                "baseline_metric": baseline_metric,
+                "stress_prediction_failed": True,
+                "error": stressed_result.message,
+            },
+        )
+
+    stressed_metric = float(stressed_result.evidence["metric_value"])
+
+    degradation = calculate_degradation(
+        baseline_metric,
+        stressed_metric,
+    )
+
+    return StressTestOutcome(
+        stress_type="distribution_perturbation",
+        baseline_metric=baseline_metric,
+        stressed_metric=stressed_metric,
+        degradation=degradation,
+        passed=True,
+        evidence={
+            "shift": shift,
+            "scale": scale,
+            "random_state": random_state,
+            "baseline_metric": baseline_metric,
+            "stressed_metric": stressed_metric,
+            "degradation": degradation,
+        },
+    )

ml_reliability/checks/stress/feature_dropout.py

@@ -0,0 +1,122 @@
+from collections.abc import Sequence
+from typing import Any
+
+import pandas as pd
+
+from ml_reliability.checks.model_performance import check_model_performance
+
+from .base import StressTestOutcome, calculate_degradation
+
+
+def apply_feature_dropout(X: pd.DataFrame, features: Sequence[str]) -> pd.DataFrame:
+    """Return a copy of X with selected features masked as missing."""
+    if isinstance(features, (str, bytes)) or not isinstance(features, Sequence):
+        raise TypeError("features must be a sequence of column names")
+
+    if any(not isinstance(feature, str) for feature in features):
+        raise ValueError("features must contain only column names")
+
+    stressed_X = X.copy()
+
+    if stressed_X.columns.has_duplicates:
+        duplicated_columns = (
+            stressed_X.columns[stressed_X.columns.duplicated()].unique().tolist()
+        )
+
+        requested_duplicates = [
+            feature for feature in features if feature in duplicated_columns
+        ]
+
+        if requested_duplicates:
+            raise ValueError(
+                f"duplicate column names are ambiguous: {requested_duplicates}"
+            )
+
+    if not features:
+        return stressed_X
+
+    missing_features = [
+        feature for feature in features if feature not in stressed_X.columns
+    ]
+    if missing_features:
+        raise ValueError(f"Features not found in X: {missing_features}")
+
+    for feature in features:
+        stressed_X[feature] = stressed_X[feature].mask(
+            pd.Series(True, index=stressed_X.index)
+        )
+
+    return stressed_X
+
+
+def evaluate_feature_dropout_stress(
+    model: Any,
+    X_test: pd.DataFrame,
+    y_test: Any,
+    task: str,
+    features: Sequence[str],
+) -> StressTestOutcome:
+    """Evaluate model performance after dropping selected features."""
+
+    stressed_X = apply_feature_dropout(
+        X_test,
+        features=features,
+    )
+
+    baseline_result = check_model_performance(
+        model=model,
+        X_test=X_test,
+        y_test=y_test,
+        task=task,
+    )
+
+    if not baseline_result.passed:
+        raise ValueError(
+            f"Baseline model performance could not be calculated: "
+            f"{baseline_result.message}"
+        )
+
+    stressed_result = check_model_performance(
+        model=model,
+        X_test=stressed_X,
+        y_test=y_test,
+        task=task,
+    )
+
+    baseline_metric = float(baseline_result.evidence["metric_value"])
+
+    if not stressed_result.passed:
+        return StressTestOutcome(
+            stress_type="feature_dropout",
+            baseline_metric=baseline_metric,
+            stressed_metric=float("nan"),
+            degradation=float("inf"),
+            passed=False,
+            evidence={
+                "features": list(features),
+                "baseline_metric": baseline_metric,
+                "stress_prediction_failed": True,
+                "error": stressed_result.message,
+            },
+        )
+
+    stressed_metric = float(stressed_result.evidence["metric_value"])
+
+    degradation = calculate_degradation(
+        baseline_metric,
+        stressed_metric,
+    )
+
+    return StressTestOutcome(
+        stress_type="feature_dropout",
+        baseline_metric=baseline_metric,
+        stressed_metric=stressed_metric,
+        degradation=degradation,
+        passed=True,
+        evidence={
+            "features": list(features),
+            "baseline_metric": baseline_metric,
+            "stressed_metric": stressed_metric,
+            "degradation": degradation,
+        },
+    )

ml_reliability/checks/stress/missing_values.py

@@ -0,0 +1,142 @@
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from ml_reliability.checks.model_performance import check_model_performance
+
+from .base import (
+    StressTestOutcome,
+    calculate_degradation,
+    validate_rate,
+)
+
+
+def inject_missing_values(
+    X: pd.DataFrame,
+    rate: float,
+    random_state: int = 42,
+) -> pd.DataFrame:
+    """Return a copy of X with a controlled proportion of eligible values replaced by NaN."""
+
+    if (
+        isinstance(random_state, bool)
+        or not isinstance(random_state, (int, np.integer))
+        or random_state < 0
+    ):
+        raise ValueError("random_state must be a non-negative integer")
+
+    validate_rate(rate)
+
+    stressed_X = X.copy()
+
+    if X.empty or rate == 0.0:
+        return stressed_X
+
+    # Sample only existing non-missing cells so the requested corruption
+    # count represents newly introduced missing values.
+    valid_mask = ~stressed_X.isna().to_numpy()
+    valid_positions = np.flatnonzero(valid_mask.ravel())
+
+    values_to_corrupt = min(
+        len(valid_positions),
+        max(1, int(len(valid_positions) * rate)),
+    )
+
+    if values_to_corrupt == 0:
+        return stressed_X
+
+    random_generator = np.random.default_rng(random_state)
+
+    selected_positions = random_generator.choice(
+        valid_positions,
+        size=values_to_corrupt,
+        replace=False,
+    )
+
+    rows, columns = np.unravel_index(
+        selected_positions,
+        stressed_X.shape,
+    )
+
+    for row, column in zip(rows, columns):
+        stressed_X.iat[row, column] = float("nan")
+
+    return stressed_X
+
+
+def evaluate_missing_value_stress(
+    model: Any,
+    X_test: pd.DataFrame,
+    y_test: Any,
+    task: str,
+    rate: float,
+    random_state: int = 42,
+) -> StressTestOutcome:
+    """Evaluate model performance after injecting missing values."""
+
+    stressed_X = inject_missing_values(
+        X_test,
+        rate=rate,
+        random_state=random_state,
+    )
+
+    baseline_result = check_model_performance(
+        model=model,
+        X_test=X_test,
+        y_test=y_test,
+        task=task,
+    )
+
+    if not baseline_result.passed:
+        raise ValueError(
+            f"Baseline model performance could not be calculated: "
+            f"{baseline_result.message}"
+        )
+
+    stressed_result = check_model_performance(
+        model=model,
+        X_test=stressed_X,
+        y_test=y_test,
+        task=task,
+    )
+
+    baseline_metric = float(baseline_result.evidence["metric_value"])
+
+    if not stressed_result.passed:
+        return StressTestOutcome(
+            stress_type="missing_values",
+            baseline_metric=baseline_metric,
+            stressed_metric=float("nan"),
+            degradation=float("inf"),
+            passed=False,
+            evidence={
+                "rate": rate,
+                "random_state": random_state,
+                "baseline_metric": baseline_metric,
+                "stress_prediction_failed": True,
+                "error": stressed_result.message,
+            },
+        )
+
+    stressed_metric = float(stressed_result.evidence["metric_value"])
+
+    degradation = calculate_degradation(
+        baseline_metric,
+        stressed_metric,
+    )
+
+    return StressTestOutcome(
+        stress_type="missing_values",
+        baseline_metric=baseline_metric,
+        stressed_metric=stressed_metric,
+        degradation=degradation,
+        passed=True,
+        evidence={
+            "rate": rate,
+            "random_state": random_state,
+            "baseline_metric": baseline_metric,
+            "stressed_metric": stressed_metric,
+            "degradation": degradation,
+        },
+    )