validmind 2.6.10__py3-none-any.whl → 2.7.4__py3-none-any.whl

validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.py

@@ -0,0 +1,261 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+from sklearn.metrics import (
+    roc_curve,
+    precision_recall_curve,
+    confusion_matrix,
+)
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+def find_optimal_threshold(y_true, y_prob, method="youden", target_recall=None):
+    """
+    Find the optimal classification threshold using various methods.
+
+    Args:
+        y_true: True binary labels
+        y_prob: Predicted probabilities
+        method: Method to use for finding optimal threshold
+        target_recall: Required if method='target_recall'
+
+    Returns:
+        dict: Dictionary containing threshold and metrics
+    """
+    # Get ROC and PR curve points
+    fpr, tpr, thresholds_roc = roc_curve(y_true, y_prob)
+    precision, recall, thresholds_pr = precision_recall_curve(y_true, y_prob)
+
+    # Find optimal threshold based on method
+    if method == "naive":
+        optimal_threshold = 0.5
+    elif method == "youden":
+        j_scores = tpr - fpr
+        best_idx = np.argmax(j_scores)
+        optimal_threshold = thresholds_roc[best_idx]
+    elif method == "f1":
+        f1_scores = 2 * (precision * recall) / (precision + recall)
+        best_idx = np.argmax(f1_scores)
+        optimal_threshold = (
+            thresholds_pr[best_idx] if best_idx < len(thresholds_pr) else 1.0
+        )
+    elif method == "precision_recall":
+        diff = abs(precision - recall)
+        best_idx = np.argmin(diff)
+        optimal_threshold = (
+            thresholds_pr[best_idx] if best_idx < len(thresholds_pr) else 1.0
+        )
+    elif method == "target_recall":
+        if target_recall is None:
+            raise ValueError(
+                "target_recall must be specified when method='target_recall'"
+            )
+        idx = np.argmin(abs(recall - target_recall))
+        optimal_threshold = thresholds_pr[idx] if idx < len(thresholds_pr) else 1.0
+    else:
+        raise ValueError(f"Unknown method: {method}")
+
+    # Calculate predictions with optimal threshold
+    y_pred = (y_prob >= optimal_threshold).astype(int)
+
+    # Calculate confusion matrix
+    tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
+
+    # Calculate metrics directly
+    metrics = {
+        "method": method,
+        "threshold": optimal_threshold,
+        "precision": tp / (tp + fp) if (tp + fp) > 0 else 0,
+        "recall": tp / (tp + fn) if (tp + fn) > 0 else 0,
+        "f1_score": 2 * tp / (2 * tp + fp + fn) if (2 * tp + fp + fn) > 0 else 0,
+        "accuracy": (tp + tn) / (tp + tn + fp + fn),
+    }
+
+    return metrics
+
+
+@tags("model_validation", "threshold_optimization", "classification_metrics")
+@tasks("classification")
+def ClassifierThresholdOptimization(
+    dataset: VMDataset, model: VMModel, methods=None, target_recall=None
+):
+    """
+    Analyzes and visualizes different threshold optimization methods for binary classification models.
+
+    ### Purpose
+
+    The Classifier Threshold Optimization test identifies optimal decision thresholds using various
+    methods to balance different performance metrics. This helps adapt the model's decision boundary
+    to specific business requirements, such as minimizing false positives in fraud detection or
+    achieving target recall in medical diagnosis.
+
+    ### Test Mechanism
+
+    The test implements multiple threshold optimization methods:
+    1. Youden's J statistic (maximizing sensitivity + specificity - 1)
+    2. F1-score optimization (balancing precision and recall)
+    3. Precision-Recall equality point
+    4. Target recall achievement
+    5. Naive (0.5) threshold
+    For each method, it computes ROC and PR curves, identifies optimal points, and provides
+    comprehensive performance metrics at each threshold.
+
+    ### Signs of High Risk
+
+    - Large discrepancies between different optimization methods
+    - Optimal thresholds far from the default 0.5
+    - Poor performance metrics across all thresholds
+    - Significant gap between achieved and target recall
+    - Unstable thresholds across different methods
+    - Extreme trade-offs between precision and recall
+    - Threshold optimization showing minimal impact
+    - Business metrics not improving with optimization
+
+    ### Strengths
+
+    - Multiple optimization strategies for different needs
+    - Visual and numerical results for comparison
+    - Support for business-driven optimization (target recall)
+    - Comprehensive performance metrics at each threshold
+    - Integration with ROC and PR curves
+    - Handles class imbalance through various metrics
+    - Enables informed threshold selection
+    - Supports cost-sensitive decision making
+
+    ### Limitations
+
+    - Assumes cost of false positives/negatives are known
+    - May need adjustment for highly imbalanced datasets
+    - Threshold might not be stable across different samples
+    - Cannot handle multi-class problems directly
+    - Optimization methods may conflict with business needs
+    - Requires sufficient validation data
+    - May not capture temporal changes in optimal threshold
+    - Single threshold may not be optimal for all subgroups
+
+    Args:
+        dataset: VMDataset containing features and target
+        model: VMModel containing predictions
+        methods: List of methods to compare (default: ['youden', 'f1', 'precision_recall'])
+        target_recall: Target recall value if using 'target_recall' method
+
+    Returns:
+        Dictionary containing:
+            - table: DataFrame comparing different threshold optimization methods
+                    (using weighted averages for precision, recall, and f1)
+            - figure: Plotly figure showing ROC and PR curves with optimal thresholds
+    """
+    # Verify binary classification
+    unique_values = np.unique(dataset.y)
+    if len(unique_values) != 2:
+        raise ValueError("Target variable must be binary")
+
+    if methods is None:
+        methods = ["naive", "youden", "f1", "precision_recall"]
+        if target_recall is not None:
+            methods.append("target_recall")
+
+    y_true = dataset.y
+    y_prob = dataset.y_prob(model)
+
+    # Get curve points for plotting
+    fpr, tpr, thresholds_roc = roc_curve(y_true, y_prob)
+    precision, recall, thresholds_pr = precision_recall_curve(y_true, y_prob)
+
+    # Calculate optimal thresholds and metrics
+    results = []
+    optimal_points = {}
+
+    for method in methods:
+        metrics = find_optimal_threshold(y_true, y_prob, method, target_recall)
+        results.append(metrics)
+
+        # Store optimal points for plotting
+        if method == "youden":
+            idx = np.argmax(tpr - fpr)
+            optimal_points[method] = {
+                "x": fpr[idx],
+                "y": tpr[idx],
+                "threshold": thresholds_roc[idx],
+            }
+        elif method in ["f1", "precision_recall", "target_recall"]:
+            idx = np.argmin(abs(thresholds_pr - metrics["threshold"]))
+            optimal_points[method] = {
+                "x": recall[idx],
+                "y": precision[idx],
+                "threshold": metrics["threshold"],
+            }
+
+    # Create visualization
+    fig = make_subplots(
+        rows=1, cols=2, subplot_titles=("ROC Curve", "Precision-Recall Curve")
+    )
+
+    # Plot ROC curve
+    fig.add_trace(
+        go.Scatter(x=fpr, y=tpr, name="ROC Curve", line=dict(color="blue")),
+        row=1,
+        col=1,
+    )
+
+    # Plot PR curve
+    fig.add_trace(
+        go.Scatter(x=recall, y=precision, name="PR Curve", line=dict(color="green")),
+        row=1,
+        col=2,
+    )
+
+    # Add optimal points
+    colors = {
+        "youden": "red",
+        "f1": "orange",
+        "precision_recall": "purple",
+        "target_recall": "brown",
+    }
+
+    for method, points in optimal_points.items():
+        if method == "youden":
+            fig.add_trace(
+                go.Scatter(
+                    x=[points["x"]],
+                    y=[points["y"]],
+                    name=f'{method} (t={points["threshold"]:.2f})',
+                    mode="markers",
+                    marker=dict(size=10, color=colors[method]),
+                ),
+                row=1,
+                col=1,
+            )
+        else:
+            fig.add_trace(
+                go.Scatter(
+                    x=[points["x"]],
+                    y=[points["y"]],
+                    name=f'{method} (t={points["threshold"]:.2f})',
+                    mode="markers",
+                    marker=dict(size=10, color=colors[method]),
+                ),
+                row=1,
+                col=2,
+            )
+
+    # Update layout
+    fig.update_layout(
+        height=500, title_text="Threshold Optimization Analysis", showlegend=True
+    )
+
+    fig.update_xaxes(title_text="False Positive Rate", row=1, col=1)
+    fig.update_xaxes(title_text="Recall", row=1, col=2)
+    fig.update_yaxes(title_text="True Positive Rate", row=1, col=1)
+    fig.update_yaxes(title_text="Precision", row=1, col=2)
+
+    # Create results table and sort by threshold descending
+    table = pd.DataFrame(results).sort_values("threshold", ascending=False)
+
+    return fig, table

validmind/tests/model_validation/sklearn/ConfusionMatrix.py

@@ -106,6 +106,7 @@ def ConfusionMatrix(dataset: VMDataset, model: VMModel):
         autosize=False,
         width=600,
         height=600,
+        title_text="Confusion Matrix",
     )
 
     fig.add_annotation(

validmind/tests/model_validation/sklearn/HyperParametersTuning.py

@@ -2,73 +2,161 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from typing import Union
-
+from typing import Union, Dict, List
 from sklearn.model_selection import GridSearchCV
+from sklearn.metrics import make_scorer, recall_score
 
 from validmind import tags, tasks
-from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("sklearn", "model_performance")
 @tasks("classification", "clustering")
+def custom_recall(y_true, y_pred_proba, threshold=0.5):
+    y_pred = (y_pred_proba >= threshold).astype(int)
+    return recall_score(y_true, y_pred)
+
+
+def _get_metrics(scoring):
+    """Convert scoring parameter to list of metrics."""
+    if scoring is None:
+        return ["accuracy"]
+    return (
+        scoring
+        if isinstance(scoring, list)
+        else list(scoring.keys()) if isinstance(scoring, dict) else [scoring]
+    )
+
+
+def _get_thresholds(thresholds):
+    """Convert thresholds parameter to list."""
+    if thresholds is None:
+        return [0.5]
+    return [thresholds] if isinstance(thresholds, (int, float)) else thresholds
+
+
+def _create_scoring_dict(scoring, metrics, threshold):
+    """Create scoring dictionary for GridSearchCV."""
+    if scoring is None:
+        return None
+
+    scoring_dict = {}
+    for metric in metrics:
+        if metric == "recall":
+            scoring_dict[metric] = make_scorer(
+                custom_recall, needs_proba=True, threshold=threshold
+            )
+        elif metric == "roc_auc":
+            scoring_dict[metric] = "roc_auc"
+        else:
+            scoring_dict[metric] = metric
+    return scoring_dict
+
+
+@tags("sklearn", "model_performance")
+@tasks("clustering", "classification")
 def HyperParametersTuning(
     model: VMModel,
     dataset: VMDataset,
-    param_grid: Union[dict, None] = None,
-    scoring: Union[str, None] = None,
+    param_grid: dict,
+    scoring: Union[str, List, Dict] = None,
+    thresholds: Union[float, List[float]] = None,
+    fit_params: dict = None,
 ):
     """
-    Exerts exhaustive grid search to identify optimal hyperparameters for the model, improving performance.
-
-    ### Purpose:
-
-    The "HyperParametersTuning" metric aims to find the optimal set of hyperparameters for a given model. The test is
-    designed to enhance the performance of the model by determining the best configuration of hyperparameters. The
-    parameters that are being optimized are defined by the parameter grid provided to the metric.
-
-    ### Test Mechanism:
-
-    The HyperParametersTuning test employs a grid search mechanism using the GridSearchCV function from the
-    scikit-learn library. The grid search algorithm systematically works through multiple combinations of parameter
-    values, cross-validating to determine which combination gives the best model performance. The chosen model and the
-    parameter grid passed for tuning are necessary inputs. Once the grid search is complete, the test caches and
-    returns details of the best model and its associated parameters.
-
-    ### Signs of High Risk:
-
-    - The test raises a SkipTestError if the param_grid is not supplied, indicating a lack of specific parameters to
-    optimize, which can be risky for certain model types reliant on parameter tuning.
-    - Poorly chosen scoring metrics that do not align well with the specific model or problem at hand could reflect
-    potential risks or failures in achieving optimal performance.
-
-    ### Strengths:
-
-    - Provides a comprehensive exploration mechanism to identify the best set of hyperparameters for the supplied
-    model, thereby enhancing its performance.
-    - Implements GridSearchCV, simplifying and automating the time-consuming task of hyperparameter tuning.
-
-    ### Limitations:
-
-    - The grid search algorithm can be computationally expensive, especially with large datasets or complex models, and
-    can be time-consuming as it tests all possible combinations within the specified parameter grid.
-    - The effectiveness of the tuning is heavily dependent on the quality of data and only accepts datasets with
-    numerical or ordered categories.
-    - Assumes that the same set of hyperparameters is optimal for all problem sets, which may not be true in every
-    scenario.
-    - There's a potential risk of overfitting the model if the training set is not representative of the data that the
-    model will be applied to.
+    Performs exhaustive grid search over specified parameter ranges to find optimal model configurations
+    across different metrics and decision thresholds.
+
+    ### Purpose
+
+    The Hyperparameter Tuning test systematically explores the model's parameter space to identify optimal
+    configurations. It supports multiple optimization metrics and decision thresholds, providing a comprehensive
+    view of how different parameter combinations affect various aspects of model performance.
+
+    ### Test Mechanism
+
+    The test uses scikit-learn's GridSearchCV to perform cross-validation for each parameter combination.
+    For each specified threshold and optimization metric, it creates a scoring dictionary with
+    threshold-adjusted metrics, performs grid search with cross-validation, records best parameters and
+    corresponding scores, and combines results into a comparative table. This process is repeated for each
+    optimization metric to provide a comprehensive view of model performance under different configurations.
+
+    ### Signs of High Risk
+
+    - Large performance variations across different parameter combinations
+    - Significant discrepancies between different optimization metrics
+    - Best parameters at the edges of the parameter grid
+    - Unstable performance across different thresholds
+    - Overly complex model configurations (risk of overfitting)
+    - Very different optimal parameters for different metrics
+    - Cross-validation scores showing high variance
+    - Extreme parameter values in best configurations
+
+    ### Strengths
+
+    - Comprehensive exploration of parameter space
+    - Supports multiple optimization metrics
+    - Allows threshold optimization
+    - Provides comparative view across different configurations
+    - Uses cross-validation for robust evaluation
+    - Helps understand trade-offs between different metrics
+    - Enables systematic parameter selection
+    - Supports both classification and clustering tasks
+
+    ### Limitations
+
+    - Computationally expensive for large parameter grids
+    - May not find global optimum (limited to grid points)
+    - Cannot handle dependencies between parameters
+    - Memory intensive for large datasets
+    - Limited to scikit-learn compatible models
+    - Cross-validation splits may not preserve time series structure
+    - Grid search may miss optimal values between grid points
+    - Resource intensive for high-dimensional parameter spaces
     """
-    if not param_grid:
-        raise SkipTestError("'param_grid' dictionary must be provided to run this test")
-
-    estimators = GridSearchCV(model.model, param_grid=param_grid, scoring=scoring)
-    estimators.fit(dataset.x, dataset.y)
-
-    return [
-        {
-            "Best Model": estimators.best_estimator_,
-            "Best Parameters": estimators.best_params_,
-        }
-    ]
+    fit_params = fit_params or {}
+
+    # Simple case: no scoring and no thresholds
+    if scoring is None and thresholds is None:
+        estimators = GridSearchCV(model.model, param_grid=param_grid, scoring=None)
+        estimators.fit(dataset.x_df(), dataset.y, **fit_params)
+        return [
+            {
+                "Best Model": estimators.best_estimator_,
+                "Best Parameters": estimators.best_params_,
+            }
+        ]
+
+    # Complex case: with scoring or thresholds
+    results = []
+    metrics = _get_metrics(scoring)
+    thresholds = _get_thresholds(thresholds)
+
+    for threshold in thresholds:
+        scoring_dict = _create_scoring_dict(scoring, metrics, threshold)
+
+        for optimize_for in metrics:
+            estimators = GridSearchCV(
+                model.model,
+                param_grid=param_grid,
+                scoring=scoring_dict,
+                refit=optimize_for if scoring is not None else True,
+            )
+
+            estimators.fit(dataset.x_df(), dataset.y, **fit_params)
+
+            best_index = estimators.best_index_
+            row_result = {
+                "Optimized for": optimize_for,
+                "Threshold": threshold,
+                "Best Parameters": estimators.best_params_,
+            }
+
+            score_key = (
+                "mean_test_score" if scoring is None else f"mean_test_{optimize_for}"
+            )
+            row_result[optimize_for] = estimators.cv_results_[score_key][best_index]
+
+            results.append(row_result)
+
+    return results

validmind/tests/model_validation/sklearn/ModelParameters.py

@@ -0,0 +1,74 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+from validmind import tags, tasks
+
+
+@tags("model_training", "metadata")
+@tasks("classification", "regression")
+def ModelParameters(model, model_params=None):
+    """
+    Extracts and displays model parameters in a structured format for transparency and reproducibility.
+
+    ### Purpose
+
+    The Model Parameters test is designed to provide transparency into model configuration and ensure
+    reproducibility of machine learning models. It accomplishes this by extracting and presenting all
+    relevant parameters that define the model's behavior, making it easier to audit, validate, and
+    reproduce model training.
+
+    ### Test Mechanism
+
+    The test leverages scikit-learn's API convention of get_params() to extract model parameters. It
+    produces a structured DataFrame containing parameter names and their corresponding values. For models
+    that follow scikit-learn's API (including XGBoost, RandomForest, and other estimators), all
+    parameters are automatically extracted and displayed.
+
+    ### Signs of High Risk
+
+    - Missing crucial parameters that should be explicitly set
+    - Extreme parameter values that could indicate overfitting (e.g., unlimited tree depth)
+    - Inconsistent parameters across different versions of the same model type
+    - Parameter combinations known to cause instability or poor performance
+    - Default values used for critical parameters that should be tuned
+
+    ### Strengths
+
+    - Universal compatibility with scikit-learn API-compliant models
+    - Ensures transparency in model configuration
+    - Facilitates model reproducibility and version control
+    - Enables systematic parameter auditing
+    - Supports both classification and regression models
+    - Helps identify potential configuration issues
+
+    ### Limitations
+
+    - Only works with models implementing scikit-learn's get_params() method
+    - Cannot capture dynamic parameters set during model training
+    - Does not validate parameter values for model-specific appropriateness
+    - Parameter meanings and impacts may vary across different model types
+    - Cannot detect indirect parameter interactions or their effects on model performance
+    """
+    # Check if model implements get_params()
+    if not hasattr(model.model, "get_params"):
+        return pd.DataFrame()
+
+    # Get all model parameters
+    params = model.model.get_params()
+
+    # If model_params is None, use all parameters from get_params()
+    if model_params is None:
+        model_params = sorted(params.keys())  # Sort for consistent ordering
+
+    # Create DataFrame with parameters and their values
+    param_df = pd.DataFrame(
+        [
+            {"Parameter": param, "Value": str(params.get(param, "Not specified"))}
+            for param in model_params
+            if params.get(param) is not None
+        ]
+    )
+
+    return param_df

validmind/tests/model_validation/sklearn/ROCCurve.py

@@ -6,7 +6,7 @@ import numpy as np
 import plotly.graph_objects as go
 from sklearn.metrics import roc_auc_score, roc_curve
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 
@@ -77,28 +77,31 @@ def ROCCurve(model: VMModel, dataset: VMDataset):
     fpr, tpr, _ = roc_curve(y_true, y_prob, drop_intermediate=False)
     auc = roc_auc_score(y_true, y_prob)
 
-    return go.Figure(
-        data=[
-            go.Scatter(
-                x=fpr,
-                y=tpr,
-                mode="lines",
-                name=f"ROC curve (AUC = {auc:.2f})",
-                line=dict(color="#DE257E"),
+    return (
+        RawData(fpr=fpr, tpr=tpr, auc=auc),
+        go.Figure(
+            data=[
+                go.Scatter(
+                    x=fpr,
+                    y=tpr,
+                    mode="lines",
+                    name=f"ROC curve (AUC = {auc:.2f})",
+                    line=dict(color="#DE257E"),
+                ),
+                go.Scatter(
+                    x=[0, 1],
+                    y=[0, 1],
+                    mode="lines",
+                    name="Random (AUC = 0.5)",
+                    line=dict(color="grey", dash="dash"),
+                ),
+            ],
+            layout=go.Layout(
+                title=f"ROC Curve for {model.input_id} on {dataset.input_id}",
+                xaxis=dict(title="False Positive Rate"),
+                yaxis=dict(title="True Positive Rate"),
+                width=700,
+                height=500,
             ),
-            go.Scatter(
-                x=[0, 1],
-                y=[0, 1],
-                mode="lines",
-                name="Random (AUC = 0.5)",
-                line=dict(color="grey", dash="dash"),
-            ),
-        ],
-        layout=go.Layout(
-            title=f"ROC Curve for {model.input_id} on {dataset.input_id}",
-            xaxis=dict(title="False Positive Rate"),
-            yaxis=dict(title="True Positive Rate"),
-            width=700,
-            height=500,
         ),
     )