validmind 2.3.1__py3-none-any.whl → 2.3.5__py3-none-any.whl

validmind/tests/model_validation/ModelPredictionResiduals.py

@@ -0,0 +1,103 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+import plotly.graph_objects as go
+from scipy.stats import kstest
+
+from validmind import tags, tasks
+
+
+@tags("regression")
+@tasks("residual_analysis", "visualization")
+def ModelPredictionResiduals(
+    datasets, models, nbins=100, p_value_threshold=0.05, start_date=None, end_date=None
+):
+    """
+    Plot the residuals and histograms for each model, and generate a summary table
+    with the Kolmogorov-Smirnov normality test results.
+
+    **Purpose**: The purpose of this function is to visualize the residuals of model predictions and
+    assess the normality of residuals using the Kolmogorov-Smirnov test.
+
+    **Test Mechanism**: The function iterates through each dataset-model pair, calculates residuals, and generates
+    two figures for each model: one for the time series of residuals and one for the histogram of residuals.
+    It also calculates the KS test for normality and summarizes the results in a table.
+
+    **Signs of High Risk**:
+    - If the residuals are not normally distributed, it could indicate issues with model assumptions.
+    - High skewness or kurtosis in the residuals may indicate model misspecification.
+
+    **Strengths**:
+    - Provides a clear visualization of residuals over time and their distribution.
+    - Includes statistical tests to assess the normality of residuals.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with a .df attribute to access
+      the pandas DataFrame.
+    - Only generates plots for datasets with a datetime index, and will raise an error for other types of indices.
+    """
+
+    figures = []
+    summary = []
+
+    for dataset, model in zip(datasets, models):
+        df = dataset.df.copy()
+
+        # Filter DataFrame by date range if specified
+        if start_date:
+            df = df[df.index >= pd.to_datetime(start_date)]
+        if end_date:
+            df = df[df.index <= pd.to_datetime(end_date)]
+
+        y_true = dataset.y
+        y_pred = dataset.y_pred(model)
+        residuals = y_true - y_pred
+
+        # Plot residuals
+        residuals_fig = go.Figure()
+        residuals_fig.add_trace(
+            go.Scatter(x=df.index, y=residuals, mode="lines", name="Residuals")
+        )
+        residuals_fig.update_layout(
+            title=f"Residuals for {model.input_id}",
+            xaxis_title="Date",
+            yaxis_title="Residuals",
+            font=dict(size=16),
+            showlegend=False,
+        )
+        figures.append(residuals_fig)
+
+        # Plot histogram of residuals
+        hist_fig = go.Figure()
+        hist_fig.add_trace(go.Histogram(x=residuals, nbinsx=nbins, name="Residuals"))
+        hist_fig.update_layout(
+            title=f"Histogram of Residuals for {model.input_id}",
+            xaxis_title="Residuals",
+            yaxis_title="Frequency",
+            font=dict(size=16),
+            showlegend=False,
+        )
+        figures.append(hist_fig)
+
+        # Perform KS normality test
+        ks_stat, p_value = kstest(
+            residuals, "norm", args=(residuals.mean(), residuals.std())
+        )
+        ks_normality = "Normal" if p_value > p_value_threshold else "Not Normal"
+
+        summary.append(
+            {
+                "Model": model.input_id,
+                "KS Statistic": ks_stat,
+                "p-value": p_value,
+                "KS Normality": ks_normality,
+                "p-value Threshold": p_value_threshold,
+            }
+        )
+
+    # Create a summary DataFrame for the KS normality test results
+    summary_df = pd.DataFrame(summary)
+
+    return (summary_df, *figures)

validmind/tests/model_validation/TimeSeriesPredictionWithCI.py

@@ -0,0 +1,131 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+from scipy.stats import norm
+
+from validmind import tags, tasks
+
+
+@tags("model_predictions", "visualization")
+@tasks("regression", "time_series_forecasting")
+def TimeSeriesPredictionWithCI(dataset, model, confidence=0.95):
+    """
+    Plot actual vs predicted values for a time series with confidence intervals and compute breaches.
+
+    **Purpose**: The purpose of this function is to visualize the actual versus predicted values for time series data, including confidence intervals, and to compute and report the number of breaches beyond these intervals.
+
+    **Test Mechanism**: The function calculates the standard deviation of prediction errors, determines the confidence intervals, and counts the number of actual values that fall outside these intervals (breaches). It then generates a plot with the actual values, predicted values, and confidence intervals, and returns a DataFrame summarizing the breach information.
+
+    **Signs of High Risk**:
+    - A high number of breaches indicates that the model's predictions are not reliable within the specified confidence level.
+    - Significant deviations between actual and predicted values may highlight model inadequacies or issues with data quality.
+
+    **Strengths**:
+    - Provides a visual representation of prediction accuracy and the uncertainty around predictions.
+    - Includes a statistical measure of prediction reliability through confidence intervals.
+    - Computes and reports breaches, offering a quantitative assessment of prediction performance.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with a datetime index.
+    - Requires that `dataset.y_pred(model)` returns the predicted values for the model.
+    - The calculation of confidence intervals assumes normally distributed errors, which may not hold for all datasets.
+    """
+    dataset_name = dataset.input_id
+    model_name = model.input_id
+    time_index = dataset.df.index  # Assuming the index of the dataset is datetime
+
+    # Get actual and predicted values
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Compute the standard deviation of the errors
+    errors = y_true - y_pred
+    std_error = np.std(errors)
+
+    # Compute z-score for the given confidence level
+    z_score = norm.ppf(1 - (1 - confidence) / 2)
+
+    # Compute confidence intervals
+    lower_conf = y_pred - z_score * std_error
+    upper_conf = y_pred + z_score * std_error
+
+    # Calculate breaches
+    upper_breaches = (y_true > upper_conf).sum()
+    lower_breaches = (y_true < lower_conf).sum()
+    total_breaches = upper_breaches + lower_breaches
+
+    # Create DataFrame
+    breaches_df = pd.DataFrame(
+        {
+            "Confidence Level": [confidence],
+            "Total Breaches": [total_breaches],
+            "Upper Breaches": [upper_breaches],
+            "Lower Breaches": [lower_breaches],
+        }
+    )
+
+    # Plotting
+    fig = go.Figure()
+
+    # Plot actual values
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=y_true,
+            mode="lines",
+            name="Actual Values",
+            line=dict(color="blue"),
+        )
+    )
+
+    # Plot predicted values
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=y_pred,
+            mode="lines",
+            name=f"Predicted by {model_name}",
+            line=dict(color="red"),
+        )
+    )
+
+    # Add confidence interval lower bound as an invisible line
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=lower_conf,
+            mode="lines",
+            line=dict(width=0),
+            showlegend=False,
+            name="CI Lower",
+        )
+    )
+
+    # Add confidence interval upper bound and fill area
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=upper_conf,
+            mode="lines",
+            fill="tonexty",
+            fillcolor="rgba(200, 200, 200, 0.5)",
+            line=dict(width=0),
+            showlegend=True,
+            name="Confidence Interval",
+        )
+    )
+
+    # Update layout
+    fig.update_layout(
+        title=f"Time Series Actual vs Predicted Values for {dataset_name} and {model_name}",
+        xaxis_title="Time",
+        yaxis_title="Values",
+        legend_title="Legend",
+        template="plotly_white",
+    )
+
+    return fig, breaches_df

validmind/tests/model_validation/TimeSeriesPredictionsPlot.py

@@ -0,0 +1,76 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import plotly.express as px
+import plotly.graph_objects as go
+
+from validmind import tags, tasks
+
+
+@tags("model_predictions", "visualization")
+@tasks("regression", "time_series_forecasting")
+def TimeSeriesPredictionsPlot(datasets, models):
+    """
+    Plot actual vs predicted values for time series data and generate a visual comparison for each model.
+
+    **Purpose**: The purpose of this function is to visualize the actual versus predicted values for time series data across different models.
+
+    **Test Mechanism**: The function iterates through each dataset-model pair, plots the actual values from the dataset, and overlays the predicted values from each model using Plotly for interactive visualization.
+
+    **Signs of High Risk**:
+    - Large discrepancies between actual and predicted values indicate poor model performance.
+    - Systematic deviations in predicted values can highlight model bias or issues with data patterns.
+
+    **Strengths**:
+    - Provides a clear visual comparison of model predictions against actual values.
+    - Uses Plotly for interactive and visually appealing plots.
+    - Can handle multiple models and datasets, displaying them with distinct colors.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with a datetime index.
+    - Requires that `dataset.y_pred(model)` returns the predicted values for the model.
+    - Visualization might become cluttered with a large number of models or datasets.
+    """
+    fig = go.Figure()
+
+    # Use Plotly's color sequence for different model predictions
+    colors = px.colors.qualitative.Plotly
+
+    # Plot actual values from the first dataset
+    dataset = datasets[0]
+    time_index = dataset.df.index  # Assuming the index of the dataset is datetime
+    fig.add_trace(
+        go.Scatter(
+            x=time_index,
+            y=dataset.y,
+            mode="lines",
+            name="Actual Values",
+            line=dict(color="blue"),
+        )
+    )
+
+    # Plot predicted values for each dataset-model pair
+    for idx, (dataset, model) in enumerate(zip(datasets, models)):
+        model_name = model.input_id
+        y_pred = dataset.y_pred(model)
+        fig.add_trace(
+            go.Scatter(
+                x=time_index,
+                y=y_pred,
+                mode="lines",
+                name=f"Predicted by {model_name}",
+                line=dict(color=colors[idx % len(colors)]),
+            )
+        )
+
+    # Update layout
+    fig.update_layout(
+        title="Time Series Actual vs Predicted Values",
+        xaxis_title="Time",
+        yaxis_title="Values",
+        legend_title="Legend",
+        template="plotly_white",
+    )
+
+    return fig

validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py

@@ -0,0 +1,103 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+
+import pandas as pd
+import plotly.express as px
+from sklearn import metrics
+
+from validmind import tags, tasks
+
+
+@tags("model_performance", "sklearn")
+@tasks("regression", "time_series_forecasting")
+def TimeSeriesR2SquareBySegments(datasets, models, segments=None):
+    """
+    Plot R-Squared values for each model over specified time segments and generate a bar chart
+    with the results.
+
+    **Purpose**: The purpose of this function is to plot the R-Squared values for different models applied to various segments of the time series data.
+
+    **Parameters**:
+    - datasets: List of datasets to evaluate.
+    - models: List of models to evaluate.
+    - segments: Dictionary with 'start_date' and 'end_date' keys containing lists of start and end dates for each segments. If None, the time series will be segmented into two halves.
+
+    **Test Mechanism**: The function iterates through each dataset-model pair, calculates the R-Squared values for specified time segments, and generates a bar chart with these results.
+
+    **Signs of High Risk**:
+    - If the R-Squared values are significantly low for certain segments, it could indicate that the model is not explaining much of the variability in the dataset for those segments.
+
+    **Strengths**:
+    - Provides a visual representation of model performance across different time segments.
+    - Allows for identification of segments where models perform poorly.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with `y`, `y_pred`, and `feature_columns` attributes.
+    - Requires that `dataset.y_pred(model)` returns the predicted values for the model.
+    - Assumes that `y_true` and `y_pred` are pandas Series with datetime indices.
+    """
+    results_list = []
+
+    for dataset, model in zip(datasets, models):
+        dataset_name = dataset.input_id
+        model_name = model.input_id
+
+        y_true = dataset.y
+        y_pred = dataset.y_pred(model)
+
+        # Ensure y_true and y_pred are pandas Series with the same index
+        if not isinstance(y_true, pd.Series):
+            y_true = pd.Series(y_true, index=dataset.df.index)
+        if not isinstance(y_pred, pd.Series):
+            y_pred = pd.Series(y_pred, index=dataset.df.index)
+
+        index = dataset.df.index
+
+        if segments is None:
+            mid_point = len(index) // 2
+            segments = {
+                "start_date": [index.min(), index[mid_point]],
+                "end_date": [index[mid_point - 1], index.max()],
+            }
+
+        for segment_index, (start_date, end_date) in enumerate(
+            zip(segments["start_date"], segments["end_date"])
+        ):
+            mask = (index >= start_date) & (index <= end_date)
+            y_true_segment = y_true.loc[mask]
+            y_pred_segment = y_pred.loc[mask]
+
+            if len(y_true_segment) > 0 and len(y_pred_segment) > 0:
+                r2s = metrics.r2_score(y_true_segment, y_pred_segment)
+                results_list.append(
+                    {
+                        "Model": model_name,
+                        "Dataset": dataset_name,
+                        "Segments": f"Segment {segment_index + 1}",
+                        "Start Date": start_date,
+                        "End Date": end_date,
+                        "R-Squared": r2s,
+                    }
+                )
+
+    # Convert results list to a DataFrame
+    results_df = pd.DataFrame(results_list)
+
+    # Plotting
+    fig = px.bar(
+        results_df,
+        x="Segments",
+        y="R-Squared",
+        color="Model",
+        barmode="group",
+        title="R-Squared Comparison by Segment and Model",
+        labels={
+            "R-Squared": "R-Squared Value",
+            "Segment": "Time Segment",
+            "Model": "Model",
+        },
+    )
+
+    return fig, results_df

validmind/tests/model_validation/sklearn/FeatureImportanceComparison.py

@@ -0,0 +1,83 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+from sklearn.inspection import permutation_importance
+
+from validmind import tags, tasks
+
+
+@tags("model_explainability", "sklearn")
+@tasks("regression", "time_series_forecasting")
+def FeatureImportanceComparison(datasets, models, num_features=3):
+    """
+    Compare feature importance scores for each model and generate a summary table
+    with the top important features.
+
+    **Purpose**: The purpose of this function is to compare the feature importance scores for different models applied to various datasets.
+
+    **Test Mechanism**: The function iterates through each dataset-model pair, calculates permutation feature importance (PFI) scores, and generates a summary table with the top `num_features` important features for each model.
+
+    **Signs of High Risk**:
+    - If key features expected to be important are ranked low, it could indicate potential issues with model training or data quality.
+    - High variance in feature importance scores across different models may suggest instability in feature selection.
+
+    **Strengths**:
+    - Provides a clear comparison of the most important features for each model.
+    - Uses permutation importance, which is a model-agnostic method and can be applied to any estimator.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with `x_df` and `y_df` methods to access feature and target data.
+    - Requires that `model.model` is compatible with `sklearn.inspection.permutation_importance`.
+    - The function's output is dependent on the number of features specified by `num_features`, which defaults to 3 but can be adjusted.
+
+
+    """
+    results_list = []
+
+    for dataset, model in zip(datasets, models):
+        x = dataset.x_df()
+        y = dataset.y_df()
+
+        pfi_values = permutation_importance(
+            model.model,
+            x,
+            y,
+            random_state=0,
+            n_jobs=-2,
+        )
+
+        # Create a dictionary to store PFI scores
+        pfi = {
+            column: pfi_values["importances_mean"][i]
+            for i, column in enumerate(x.columns)
+        }
+
+        # Sort features by their importance
+        sorted_features = sorted(pfi.items(), key=lambda item: item[1], reverse=True)
+
+        # Extract the top `num_features` features
+        top_features = sorted_features[:num_features]
+
+        # Prepare the result for the current model and dataset
+        result = {
+            "Model": model.input_id,
+            "Dataset": dataset.input_id,
+        }
+
+        # Dynamically add feature columns to the result
+        for i in range(num_features):
+            if i < len(top_features):
+                result[
+                    f"Feature {i + 1}"
+                ] = f"[{top_features[i][0]}; {top_features[i][1]:.4f}]"
+            else:
+                result[f"Feature {i + 1}"] = None
+
+        # Append the result to the list
+        results_list.append(result)
+
+    # Convert the results list to a DataFrame
+    results_df = pd.DataFrame(results_list)
+    return results_df

validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py

@@ -121,7 +121,7 @@ class PermutationFeatureImportance(Metric):
             figures=[
                 Figure(
                     for_object=self,
-                    key="pfi",
+                    key=f"pfi_{self.inputs.dataset.input_id}_{self.inputs.model.input_id}",
                     figure=fig,
                 ),
             ],

validmind/tests/model_validation/sklearn/RegressionErrorsComparison.py

@@ -0,0 +1,76 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+import pandas as pd
+from sklearn import metrics
+
+from validmind import tags, tasks
+from validmind.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+@tags("model_performance", "sklearn")
+@tasks("regression", "time_series_forecasting")
+def RegressionErrorsComparison(datasets, models):
+    """
+    Compare regression error metrics for each model and generate a summary table
+    with the results.
+
+    **Purpose**: The purpose of this function is to compare the regression errors for different models applied to various datasets.
+
+    **Test Mechanism**: The function iterates through each dataset-model pair, calculates various error metrics (MAE, MSE, MAPE, MBD), and generates a summary table with these results.
+
+    **Signs of High Risk**:
+    - High Mean Absolute Error (MAE) or Mean Squared Error (MSE) indicates poor model performance.
+    - High Mean Absolute Percentage Error (MAPE) suggests large percentage errors, especially problematic if the true values are small.
+    - Mean Bias Deviation (MBD) significantly different from zero indicates systematic overestimation or underestimation by the model.
+
+    **Strengths**:
+    - Provides multiple error metrics to assess model performance from different perspectives.
+    - Includes a check to avoid division by zero when calculating MAPE.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with `y`, `y_pred`, and `feature_columns` attributes.
+    - The function relies on the `logger` from `validmind.logging` to warn about zero values in `y_true`, which should be correctly implemented and imported.
+    - Requires that `dataset.y_pred(model)` returns the predicted values for the model.
+    """
+    results_list = []
+
+    for dataset, model in zip(datasets, models):
+        dataset_name = dataset.input_id
+        model_name = model.input_id
+
+        y_true = dataset.y
+        y_pred = dataset.y_pred(model)  # Assuming dataset has X for features
+        y_true = y_true.astype(y_pred.dtype)
+
+        mae = metrics.mean_absolute_error(y_true, y_pred)
+        mse = metrics.mean_squared_error(y_true, y_pred)
+
+        if np.any(y_true == 0):
+            logger.warning(
+                "y_true contains zero values. Skipping MAPE calculation to avoid division by zero."
+            )
+            mape = None
+        else:
+            mape = np.mean(np.abs((y_true - y_pred) / y_true)) * 100
+        mbd = np.mean(y_pred - y_true)
+
+        # Append results to the list
+        results_list.append(
+            {
+                "Model": model_name,
+                "Dataset": dataset_name,
+                "Mean Absolute Error (MAE)": mae,
+                "Mean Squared Error (MSE)": mse,
+                "Mean Absolute Percentage Error (MAPE)": mape,
+                "Mean Bias Deviation (MBD)": mbd,
+            }
+        )
+
+    # Convert results list to a DataFrame
+    results_df = pd.DataFrame(results_list)
+    return results_df

validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.py

@@ -0,0 +1,63 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+from sklearn import metrics
+
+from validmind import tags, tasks
+from validmind.tests.model_validation.statsmodels.statsutils import adj_r2_score
+
+
+@tags("model_performance", "sklearn")
+@tasks("regression", "time_series_forecasting")
+def RegressionR2SquareComparison(datasets, models):
+    """
+    Compare R-Squared and Adjusted R-Squared values for each model and generate a summary table
+    with the results.
+
+    **Purpose**: The purpose of this function is to compare the R-Squared and Adjusted R-Squared values for different models applied to various datasets.
+
+    **Test Mechanism**: The function iterates through each dataset-model pair, calculates the R-Squared and Adjusted R-Squared values, and generates a summary table with these results.
+
+    **Signs of High Risk**:
+    - If the R-Squared values are significantly low, it could indicate that the model is not explaining much of the variability in the dataset.
+    - A significant difference between R-Squared and Adjusted R-Squared values might indicate that the model includes irrelevant features.
+
+    **Strengths**:
+    - Provides a quantitative measure of model performance in terms of variance explained.
+    - Adjusted R-Squared accounts for the number of predictors, making it a more reliable measure when comparing models with different numbers of features.
+
+    **Limitations**:
+    - Assumes that the dataset is provided as a DataFrameDataset object with `y`, `y_pred`, and `feature_columns` attributes.
+    - The function relies on `adj_r2_score` from the `statsmodels.statsutils` module, which should be correctly implemented and imported.
+    - Requires that `dataset.y_pred(model)` returns the predicted values for the model.
+
+    """
+    results_list = []
+
+    for dataset, model in zip(datasets, models):
+        dataset_name = dataset.input_id
+        model_name = model.input_id
+
+        y_true = dataset.y
+        y_pred = dataset.y_pred(model)  # Assuming dataset has X for features
+        y_true = y_true.astype(y_pred.dtype)
+
+        r2s = metrics.r2_score(y_true, y_pred)
+        X_columns = dataset.feature_columns
+        adj_r2 = adj_r2_score(y_true, y_pred, len(y_true), len(X_columns))
+
+        # Append results to the list
+        results_list.append(
+            {
+                "Model": model_name,
+                "Dataset": dataset_name,
+                "R-Squared": r2s,
+                "Adjusted R-Squared": adj_r2,
+            }
+        )
+
+    # Convert results list to a DataFrame
+    results_df = pd.DataFrame(results_list)
+    return results_df