validmind 2.5.8__py3-none-any.whl → 2.5.15__py3-none-any.whl

validmind/tests/model_validation/sklearn/FeatureImportanceComparison.py

@@ -1,83 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-import pandas as pd
-from sklearn.inspection import permutation_importance
-
-from validmind import tags, tasks
-
-
-@tags("model_explainability", "sklearn")
-@tasks("regression", "time_series_forecasting")
-def FeatureImportanceComparison(datasets, models, num_features=3):
-    """
-    Compare feature importance scores for each model and generate a summary table
-    with the top important features.
-
-    **Purpose**: The purpose of this function is to compare the feature importance scores for different models applied to various datasets.
-
-    **Test Mechanism**: The function iterates through each dataset-model pair, calculates permutation feature importance (PFI) scores, and generates a summary table with the top `num_features` important features for each model.
-
-    **Signs of High Risk**:
-    - If key features expected to be important are ranked low, it could indicate potential issues with model training or data quality.
-    - High variance in feature importance scores across different models may suggest instability in feature selection.
-
-    **Strengths**:
-    - Provides a clear comparison of the most important features for each model.
-    - Uses permutation importance, which is a model-agnostic method and can be applied to any estimator.
-
-    **Limitations**:
-    - Assumes that the dataset is provided as a DataFrameDataset object with `x_df` and `y_df` methods to access feature and target data.
-    - Requires that `model.model` is compatible with `sklearn.inspection.permutation_importance`.
-    - The function's output is dependent on the number of features specified by `num_features`, which defaults to 3 but can be adjusted.
-
-
-    """
-    results_list = []
-
-    for dataset, model in zip(datasets, models):
-        x = dataset.x_df()
-        y = dataset.y_df()
-
-        pfi_values = permutation_importance(
-            model.model,
-            x,
-            y,
-            random_state=0,
-            n_jobs=-2,
-        )
-
-        # Create a dictionary to store PFI scores
-        pfi = {
-            column: pfi_values["importances_mean"][i]
-            for i, column in enumerate(x.columns)
-        }
-
-        # Sort features by their importance
-        sorted_features = sorted(pfi.items(), key=lambda item: item[1], reverse=True)
-
-        # Extract the top `num_features` features
-        top_features = sorted_features[:num_features]
-
-        # Prepare the result for the current model and dataset
-        result = {
-            "Model": model.input_id,
-            "Dataset": dataset.input_id,
-        }
-
-        # Dynamically add feature columns to the result
-        for i in range(num_features):
-            if i < len(top_features):
-                result[
-                    f"Feature {i + 1}"
-                ] = f"[{top_features[i][0]}; {top_features[i][1]:.4f}]"
-            else:
-                result[f"Feature {i + 1}"] = None
-
-        # Append the result to the list
-        results_list.append(result)
-
-    # Convert the results list to a DataFrame
-    results_df = pd.DataFrame(results_list)
-    return results_df

validmind/tests/model_validation/statsmodels/RegressionCoeffsPlot.py

@@ -1,135 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import pandas as pd
-import plotly.graph_objects as go
-from scipy import stats
-
-from validmind.errors import SkipTestError
-from validmind.vm_models import Figure, Metric
-
-
-@dataclass
-class RegressionCoeffsPlot(Metric):
-    """
-    Visualizes regression coefficients with 95% confidence intervals to assess predictor variables' impact on response
-    variable.
-
-    **Purpose**: The Regression Coefficients with Confidence Intervals plot and metric aims to understand the impact of
-    predictor variables on the response variable in question. This understanding is achieved via the visualization and
-    analysis of the regression model by presenting the coefficients derived from the model along with their associated
-    95% confidence intervals. By doing so, it offers insights into the variability and uncertainty associated with the
-    model's estimates.
-
-    **Test Mechanism**: The test begins by extracting the estimated coefficients and their related standard errors from
-    the regression model under test. It then calculates and draws confidence intervals based on a 95% confidence level
-    (a standard convention in statistics). These intervals provide a range wherein the true value can be expected to
-    fall 95% of the time if the same regression were re-run multiple times with samples drawn from the same population.
-    This information is then visualized as a bar plot, with the predictor variables and their coefficients on the
-    x-axis and y-axis respectively and the confidence intervals represented as error bars.
-
-    **Signs of High Risk**:
-    * If the calculated confidence interval contains the zero value, it could mean the feature/coefficient in question
-    doesn't significantly contribute to prediction in the model.
-    * If there are multiple coefficients exhibiting this behavior, it might raise concerns about overall model
-    reliability.
-    * Very wide confidence intervals might indicate high uncertainty in the associated coefficient estimates.
-
-    **Strengths**:
-    * This metric offers a simple and easily comprehendible visualization of the significance and impact of individual
-    predictor variables in a regression model.
-    * By including confidence intervals, it enables an observer to evaluate the uncertainty around each coefficient
-    estimate.
-
-    **Limitations**:
-    * The test is dependent on a few assumptions about the data, namely normality of residuals and independence of
-    observations, which may not always be true for all types of datasets.
-    * The test does not consider multi-collinearity (correlation among predictor variables), which can potentially
-    distort the model and make interpretation of coefficients challenging.
-    * The test's application is limited to regression tasks and tabular datasets and is not suitable for other types of
-    machine learning assignments or data structures.
-    """
-
-    name = "regression_coeffs_plot"
-    required_inputs = ["models"]
-    tasks = ["regression"]
-    tags = ["tabular_data", "visualization", "model_interpretation"]
-
-    @staticmethod
-    def plot_coefficients_with_ci(model, model_name):
-        # Extract estimated coefficients and standard errors
-        coefficients = model.regression_coefficients()
-        coef = pd.to_numeric(coefficients["coef"])
-        std_err = pd.to_numeric(coefficients["std err"])
-
-        # Calculate confidence intervals
-        confidence_level = 0.95  # 95% confidence interval
-        z_value = stats.norm.ppf((1 + confidence_level) / 2)  # Calculate Z-value
-        lower_ci = coef - z_value * std_err
-        upper_ci = coef + z_value * std_err
-
-        # Create a bar plot with confidence intervals
-        fig = go.Figure()
-
-        fig.add_trace(
-            go.Bar(
-                x=list(coefficients["Feature"].values),
-                y=coef,
-                name="Estimated Coefficients",
-                error_y=dict(
-                    type="data",
-                    symmetric=False,
-                    arrayminus=lower_ci,
-                    array=upper_ci,
-                    visible=True,
-                ),
-            )
-        )
-
-        fig.update_layout(
-            title=f"{model_name} Coefficients with Confidence Intervals",
-            xaxis_title="Predictor Variables",
-            yaxis_title="Coefficients",
-        )
-
-        return fig, {
-            "values": list(coef),
-            "lower_ci": list(lower_ci),
-            "upper_ci": list(upper_ci),
-        }
-
-    def run(self):
-        # Check models list is not empty
-        if not self.inputs.models:
-            raise ValueError("List of models must be provided in the models parameter")
-
-        all_models = []
-        all_figures = []
-        all_metric_values = []
-
-        if self.inputs.models is not None:
-            all_models.extend(self.inputs.models)
-
-        for i, model in enumerate(all_models):
-            if model.library != "statsmodels":
-                raise SkipTestError("Only statsmodels are supported for this metric")
-
-            model_name = f"Model {i+1}"
-
-            fig, metric_values = self.plot_coefficients_with_ci(model, model_name)
-            all_figures.append(
-                Figure(
-                    for_object=self,
-                    key=f"{model_name}_coefficients_ci_plot",
-                    figure=fig,
-                )
-            )
-            all_metric_values.append({"name": model_name, "metrics": metric_values})
-
-        return self.cache_results(
-            metric_value=all_metric_values,
-            figures=all_figures,
-        )

validmind/tests/model_validation/statsmodels/RegressionModelsCoeffs.py

@@ -1,103 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import pandas as pd
-
-from validmind.errors import SkipTestError
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
-
-
-@dataclass
-class RegressionModelsCoeffs(Metric):
-    """
-    Compares feature importance by evaluating and contrasting coefficients of different regression models.
-
-    **Purpose**: The 'RegressionModelsCoeffs' metric is utilized to evaluate and compare coefficients of different
-    regression models trained on the same dataset. By examining how each model weighted the importance of different
-    features during training, this metric provides key insights into which factors have the most impact on the model's
-    predictions and how these patterns differ across models.
-
-    **Test Mechanism**: The test operates by extracting the coefficients of each regression model using the
-    'regression_coefficients()' method. These coefficients are then consolidated into a dataframe, with each row
-    representing a model and columns corresponding to each feature's coefficient. It must be noted that this test is
-    exclusive to 'statsmodels' and 'R' models, other models will result in a 'SkipTestError'.
-
-    **Signs of High Risk**:
-    - Discrepancies in how different models weight the same features
-    - Unexpectedly high or low coefficients
-    - The test is inapplicable to certain models because they are not from 'statsmodels' or 'R' libraries
-
-    **Strengths**:
-    - Enables insight into the training process of different models
-    - Allows comparison of feature importance across models
-    - Through the review of feature coefficients, the test provides a more transparent evaluation of the model and
-    highlights significant weights and biases in the training procedure
-
-    **Limitations**:
-    - The test is only compatible with 'statsmodels' and 'R' regression models
-    - While the test provides contrast in feature weightings among models, it does not establish the most appropriate
-    or accurate weighting, thus remaining subject to interpretation
-    - It does not account for potential overfitting or underfitting of models
-    - The computed coefficients might not lead to effective performance on unseen data
-    """
-
-    name = "regression_models_coefficients"
-    required_inputs = ["models"]
-    tasks = ["regression"]
-    tags = ["model_comparison"]
-
-    def _build_model_summaries(self, all_coefficients):
-        all_models_df = pd.DataFrame()
-
-        for i, coefficients in enumerate(all_coefficients):
-            model_name = f"Model {i+1}"
-            # The coefficients summary object needs an additional "Model" column at the beginning
-            coefficients["Model"] = model_name
-            all_models_df = pd.concat([all_models_df, coefficients])
-
-        # Reorder columns to have 'Model' as the first column and reset the index
-        all_models_df = all_models_df.reset_index(drop=True)[
-            ["Model"] + [col for col in all_models_df.columns if col != "Model"]
-        ]
-
-        return all_models_df
-
-    def run(self):
-        # Check models list is not empty
-        if not self.inputs.models or len(self.inputs.models) == 0:
-            raise ValueError("List of models must be provided in the models parameter")
-
-        for model in self.inputs.models:
-            if model.library != "statsmodels":
-                raise SkipTestError(
-                    "Only statsmodels models are supported for this metric"
-                )
-
-        coefficients = [m.regression_coefficients() for m in self.inputs.models]
-        all_models_summary = self._build_model_summaries(coefficients)
-
-        return self.cache_results(
-            {
-                "coefficients_summary": all_models_summary.to_dict(orient="records"),
-            }
-        )
-
-    def summary(self, metric_value):
-        """
-        Build one table for summarizing the regression models' coefficients
-        """
-        coefficients_summary = metric_value["coefficients_summary"]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=coefficients_summary,
-                    metadata=ResultTableMetadata(
-                        title="Regression Models' Coefficients"
-                    ),
-                ),
-            ]
-        )