validmind 2.2.5__py3-none-any.whl → 2.3.1__py3-none-any.whl

validmind/tests/model_validation/statsmodels/PDRatingClassPlot.py

@@ -1,151 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import pandas as pd
-import plotly.graph_objects as go
-
-from validmind.vm_models import Figure, Metric
-
-
-@dataclass
-class PDRatingClassPlot(Metric):
-    """
-    Assesses and visualizes credit risk distribution across different rating classes within a dataset via default
-    probabilities.
-
-    **Purpose**: The purpose of the Probability of Default (PD) Rating Class Plot test is to measure and evaluate the
-    distribution of calculated default probabilities across different rating classes. This is critical for
-    understanding and inferring credit risk and can provide insights into how effectively the model is differentiating
-    between different risk levels in a credit dataset.
-
-    **Test Mechanism**: This metric is implemented via a visualization mechanism. It sorts the predicted probabilities
-    of defaults into user-defined rating classes defined in "rating_classes" in default parameters. When it has
-    classified the probabilities, it then calculates the average default rates within each rating class. Subsequently,
-    it produces bar plots for each of these rating classes, illustrating the average likelihood of a default within
-    each class. This process is executed separately for both the training and testing data sets. The classification of
-    predicted probabilities utilizes the pandas "cut" function, sorting and sectioning the data values into bins.
-
-    **Signs of High Risk**:
-
-    - If lower rating classes present higher average likelihoods of default than higher rating classes
-    - If there is poor differentiation between the averages across the different rating classes
-    - If the model generates a significant contrast between the likelihoods for the training set and the testing set,
-    suggestive of model overfitting
-
-    **Strengths**:
-
-    - Presents a clear visual representation of how efficient the model is at predicting credit risk across different
-    risk levels
-    - Allows for rapid identification and understanding of model performance per rating class
-    - Highlights potential overfitting issues by including both training and testing datasets in the analysis
-
-    **Limitations**:
-
-    - Making an incorrect choice for the number of rating classes, either oversimplifying or overcomplicating the
-    distribution of default rates
-    - Relying on the assumption that the rating classes are effective at differentiating risk levels and that the
-    boundaries between classes truly represent the risk distribution
-    - Not accounting for data set class imbalance, which could cause skewed average probabilities
-    - Inability to gauge the overall performance of the model only based on this metric, emphasizing the requirement of
-    combining it with other evaluation metrics
-    """
-
-    name = "pd_rating_class_plot"
-    required_inputs = ["model", "datasets"]
-
-    metadata = {
-        "task_types": ["classification"],
-        "tags": ["visualization", "credit_risk"],
-    }
-
-    default_params = {
-        "title": "PD by Rating Class",
-        "rating_classes": ["A", "B", "C", "D"],
-    }
-
-    @staticmethod
-    def plot_bucket_analysis(df, prob_col, target_col, title, rating_classes):
-        df["bucket"] = pd.cut(
-            df[prob_col], bins=len(rating_classes), labels=rating_classes, right=False
-        )
-        default_rate = df.groupby("bucket")[target_col].mean()
-
-        # Sort the data based on the order of rating_classes
-        sorted_data = sorted(
-            zip(rating_classes, default_rate),
-            key=lambda x: rating_classes.index(x[0]),
-        )
-        rating_classes_sorted, default_rate_sorted = zip(*sorted_data)
-
-        fig = go.Figure()
-
-        # Iterate through the sorted data and create a bar for each score bucket
-        for i, (bucket, rate) in enumerate(
-            zip(rating_classes_sorted, default_rate_sorted)
-        ):
-            fig.add_trace(go.Bar(x=[bucket], y=[rate], name=bucket))
-
-        fig.update_layout(
-            title_text=title,
-            xaxis_title="Rating Class",
-            yaxis_title="Probability of Default",
-            barmode="group",
-        )
-
-        return fig
-
-    def run(self):
-        target_column = self.inputs.model.train_ds.target_column
-        title = self.params["title"]
-        rating_classes = self.params["rating_classes"]
-
-        X_train = self.inputs.datasets[0].x.copy()
-        y_train = self.inputs.datasets[0].y.copy()
-        X_test = self.inputs.datasets[1].x.copy()
-        y_test = self.inputs.datasets[1].y.copy()
-
-        # Compute probabilities
-        X_train["probability"] = self.inputs.model.predict_proba(X_train)
-        X_test["probability"] = self.inputs.model.predict_proba(X_test)
-
-        df_train = pd.concat([X_train, y_train], axis=1)
-        df_test = pd.concat([X_test, y_test], axis=1)
-
-        fig_train = self.plot_bucket_analysis(
-            df_train,
-            "probability",
-            target_column,
-            title + " - Train Data",
-            rating_classes,
-        )
-        fig_test = self.plot_bucket_analysis(
-            df_test,
-            "probability",
-            target_column,
-            title + " - Test Data",
-            rating_classes,
-        )
-
-        return self.cache_results(
-            metric_value={
-                "bucket_analysis": {
-                    "train_probs": list(X_train["probability"]),
-                    "test_probs": list(X_test["probability"]),
-                },
-            },
-            figures=[
-                Figure(
-                    for_object=self,
-                    key="bucket_analysis_train",
-                    figure=fig_train,
-                ),
-                Figure(
-                    for_object=self,
-                    key="bucket_analysis_test",
-                    figure=fig_test,
-                ),
-            ],
-        )

validmind/tests/model_validation/statsmodels/RegressionModelInsampleComparison.py

@@ -1,146 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import pandas as pd
-from sklearn.metrics import mean_squared_error, r2_score
-
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
-
-from .statsutils import adj_r2_score
-
-
-@dataclass
-class RegressionModelInsampleComparison(Metric):
-    """
-    Evaluates and compares in-sample performance of multiple regression models using R-Squared, Adjusted R-Squared,
-    MSE, and RMSE.
-
-    **Purpose**: The RegressionModelInsampleComparison test metric is utilized to evaluate and compare the performance
-    of multiple regression models trained on the same dataset. Key performance indicators for this comparison include
-    statistics related to the goodness of fit - R-Squared, Adjusted R-Squared, Mean Squared Error (MSE), and Root Mean
-    Squared Error (RMSE).
-
-    **Test Mechanism**: The methodology behind this test is as follows -
-    - Firstly, a verification that the list of models to be tested is indeed not empty occurs.
-    - Once confirmed, the In-Sample performance of the models is calculated by a private function,
-    `_in_sample_performance_ols`, that executes the following steps:
-      - Iterates through each model in the supplied list.
-      - For each model, the function extracts the features (`X`) and the target (`y_true`) from the training dataset
-    and computes the predicted target values (`y_pred`).
-      - The performance metrics for the model are calculated using formulas for R-Squared, Adjusted R-Squared, MSE, and
-    RMSE.
-      - The results, including the computed metrics, variables of the model, and the model's identifier, are stored in
-    a dictionary that is appended to a list.
-    - The collected results are finally returned as a pandas dataframe.
-
-    **Signs of High Risk**:
-    - Significantly low values for R-Squared or Adjusted R-Squared.
-    - Significantly high values for MSE and RMSE.
-    Please note that what constitutes as "low" or "high" will vary based on the specific context or domain in which the
-    model is being utilized.
-
-    **Strengths**:
-    - Enables comparison of in-sample performance across different models on the same dataset, providing insights into
-    which model fits the data the best.
-    - Utilizes multiple evaluation methods (R-Squared, Adjusted R-Squared, MSE, RMSE), offering a comprehensive review
-    of a model's performance.
-
-    **Limitations**:
-    - The test measures only in-sample performance, i.e., how well a model fits the data it was trained on. However, it
-    does not give any information on the performance of the model on new, unseen, or out-of-sample data.
-    - Higher in-sample performance might be a result of overfitting, where the model is just memorizing the training
-    data. This test is sensitive to such cases.
-    - The test does not consider additional key factors such as the temporal dynamics of the data, that is, the pattern
-    of changes in data over time.
-    - The test does not provide an automated mechanism to determine if the reported metrics are within acceptable
-    ranges, necessitating human judgment.
-    """
-
-    name = "regression_insample_performance"
-    required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["model_comparison"],
-    }
-
-    def run(self):
-        # Check models list is not empty
-        if not self.inputs.models:
-            raise ValueError("List of models must be provided in the models parameter")
-        all_models = []
-
-        if self.inputs.models is not None:
-            all_models.extend(self.inputs.models)
-
-        in_sample_performance = self._in_sample_performance_ols(
-            all_models, self.inputs.dataset
-        )
-        in_sample_performance_df = pd.DataFrame(in_sample_performance)
-
-        return self.cache_results(
-            {
-                "in_sample_performance": in_sample_performance_df.to_dict(
-                    orient="records"
-                ),
-            }
-        )
-
-    def _in_sample_performance_ols(self, models, dataset):
-        """
-        Computes the in-sample performance evaluation metrics for a list of OLS models.
-        Args:
-        models (list): A list of statsmodels OLS models.
-        Returns:
-        list: A list of dictionaries containing the evaluation results for each model.
-        Each dictionary contains the following keys:
-        - 'Model': A string identifying the model.
-        - 'Independent Variables': A list of strings identifying the independent variables used in the model.
-        - 'R-Squared': The R-squared value of the model.
-        - 'Adjusted R-Squared': The adjusted R-squared value of the model.
-        - 'MSE': The mean squared error of the model.
-        - 'RMSE': The root mean squared error of the model.
-        """
-        evaluation_results = []
-
-        for i, model in enumerate(models):
-            X_columns = dataset.feature_columns
-            y_true = dataset.y
-            y_pred = dataset.y_pred(model)
-
-            # Extract R-squared and Adjusted R-squared
-            r2 = r2_score(y_true, y_pred)
-            adj_r2 = adj_r2_score(y_true, y_pred, len(y_true), len(X_columns))
-            mse = mean_squared_error(y_true=y_true, y_pred=y_pred, squared=True)
-            rmse = mean_squared_error(y_true=y_true, y_pred=y_pred, squared=False)
-
-            # Append the results to the evaluation_results list
-            evaluation_results.append(
-                {
-                    "Model": f"Model {i + 1}",
-                    "Independent Variables": X_columns,
-                    "R-Squared": r2,
-                    "Adjusted R-Squared": adj_r2,
-                    "MSE": mse,
-                    "RMSE": rmse,
-                }
-            )
-
-        return evaluation_results
-
-    def summary(self, metric_value):
-        """
-        Build one table for summarizing the in-sample performance results
-        """
-        summary_in_sample_performance = metric_value["in_sample_performance"]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=summary_in_sample_performance,
-                    metadata=ResultTableMetadata(title="In-Sample Performance Results"),
-                ),
-            ]
-        )

validmind/tests/model_validation/statsmodels/RegressionModelOutsampleComparison.py

@@ -1,144 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import numpy as np
-import pandas as pd
-
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
-
-
-@dataclass
-class RegressionModelOutsampleComparison(Metric):
-    """
-    Computes MSE and RMSE for multiple regression models using out-of-sample test to assess model's prediction accuracy
-    on unseen data.
-
-    **Purpose**: The RegressionModelOutsampleComparison test is designed to evaluate the predictive performance of
-    multiple regression models by means of an out-of-sample test. The primary aim of this test is to validate the
-    model's ability to generalize to unseen data, a common challenge in the context of overfitting. It does this by
-    computing two critical metrics — Mean Squared Error (MSE) and Root Mean Squared Error (RMSE), which provide a
-    quantifiable measure of the model's prediction accuracy on the testing dataset.
-
-    **Test Mechanism**: This test requires multiple models (specifically Ordinary Least Squares - OLS regression
-    models) and a test dataset as inputs. Each model generates predictions using the test dataset. The residuals are
-    then calculated and used to compute the MSE and RMSE for each model. The test outcomes, which include the model's
-    name, its MSE, and RMSE, are recorded and returned in a structured dataframe format.
-
-    **Signs of High Risk**:
-    - High values of MSE or RMSE indicate significant risk, signifying that the model's predictions considerably
-    deviate from the actual values in the test dataset.
-    - Consistently large discrepancies between training and testing performance across various models may indicate an
-    issue with the input data itself or the model selection strategies employed.
-
-    **Strengths**:
-    - This test offers a comparative evaluation of multiple models' out-of-sample performance, enabling the selection
-    of the best performing model.
-    - The use of both MSE and RMSE provides insights into the model's prediction error. While MSE is sensitive to
-    outliers, emphasizing larger errors, RMSE provides a more interpretable measure of average prediction error given
-    that it's in the same unit as the dependent variable.
-
-    **Limitations**:
-    - The applicability of this test is limited to regression tasks, specifically OLS models.
-    - The test operates under the assumption that the test dataset is a representative sample of the population. This
-    might not always hold true and can result in less accurate insights.
-    - The interpretability and the objectivity of the output (MSE and RMSE) can be influenced when the scale of the
-    dependent variable varies significantly, or the distribution of residuals is heavily skewed or contains outliers.
-    """
-
-    name = "regression_outsample_performance"
-    required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["model_comparison"],
-    }
-
-    def run(self):
-        # Check models list is not empty
-        if not self.inputs.models:
-            raise ValueError("List of models must be provided in the models parameter")
-        all_models = []
-        if self.inputs.model is not None:
-            all_models.append(self.inputs.model)
-
-        if self.inputs.models is not None:
-            all_models.extend(self.inputs.models)
-
-        for model in all_models:
-            if model.test_ds is None:
-                raise ValueError(
-                    "Test dataset is missing in the ValidMind Model object"
-                )
-
-        results = self._out_sample_performance_ols(all_models, self.inputs.dataset)
-        return self.cache_results(
-            {
-                "out_sample_performance": results.to_dict(orient="records"),
-            }
-        )
-
-    def _out_sample_performance_ols(self, model_list, dataset):
-        """
-        Returns the out-of-sample performance evaluation metrics of a list of OLS regression models.
-        Args:
-        model_list (list): A list of OLS models to evaluate.
-        test_data (pandas.DataFrame): The test dataset containing the independent and dependent variables.
-        target_col (str): The name of the target variable column in the test dataset.
-        Returns:
-        pandas.DataFrame: A DataFrame containing the evaluation results of the OLS models. The columns are 'Model',
-        'MSE' (Mean Squared Error), and 'RMSE' (Root Mean Squared Error).
-        """
-
-        # Initialize a list to store results
-        results = []
-
-        for fitted_model in model_list:
-            # Extract the column names of the independent variables from the model
-            independent_vars = dataset.feature_columns
-
-            # Separate the target variable and features in the test dataset
-            y_test = dataset.y
-
-            # Predict the test data
-            y_pred = dataset.y_pred(fitted_model)
-
-            # Calculate the residuals
-            residuals = y_test - y_pred
-
-            # Calculate the mean squared error and root mean squared error
-            mse = np.mean(residuals**2)
-            rmse_val = np.sqrt(mse)
-
-            # Store the results
-            model_name_with_vars = f"({', '.join(independent_vars)})"
-            results.append(
-                {
-                    "Model": model_name_with_vars,
-                    "MSE": mse,
-                    "RMSE": rmse_val,
-                }
-            )
-
-        # Create a DataFrame to display the results
-        results_df = pd.DataFrame(results)
-
-        return results_df
-
-    def summary(self, metric_value):
-        """
-        Build one table for summarizing the out-of-sample performance results
-        """
-        summary_out_sample_performance = metric_value["out_sample_performance"]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=summary_out_sample_performance,
-                    metadata=ResultTableMetadata(
-                        title="Out-of-Sample Performance Results"
-                    ),
-                ),
-            ]
-        )

validmind/tests/model_validation/statsmodels/RegressionModelsPerformance.py

@@ -1,127 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-from sklearn.metrics import mean_squared_error, r2_score
-
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
-
-
-@dataclass
-class RegressionModelsPerformance(Metric):
-    """
-    Evaluates and compares regression models' performance using R-squared, Adjusted R-squared, and MSE metrics.
-
-    **Purpose**: This metric is used to evaluate and compare the performance of various regression models. Through the
-    use of key statistical measures such as R-squared, Adjusted R-squared, and Mean Squared Error (MSE), the
-    performance of different models in predicting dependent variables can be assessed both on the data used for
-    training (in-sample) and new, unseen data (out-of-sample).
-
-    **Test Mechanism**: The test evaluates a list of provided regression models. For each model, it calculates their
-    in-sample and out-of-sample performance by deriving the model predictions for the training and testing datasets
-    respectively, and then comparing these predictions to the actual values. In doing so, it calculates R-squared,
-    Adjusted R-squared, and MSE for each model, stores the results, and returns them for comparison.
-
-    **Signs of High Risk**:
-    - High Mean Squared Error (MSE) values.
-    - Strikingly low values of R-squared and Adjusted R-squared.
-    - A significant drop in performance when transitioning from in-sample to out-of-sample evaluations, signaling a
-    potential overfitting issue.
-
-    **Strengths**:
-    - The test permits comparisons of multiple models simultaneously, providing an objective base for identifying the
-    top-performing model.
-    - It delivers both in-sample and out-of-sample evaluations, presenting performance data on unseen data.
-    - The utilization of R-squared and Adjusted R-squared in conjunction with MSE allows for a detailed view of the
-    model's explainability and error rate.
-
-    **Limitations**:
-    - This test is built around the assumption that the residuals of the regression model are normally distributed,
-    which is a fundamental requirement for Ordinary Least Squares (OLS) regression; thus, it could be not suitable for
-    models where this assumption is broken.
-    - The test does not consider cases where higher R-squared or lower MSE values do not necessarily correlate with
-    better predictive performance, particularly in instances of excessively complex models.
-    """
-
-    name = "regression_models_performance"
-    required_inputs = ["models", "in_sample_datasets", "out_of_sample_datasets"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["model_performance", "model_comparison"],
-    }
-
-    def run(self):
-        # Check models list is not empty
-        if not self.inputs.models:
-            raise ValueError("List of models must be provided in the models parameter")
-
-        all_models = []
-
-        if self.inputs.models is not None:
-            all_models.extend(self.inputs.models)
-
-        in_sample_results = self.sample_performance_ols(
-            self.inputs.models, self.inputs.in_sample_datasets
-        )
-        out_of_sample_results = self.sample_performance_ols(
-            self.inputs.models, self.inputs.out_of_sample_datasets
-        )
-
-        return self.cache_results(
-            {
-                "in_sample_performance": in_sample_results,
-                "out_of_sample_performance": out_of_sample_results,
-            }
-        )
-
-    def sample_performance_ols(self, models, datasets):
-        evaluation_results = []
-
-        for model, dataset in zip(models, datasets):
-            X_columns = dataset.feature_columns
-            y_true = dataset.y
-            y_pred = dataset.y_pred(model)
-
-            # Extract R-squared and Adjusted R-squared
-            r2 = r2_score(y_true, y_pred)
-            mse = mean_squared_error(y_true, y_pred)
-            adj_r2 = 1 - ((1 - r2) * (len(y_true) - 1)) / (
-                len(y_true) - len(X_columns) - 1
-            )
-
-            # Append the results to the evaluation_results list
-            evaluation_results.append(
-                {
-                    "Model": model.input_id,
-                    "Independent Variables": X_columns,
-                    "R-Squared": r2,
-                    "Adjusted R-Squared": adj_r2,
-                    "MSE": mse,
-                }
-            )
-
-        return evaluation_results
-
-    def summary(self, metric_value):
-        """
-        Build a table for summarizing the in-sample and out-of-sample performance results
-        """
-        summary_in_sample_performance = metric_value["in_sample_performance"]
-        summary_out_of_sample_performance = metric_value["out_of_sample_performance"]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=summary_in_sample_performance,
-                    metadata=ResultTableMetadata(title="In-Sample Performance Results"),
-                ),
-                ResultTable(
-                    data=summary_out_of_sample_performance,
-                    metadata=ResultTableMetadata(
-                        title="Out-of-Sample Performance Results"
-                    ),
-                ),
-            ]
-        )