validmind 2.2.6__py3-none-any.whl → 2.3.3__py3-none-any.whl

validmind/tests/model_validation/statsmodels/RegressionModelInsampleComparison.py

@@ -1,146 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import pandas as pd
-from sklearn.metrics import mean_squared_error, r2_score
-
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
-
-from .statsutils import adj_r2_score
-
-
-@dataclass
-class RegressionModelInsampleComparison(Metric):
-    """
-    Evaluates and compares in-sample performance of multiple regression models using R-Squared, Adjusted R-Squared,
-    MSE, and RMSE.
-
-    **Purpose**: The RegressionModelInsampleComparison test metric is utilized to evaluate and compare the performance
-    of multiple regression models trained on the same dataset. Key performance indicators for this comparison include
-    statistics related to the goodness of fit - R-Squared, Adjusted R-Squared, Mean Squared Error (MSE), and Root Mean
-    Squared Error (RMSE).
-
-    **Test Mechanism**: The methodology behind this test is as follows -
-    - Firstly, a verification that the list of models to be tested is indeed not empty occurs.
-    - Once confirmed, the In-Sample performance of the models is calculated by a private function,
-    `_in_sample_performance_ols`, that executes the following steps:
-      - Iterates through each model in the supplied list.
-      - For each model, the function extracts the features (`X`) and the target (`y_true`) from the training dataset
-    and computes the predicted target values (`y_pred`).
-      - The performance metrics for the model are calculated using formulas for R-Squared, Adjusted R-Squared, MSE, and
-    RMSE.
-      - The results, including the computed metrics, variables of the model, and the model's identifier, are stored in
-    a dictionary that is appended to a list.
-    - The collected results are finally returned as a pandas dataframe.
-
-    **Signs of High Risk**:
-    - Significantly low values for R-Squared or Adjusted R-Squared.
-    - Significantly high values for MSE and RMSE.
-    Please note that what constitutes as "low" or "high" will vary based on the specific context or domain in which the
-    model is being utilized.
-
-    **Strengths**:
-    - Enables comparison of in-sample performance across different models on the same dataset, providing insights into
-    which model fits the data the best.
-    - Utilizes multiple evaluation methods (R-Squared, Adjusted R-Squared, MSE, RMSE), offering a comprehensive review
-    of a model's performance.
-
-    **Limitations**:
-    - The test measures only in-sample performance, i.e., how well a model fits the data it was trained on. However, it
-    does not give any information on the performance of the model on new, unseen, or out-of-sample data.
-    - Higher in-sample performance might be a result of overfitting, where the model is just memorizing the training
-    data. This test is sensitive to such cases.
-    - The test does not consider additional key factors such as the temporal dynamics of the data, that is, the pattern
-    of changes in data over time.
-    - The test does not provide an automated mechanism to determine if the reported metrics are within acceptable
-    ranges, necessitating human judgment.
-    """
-
-    name = "regression_insample_performance"
-    required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["model_comparison"],
-    }
-
-    def run(self):
-        # Check models list is not empty
-        if not self.inputs.models:
-            raise ValueError("List of models must be provided in the models parameter")
-        all_models = []
-
-        if self.inputs.models is not None:
-            all_models.extend(self.inputs.models)
-
-        in_sample_performance = self._in_sample_performance_ols(
-            all_models, self.inputs.dataset
-        )
-        in_sample_performance_df = pd.DataFrame(in_sample_performance)
-
-        return self.cache_results(
-            {
-                "in_sample_performance": in_sample_performance_df.to_dict(
-                    orient="records"
-                ),
-            }
-        )
-
-    def _in_sample_performance_ols(self, models, dataset):
-        """
-        Computes the in-sample performance evaluation metrics for a list of OLS models.
-        Args:
-        models (list): A list of statsmodels OLS models.
-        Returns:
-        list: A list of dictionaries containing the evaluation results for each model.
-        Each dictionary contains the following keys:
-        - 'Model': A string identifying the model.
-        - 'Independent Variables': A list of strings identifying the independent variables used in the model.
-        - 'R-Squared': The R-squared value of the model.
-        - 'Adjusted R-Squared': The adjusted R-squared value of the model.
-        - 'MSE': The mean squared error of the model.
-        - 'RMSE': The root mean squared error of the model.
-        """
-        evaluation_results = []
-
-        for i, model in enumerate(models):
-            X_columns = dataset.feature_columns
-            y_true = dataset.y
-            y_pred = dataset.y_pred(model)
-
-            # Extract R-squared and Adjusted R-squared
-            r2 = r2_score(y_true, y_pred)
-            adj_r2 = adj_r2_score(y_true, y_pred, len(y_true), len(X_columns))
-            mse = mean_squared_error(y_true=y_true, y_pred=y_pred, squared=True)
-            rmse = mean_squared_error(y_true=y_true, y_pred=y_pred, squared=False)
-
-            # Append the results to the evaluation_results list
-            evaluation_results.append(
-                {
-                    "Model": f"Model {i + 1}",
-                    "Independent Variables": X_columns,
-                    "R-Squared": r2,
-                    "Adjusted R-Squared": adj_r2,
-                    "MSE": mse,
-                    "RMSE": rmse,
-                }
-            )
-
-        return evaluation_results
-
-    def summary(self, metric_value):
-        """
-        Build one table for summarizing the in-sample performance results
-        """
-        summary_in_sample_performance = metric_value["in_sample_performance"]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=summary_in_sample_performance,
-                    metadata=ResultTableMetadata(title="In-Sample Performance Results"),
-                ),
-            ]
-        )

validmind/tests/model_validation/statsmodels/RegressionModelOutsampleComparison.py

@@ -1,144 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import numpy as np
-import pandas as pd
-
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
-
-
-@dataclass
-class RegressionModelOutsampleComparison(Metric):
-    """
-    Computes MSE and RMSE for multiple regression models using out-of-sample test to assess model's prediction accuracy
-    on unseen data.
-
-    **Purpose**: The RegressionModelOutsampleComparison test is designed to evaluate the predictive performance of
-    multiple regression models by means of an out-of-sample test. The primary aim of this test is to validate the
-    model's ability to generalize to unseen data, a common challenge in the context of overfitting. It does this by
-    computing two critical metrics — Mean Squared Error (MSE) and Root Mean Squared Error (RMSE), which provide a
-    quantifiable measure of the model's prediction accuracy on the testing dataset.
-
-    **Test Mechanism**: This test requires multiple models (specifically Ordinary Least Squares - OLS regression
-    models) and a test dataset as inputs. Each model generates predictions using the test dataset. The residuals are
-    then calculated and used to compute the MSE and RMSE for each model. The test outcomes, which include the model's
-    name, its MSE, and RMSE, are recorded and returned in a structured dataframe format.
-
-    **Signs of High Risk**:
-    - High values of MSE or RMSE indicate significant risk, signifying that the model's predictions considerably
-    deviate from the actual values in the test dataset.
-    - Consistently large discrepancies between training and testing performance across various models may indicate an
-    issue with the input data itself or the model selection strategies employed.
-
-    **Strengths**:
-    - This test offers a comparative evaluation of multiple models' out-of-sample performance, enabling the selection
-    of the best performing model.
-    - The use of both MSE and RMSE provides insights into the model's prediction error. While MSE is sensitive to
-    outliers, emphasizing larger errors, RMSE provides a more interpretable measure of average prediction error given
-    that it's in the same unit as the dependent variable.
-
-    **Limitations**:
-    - The applicability of this test is limited to regression tasks, specifically OLS models.
-    - The test operates under the assumption that the test dataset is a representative sample of the population. This
-    might not always hold true and can result in less accurate insights.
-    - The interpretability and the objectivity of the output (MSE and RMSE) can be influenced when the scale of the
-    dependent variable varies significantly, or the distribution of residuals is heavily skewed or contains outliers.
-    """
-
-    name = "regression_outsample_performance"
-    required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["model_comparison"],
-    }
-
-    def run(self):
-        # Check models list is not empty
-        if not self.inputs.models:
-            raise ValueError("List of models must be provided in the models parameter")
-        all_models = []
-        if self.inputs.model is not None:
-            all_models.append(self.inputs.model)
-
-        if self.inputs.models is not None:
-            all_models.extend(self.inputs.models)
-
-        for model in all_models:
-            if model.test_ds is None:
-                raise ValueError(
-                    "Test dataset is missing in the ValidMind Model object"
-                )
-
-        results = self._out_sample_performance_ols(all_models, self.inputs.dataset)
-        return self.cache_results(
-            {
-                "out_sample_performance": results.to_dict(orient="records"),
-            }
-        )
-
-    def _out_sample_performance_ols(self, model_list, dataset):
-        """
-        Returns the out-of-sample performance evaluation metrics of a list of OLS regression models.
-        Args:
-        model_list (list): A list of OLS models to evaluate.
-        test_data (pandas.DataFrame): The test dataset containing the independent and dependent variables.
-        target_col (str): The name of the target variable column in the test dataset.
-        Returns:
-        pandas.DataFrame: A DataFrame containing the evaluation results of the OLS models. The columns are 'Model',
-        'MSE' (Mean Squared Error), and 'RMSE' (Root Mean Squared Error).
-        """
-
-        # Initialize a list to store results
-        results = []
-
-        for fitted_model in model_list:
-            # Extract the column names of the independent variables from the model
-            independent_vars = dataset.feature_columns
-
-            # Separate the target variable and features in the test dataset
-            y_test = dataset.y
-
-            # Predict the test data
-            y_pred = dataset.y_pred(fitted_model)
-
-            # Calculate the residuals
-            residuals = y_test - y_pred
-
-            # Calculate the mean squared error and root mean squared error
-            mse = np.mean(residuals**2)
-            rmse_val = np.sqrt(mse)
-
-            # Store the results
-            model_name_with_vars = f"({', '.join(independent_vars)})"
-            results.append(
-                {
-                    "Model": model_name_with_vars,
-                    "MSE": mse,
-                    "RMSE": rmse_val,
-                }
-            )
-
-        # Create a DataFrame to display the results
-        results_df = pd.DataFrame(results)
-
-        return results_df
-
-    def summary(self, metric_value):
-        """
-        Build one table for summarizing the out-of-sample performance results
-        """
-        summary_out_sample_performance = metric_value["out_sample_performance"]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=summary_out_sample_performance,
-                    metadata=ResultTableMetadata(
-                        title="Out-of-Sample Performance Results"
-                    ),
-                ),
-            ]
-        )

validmind/tests/model_validation/statsmodels/RegressionModelsPerformance.py

@@ -1,127 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-from sklearn.metrics import mean_squared_error, r2_score
-
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
-
-
-@dataclass
-class RegressionModelsPerformance(Metric):
-    """
-    Evaluates and compares regression models' performance using R-squared, Adjusted R-squared, and MSE metrics.
-
-    **Purpose**: This metric is used to evaluate and compare the performance of various regression models. Through the
-    use of key statistical measures such as R-squared, Adjusted R-squared, and Mean Squared Error (MSE), the
-    performance of different models in predicting dependent variables can be assessed both on the data used for
-    training (in-sample) and new, unseen data (out-of-sample).
-
-    **Test Mechanism**: The test evaluates a list of provided regression models. For each model, it calculates their
-    in-sample and out-of-sample performance by deriving the model predictions for the training and testing datasets
-    respectively, and then comparing these predictions to the actual values. In doing so, it calculates R-squared,
-    Adjusted R-squared, and MSE for each model, stores the results, and returns them for comparison.
-
-    **Signs of High Risk**:
-    - High Mean Squared Error (MSE) values.
-    - Strikingly low values of R-squared and Adjusted R-squared.
-    - A significant drop in performance when transitioning from in-sample to out-of-sample evaluations, signaling a
-    potential overfitting issue.
-
-    **Strengths**:
-    - The test permits comparisons of multiple models simultaneously, providing an objective base for identifying the
-    top-performing model.
-    - It delivers both in-sample and out-of-sample evaluations, presenting performance data on unseen data.
-    - The utilization of R-squared and Adjusted R-squared in conjunction with MSE allows for a detailed view of the
-    model's explainability and error rate.
-
-    **Limitations**:
-    - This test is built around the assumption that the residuals of the regression model are normally distributed,
-    which is a fundamental requirement for Ordinary Least Squares (OLS) regression; thus, it could be not suitable for
-    models where this assumption is broken.
-    - The test does not consider cases where higher R-squared or lower MSE values do not necessarily correlate with
-    better predictive performance, particularly in instances of excessively complex models.
-    """
-
-    name = "regression_models_performance"
-    required_inputs = ["models", "in_sample_datasets", "out_of_sample_datasets"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["model_performance", "model_comparison"],
-    }
-
-    def run(self):
-        # Check models list is not empty
-        if not self.inputs.models:
-            raise ValueError("List of models must be provided in the models parameter")
-
-        all_models = []
-
-        if self.inputs.models is not None:
-            all_models.extend(self.inputs.models)
-
-        in_sample_results = self.sample_performance_ols(
-            self.inputs.models, self.inputs.in_sample_datasets
-        )
-        out_of_sample_results = self.sample_performance_ols(
-            self.inputs.models, self.inputs.out_of_sample_datasets
-        )
-
-        return self.cache_results(
-            {
-                "in_sample_performance": in_sample_results,
-                "out_of_sample_performance": out_of_sample_results,
-            }
-        )
-
-    def sample_performance_ols(self, models, datasets):
-        evaluation_results = []
-
-        for model, dataset in zip(models, datasets):
-            X_columns = dataset.feature_columns
-            y_true = dataset.y
-            y_pred = dataset.y_pred(model)
-
-            # Extract R-squared and Adjusted R-squared
-            r2 = r2_score(y_true, y_pred)
-            mse = mean_squared_error(y_true, y_pred)
-            adj_r2 = 1 - ((1 - r2) * (len(y_true) - 1)) / (
-                len(y_true) - len(X_columns) - 1
-            )
-
-            # Append the results to the evaluation_results list
-            evaluation_results.append(
-                {
-                    "Model": model.input_id,
-                    "Independent Variables": X_columns,
-                    "R-Squared": r2,
-                    "Adjusted R-Squared": adj_r2,
-                    "MSE": mse,
-                }
-            )
-
-        return evaluation_results
-
-    def summary(self, metric_value):
-        """
-        Build a table for summarizing the in-sample and out-of-sample performance results
-        """
-        summary_in_sample_performance = metric_value["in_sample_performance"]
-        summary_out_of_sample_performance = metric_value["out_of_sample_performance"]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=summary_in_sample_performance,
-                    metadata=ResultTableMetadata(title="In-Sample Performance Results"),
-                ),
-                ResultTable(
-                    data=summary_out_of_sample_performance,
-                    metadata=ResultTableMetadata(
-                        title="Out-of-Sample Performance Results"
-                    ),
-                ),
-            ]
-        )

validmind/tests/model_validation/statsmodels/ResidualsVisualInspection.py

@@ -1,130 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-import matplotlib.pyplot as plt
-import pandas as pd
-import seaborn as sns
-from scipy import stats
-from statsmodels.graphics.tsaplots import plot_acf
-from statsmodels.tsa.seasonal import seasonal_decompose
-
-from validmind.vm_models import Figure, Metric
-
-
-@dataclass
-class ResidualsVisualInspection(Metric):
-    """
-    Provides a comprehensive visual analysis of residuals for regression models utilizing various plot types.
-
-    **Purpose**: The main purpose of this metric is to visualize and analyze the residuals (the differences between the
-    observed and predicted values) of a regression problem. It allows for a graphical exploration of the model's
-    errors, helping to identify statistical patterns or anomalies that may indicate a systematic bias in the model's
-    predictions. By inspecting the residuals, we can check how well the model fits the data and meets the assumptions
-    of the model.
-
-    **Test Mechanism**: The metric generates four common types of residual plots which are: a histogram with kernel
-    density estimation, a quantile-quantile (Q-Q) plot, a residuals series dot plot, and an autocorrelation function
-    (ACF) plot.
-
-    - The residuals histogram with kernel density estimation visualizes the distribution of residuals and allows to
-    check if they are normally distributed.
-    - Q-Q plot compares the observed quantiles of the data to the quantiles of a standard normal distribution, helping
-    to assess the normality of residuals.
-    - A residuals dot plot indicates the variation in residuals over time, which helps in identifying any time-related
-    pattern in residuals.
-    - ACF plot visualizes the correlation of an observation with its previous observations, helping to pinpoint any
-    seasonality effect within residuals.
-
-    **Signs of High Risk**:
-
-    - Skewness or asymmetry in the histogram or a significant deviation from the straight line in the Q-Q plot, which
-    indicates that the residuals aren't normally distributed.
-    - Large spikes in the ACF plot, indicating that the residuals are correlated, in violation of the assumption that
-    they are independent.
-    - Non-random patterns in the dot plot of residuals, indicating potential model misspecification.
-
-    **Strengths**:
-
-    - Visual analysis of residuals is a powerful yet simple way to understand a model’s behavior across the data set
-    and to identify problems with the model's assumptions or its fit to the data.
-    - The test is applicable to any regression model, irrespective of complexity.
-    - By exploring residuals, we might uncover relationships that were not captured by the model, revealing
-    opportunities for model improvement.
-
-    **Limitations**:
-
-    - Visual tests are largely subjective and can be open to interpretation. Clear-cut decisions about the model based
-    solely on these plots may not be possible.
-    - The metrics from the test do not directly infer the action based on the results; domain-specific knowledge and
-    expert judgement is often required to interpret the results.
-    - These plots can indicate a problem with the model but they do not necessarily reveal the nature or cause of the
-    problem.
-    - The test assumes that the error terms are identically distributed, which might not always be the case in
-    real-world scenarios.
-    """
-
-    name = "residuals_visual_inspection"
-    required_inputs = ["dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": ["statsmodels", "visualization"],
-    }
-
-    @staticmethod
-    def residual_analysis(residuals, variable_name, axes):
-        residuals = residuals.dropna().reset_index(
-            drop=True
-        )  # drop NaN values and reset index
-
-        # QQ plot
-        stats.probplot(residuals, dist="norm", plot=axes[0, 1])
-        axes[0, 1].set_title(f"Residuals Q-Q Plot ({variable_name})")
-
-        # Histogram with KDE
-        sns.histplot(residuals, kde=True, ax=axes[0, 0])
-        axes[0, 0].set_xlabel("Residuals")
-        axes[0, 0].set_title(f"Residuals Histogram ({variable_name})")
-
-        # Residual series dot plot
-        sns.lineplot(data=residuals, linewidth=0.5, color="red", ax=axes[1, 0])
-        axes[1, 0].set_title(f"Residual Series Dot Plot ({variable_name})")
-
-        # ACF plot
-        n_lags = min(100, len(residuals) - 1)  # Adjust the number of lags
-        plot_acf(residuals, ax=axes[1, 1], lags=n_lags, zero=False)  # Added zero=False
-        axes[1, 1].set_title(f"ACF Plot of Residuals ({variable_name})")
-
-    def run(self):
-        x_train = self.inputs.dataset.df
-        figures = []
-
-        # TODO: specify which columns to plot via params
-        for col in x_train.columns:
-            sd = seasonal_decompose(x_train[col], model="additive")
-
-            # Remove NaN values from the residuals and reset the index
-            residuals = pd.Series(sd.resid).dropna().reset_index(drop=True)
-
-            # Create subplots
-            fig, axes = plt.subplots(nrows=2, ncols=2)
-            fig.suptitle(f"Residuals Inspection for {col}", fontsize=24)
-
-            self.residual_analysis(residuals, col, axes)
-
-            # Adjust the layout
-            plt.tight_layout()
-
-            # Do this if you want to prevent the figure from being displayed
-            plt.close("all")
-
-            figures.append(
-                Figure(
-                    for_object=self,
-                    key=self.key,
-                    figure=fig,
-                )
-            )
-        return self.cache_results(figures=figures)