validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py

@@ -2,197 +2,93 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+def integrate_diff(series_diff, start_value):
+    series_diff = np.array(series_diff)
+    series_orig = np.cumsum(series_diff)
+    series_orig += start_value
 
+    return series_orig
 
-@dataclass
-class RegressionModelForecastPlotLevels(Metric):
+
+@tags("time_series_data", "forecasting", "visualization")
+@tasks("regression")
+def RegressionModelForecastPlotLevels(
+    model: VMModel,
+    dataset: VMDataset,
+):
     """
-    Assesses the alignment between forecasted and observed values in regression models through visual plots, including
-    handling data transformations.
+    Assesses the alignment between forecasted and observed values in regression models through visual plots
 
     ### Purpose
 
-    The `RegressionModelForecastPlotLevels` test aims to visually assess the performance of a series of regression
-    models by comparing their forecasted values against the actual observed values in both training and test datasets.
-    This test helps determine the accuracy of the models and can handle specific data transformations before making the
-    comparison, providing a comprehensive evaluation of model performance.
+    This test aims to visually assess the performance of a regression model by comparing its forecasted values against
+    the actual observed values for both the raw and transformed (integrated) data. This helps determine the accuracy
+    of the model and can help identify overfitting or underfitting. The integration is applied to highlight the trend
+    rather than the absolute level.
 
     ### Test Mechanism
 
-    The test mechanism involves initializing the `RegressionModelForecastPlotLevels` class with an optional
-    `transformation` parameter. The class then:
+    This test generates two plots:
+
+    - Raw data vs forecast
+    - Transformed data vs forecast
 
-    - Checks for the presence of model objects and raises a `ValueError` if none are found.
-    - Processes each model to generate predictive forecasts for both training and testing datasets.
-    - Contrasts these forecasts with the actual observed values.
-    - Produces plots to visually compare forecasted and observed values for both raw and transformed datasets.
-    - Handles specified transformations (e.g., "integrate") by performing cumulative sums to create a new series before
-    plotting.
+    The transformed data is created by performing a cumulative sum on the raw data.
 
     ### Signs of High Risk
 
-    - Significant deviation between forecasted and observed values in training or testing datasets.
+    - Significant deviation between forecasted and observed values.
     - Patterns suggesting overfitting or underfitting.
     - Large discrepancies in the plotted forecasts, indicating potential issues with model generalizability and
     precision.
 
     ### Strengths
 
-    - **Visual Evaluations**: Provides an intuitive, visual way to assess multiple regression models, aiding in easier
-    interpretation and evaluation of forecast accuracy.
-    - **Transformation Handling**: Can process specified data transformations such as "integrate," enhancing
-    flexibility.
-    - **Detailed Perspective**: Assesses performance on both training and testing datasets, offering a comprehensive
-    view of model behavior.
+    - Provides an intuitive, visual way to assess multiple regression models, aiding in easier interpretation and
+    evaluation of forecast accuracy.
 
     ### Limitations
 
-    - **Subjectivity**: Relies heavily on visual interpretation, which may vary between individuals.
-    - **Limited Transformation Capability**: Supports only the "integrate" transformation; other complex
-    transformations might not be handled.
-    - **Overhead**: Plotting can be computationally intensive for large datasets, increasing runtime.
-    - **Numerical Measurement**: Does not provide a numerical metric to quantify forecast accuracy, relying solely on
-    visual assessment.
+    - Relies heavily on visual interpretation, which may vary between individuals.
+    - Does not provide a numerical metric to quantify forecast accuracy, relying solely on visual assessment.
     """
+    index = dataset.df.index
+
+    if not pd.api.types.is_datetime64_any_dtype(index):
+        raise ValueError("Test requires a time series dataset")
+
+    fig, axs = plt.subplots(2, 1)
+
+    y_pred = dataset.y_pred(model)
+
+    # raw data vs forecast
+    axs[0].plot(index, dataset.y, label="Observed", color="grey")
+    axs[0].plot(index, y_pred, label="Forecast")
+    axs[0].set_title("Forecast vs Observed")
+    axs[0].legend()
+
+    # transformed data
+    dataset_y_transformed = integrate_diff(dataset.y, start_value=dataset.y[0])
+    y_pred_transformed = integrate_diff(y_pred, start_value=dataset_y_transformed[0])
+
+    axs[1].plot(
+        index,
+        dataset_y_transformed,
+        label="Observed",
+        color="grey",
+    )
+    axs[1].plot(index, y_pred_transformed, label="Forecast")
+    axs[1].set_title("Integrated Forecast vs Observed")
+    axs[1].legend()
+
+    plt.close()
 
-    name = "regression_forecast_plot_levels"
-    required_inputs = ["models", "datasets"]
-    default_params = {
-        "transformation": None,
-    }
-    tasks = ["regression"]
-    tags = ["forecasting", "visualization"]
-
-    def run(self):
-        transformation = self.params["transformation"]
-
-        if not self.inputs.models:
-            raise ValueError("List of models must be provided in the models parameter")
-
-        all_models = []
-        for model in self.inputs.models:
-            all_models.append(model)
-
-        figures = self._plot_forecast(all_models, self.inputs.datasets, transformation)
-
-        return self.cache_results(figures=figures)
-
-    def integrate_diff(self, series_diff, start_value):
-        series_diff = np.array(series_diff)
-        series_orig = np.cumsum(series_diff)
-        series_orig += start_value
-        return series_orig
-
-    def _plot_forecast(
-        self,
-        model_list,
-        datasets,
-        transformation=None,
-    ):
-        figures = []
-
-        for i, fitted_model in enumerate(model_list):
-            feature_columns = datasets[0].feature_columns
-
-            train_ds = datasets[0]
-            test_ds = datasets[1]
-
-            y_pred = train_ds.y_pred(fitted_model)
-            y_pred_test = test_ds.y_pred(fitted_model)
-
-            all_dates = pd.concat([pd.Series(train_ds.index), pd.Series(test_ds.index)])
-
-            if all_dates.empty:
-                raise ValueError(
-                    "No dates in the data. Unable to determine start and end dates."
-                )
-
-            fig, axs = plt.subplots(2, 2)
-
-            # train vs forecast
-            axs[0, 0].plot(
-                train_ds.index, train_ds.y, label="Train Dataset", color="grey"
-            )
-            axs[0, 0].plot(train_ds.index, y_pred, label="Train Forecast")
-            axs[0, 0].set_title(f"Forecast vs Observed for features {feature_columns}")
-            axs[0, 0].legend()
-
-            # test vs forecast
-            axs[0, 1].plot(test_ds.index, test_ds.y, label="Test Dataset", color="grey")
-            axs[0, 1].plot(test_ds.index, y_pred_test, label="Test Forecast")
-            axs[0, 1].set_title(f"Forecast vs Observed for features {feature_columns}")
-            axs[0, 1].legend()
-
-            if transformation == "integrate":
-                train_ds_y_transformed = self.integrate_diff(
-                    train_ds.y_df().values, start_value=train_ds.y[0]
-                )
-
-                test_ds_y_transformed = self.integrate_diff(
-                    test_ds.y_df().values, start_value=test_ds.y[0]
-                )
-
-                # Use the first value of the transformed train dataset as the start_value for predicted datasets
-
-                y_pred_transformed = self.integrate_diff(
-                    y_pred, start_value=train_ds_y_transformed[0]
-                )
-                y_pred_test_transformed = self.integrate_diff(
-                    y_pred_test, start_value=test_ds_y_transformed[0]
-                )
-
-                # Create copies of the original datasets and update them to reflect transformed data
-                train_ds_transformed = train_ds.copy
-                train_ds_transformed["y"] = train_ds_y_transformed
-
-                test_ds_transformed = test_ds.copy
-                test_ds_transformed["y"] = test_ds_y_transformed
-
-                # transformed train vs forecast
-                axs[1, 0].plot(
-                    train_ds.index,
-                    train_ds_y_transformed,
-                    label="Train Dataset",
-                    color="grey",
-                )
-
-                axs[1, 0].plot(
-                    train_ds.index, y_pred_transformed, label="Train Forecast"
-                )
-
-                axs[1, 0].set_title(
-                    f"Integrated Forecast vs Observed for features {feature_columns}"
-                )
-                axs[1, 0].legend()
-
-                # transformed test vs forecast
-                axs[1, 1].plot(
-                    test_ds.index,
-                    test_ds_y_transformed,
-                    label="Test Dataset",
-                    color="grey",
-                )
-
-                axs[1, 1].plot(
-                    test_ds.index, y_pred_test_transformed, label="Test Forecast"
-                )
-                axs[1, 1].set_title(
-                    f"Integrated Forecast vs Observed for features {feature_columns}"
-                )
-                axs[1, 1].legend()
-
-            figures.append(
-                Figure(for_object=self, key=f"{self.key}:{i}", figure=fig, metadata={})
-            )
-
-            # Close the figure to prevent it from displaying
-            plt.close(fig)
-
-        return figures
+    return fig

validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py

@@ -2,19 +2,34 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
+from typing import List, Union
 
 import matplotlib.pyplot as plt
 import numpy as np
 
+from validmind import tags, tasks
 from validmind.logging import get_logger
-from validmind.vm_models import Figure, Metric
+from validmind.vm_models import VMDataset, VMModel
 
 logger = get_logger(__name__)
 
 
-@dataclass
-class RegressionModelSensitivityPlot(Metric):
+def integrate_diff(series_diff, start_value):
+    series_diff = np.asarray(series_diff, dtype=np.float64)  # Convert to float64
+    series = np.cumsum(series_diff)
+    series += start_value
+
+    return series
+
+
+@tags("senstivity_analysis", "visualization")
+@tasks("regression")
+def RegressionModelSensitivityPlot(
+    dataset: VMDataset,
+    model: VMModel,
+    shocks: List[float] = [0.1],
+    transformation: Union[str, None] = None,
+):
     """
     Assesses the sensitivity of a regression model to changes in independent variables by applying shocks and
     visualizing the impact.
@@ -57,120 +72,43 @@ class RegressionModelSensitivityPlot(Metric):
     - Provides a visual representation without a numerical risk measure, potentially introducing subjectivity in
     interpretation.
     """
-
-    name = "regression_sensitivity_plot"
-    required_inputs = ["models", "datasets"]
-    default_params = {
-        "transformation": None,
-        "shocks": [0.1],
+    features_df = dataset.x_df()
+    target_df = dataset.y_df()
+
+    shocked_dfs = {"Baseline": features_df}
+    for shock in shocks:
+        for col in dataset.feature_columns:
+            temp_df = features_df.copy()
+            temp_df[col] = temp_df[col] * (1 + shock)
+            shocked_dfs[f"Shock of {shock} to {col}"] = temp_df
+
+    predictions = {
+        label: model.predict(shocked_df) for label, shocked_df in shocked_dfs.items()
     }
-    tasks = ["regression"]
-    tags = ["senstivity_analysis", "visualization"]
 
-    def run(self):
-        logger.info(self.params)
-
-        transformation = self.params["transformation"]
-        shocks = self.params["shocks"]
-
-        if not self.inputs.models:
-            raise ValueError("List of models must be provided in the models parameter")
-
-        all_models = []
-        for model in self.inputs.models:
-            all_models.append(model)
-
-        figures = []
-        for i, model in enumerate(all_models):
-            features_df = self.inputs.datasets[1].x_df()
-            target_df = self.inputs.datasets[1].y_df()  # series
-
-            shocked_datasets = self.apply_shock(features_df, shocks)
-
-            predictions = self.predict_shocked_datasets(shocked_datasets, model)
-
-            if transformation == "integrate":
-                transformed_predictions = []
-                start_value = self.inputs.datasets[0].y[0]
-                transformed_target = self.integrate_diff(
-                    self.inputs.datasets[1].y_df().values, start_value
-                )
-
-                predictions = self.predict_shocked_datasets(shocked_datasets, model)
-                transformed_predictions = self.transform_predictions(
-                    predictions, start_value
-                )
-
-            else:
-                transformed_target = target_df.values
-                transformed_predictions = predictions
-
-            fig = self._plot_predictions(
-                target_df.index, transformed_target, transformed_predictions
-            )
-            figures.append(
-                Figure(for_object=self, key=f"{self.key}:{i}", figure=fig, metadata={})
-            )
-        return self.cache_results(figures=figures)
-
-    def transform_predictions(self, predictions, start_value):
-        transformed_predictions = (
-            {}
-        )  # Initialize an empty dictionary to store the transformed predictions
-
-        for (
-            label,
-            pred,
-        ) in predictions.items():  # Here, label is the key, pred is the value
-            transformed_pred = self.integrate_diff(pred, start_value)
-            transformed_predictions[
-                label
-            ] = transformed_pred  # Store transformed dataframe in the new dictionary
-
-        return transformed_predictions
-
-    def predict_shocked_datasets(self, shocked_datasets, model):
-        predictions = {}
-
-        for label, shocked_dataset in shocked_datasets.items():
-            y_pred = model.predict(shocked_dataset)
-            predictions[label] = y_pred
-
-        return predictions
-
-    def _plot_predictions(self, index, target, predictions):
-        fig = plt.figure()
-
-        # Plot the target
-        plt.plot(index, target, label="Observed")
-
-        # Plot each prediction
-        for label, pred in predictions.items():
-            plt.plot(index, pred, label=label)
-
-        plt.legend()
-
-        # Close the figure to prevent it from displaying
-        plt.close(fig)
-        return fig
-
-    def integrate_diff(self, series_diff, start_value):
-        series_diff = np.asarray(series_diff, dtype=np.float64)  # Convert to float64
-        series = np.cumsum(series_diff)
-        series += start_value
-        return series
-
-    def apply_shock(self, df, shocks):
-        shocked_dfs = {"Baseline": df.copy()}  # Start with the original dataset
-        cols_to_shock = df.columns  # All columns
-
-        # Apply shock one variable at a time
-        for shock in shocks:
-            for col in cols_to_shock:
-                temp_df = df.copy()
-                temp_df[col] = temp_df[col] * (1 + shock)
-                shocked_dfs[
-                    f"Shock of {shock} to {col}"
-                ] = temp_df  # Include shock value in the key
-
-        return shocked_dfs
+    if transformation is None:
+        transformed_target = target_df.values
+        transformed_predictions = predictions
+
+    elif transformation == "integrate":
+        transformed_target = integrate_diff(target_df.values, dataset.y[0])
+        transformed_predictions = {
+            label: integrate_diff(pred, dataset.y[0])
+            for label, pred in predictions.items()
+        }
+
+    else:
+        raise ValueError(f"Invalid transformation: {transformation}")
+
+    fig = plt.figure()
+
+    plt.plot(target_df.index, transformed_target, label="Observed")
+
+    for label, pred in transformed_predictions.items():
+        plt.plot(target_df.index, pred, label=label)
+
+    plt.legend()
+
+    plt.close()
+
+    return fig

validmind/tests/model_validation/statsmodels/RegressionModelSummary.py

@@ -2,18 +2,17 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
-import pandas as pd
 from sklearn.metrics import mean_squared_error, r2_score
 
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 from .statsutils import adj_r2_score
 
 
-@dataclass
-class RegressionModelSummary(Metric):
+@tags("model_performance", "regression")
+@tasks("regression")
+def RegressionModelSummary(dataset: VMDataset, model: VMModel):
     """
     Evaluates regression model performance using metrics including R-Squared, Adjusted R-Squared, MSE, and RMSE.
 
@@ -26,11 +25,8 @@ class RegressionModelSummary(Metric):
 
     ### Test Mechanism
 
-    This test employs the 'train_ds' attribute of the model to gather and analyze the training data. Initially, it
-    fetches the independent variables and uses the model to make predictions on these given features. Subsequently, it
-    calculates several standard regression performance metrics including R-Squared, Adjusted R-Squared, Mean Squared
-    Error (MSE), and Root Mean Squared Error (RMSE), which quantify the approximation of the predicted responses to the
-    actual responses.
+    This test uses the sklearn library to calculate the R-Squared, Adjusted R-Squared, MSE, and RMSE. It outputs a
+    table with the results of these metrics along with the feature columns used by the model.
 
     ### Signs of High Risk
 
@@ -45,54 +41,21 @@ class RegressionModelSummary(Metric):
 
     ### Limitations
 
-    - Applicable exclusively to regression models.
     - RMSE and MSE might be sensitive to outliers.
     - A high R-Squared or Adjusted R-Squared may not necessarily indicate a good model, especially in cases of
     overfitting.
     """
-
-    name = "regression_model_summary"
-    required_inputs = ["model", "dataset"]
-    tasks = ["regression"]
-    tags = ["model_metadata", "model_comparison"]
-
-    def run(self):
-        X_columns = self.inputs.dataset.feature_columns
-
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(self.inputs.model)
-
-        r2 = r2_score(y_true, y_pred)
-        adj_r2 = adj_r2_score(y_true, y_pred, len(y_true), len(X_columns))
-        mse = mean_squared_error(y_true=y_true, y_pred=y_pred, squared=True)
-        rmse = mean_squared_error(y_true=y_true, y_pred=y_pred, squared=False)
-
-        results = {
-            "Independent Variables": X_columns,
-            "R-Squared": r2,
-            "Adjusted R-Squared": adj_r2,
-            "MSE": mse,
-            "RMSE": rmse,
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    return [
+        {
+            "Independent Variables": dataset.feature_columns,
+            "R-Squared": r2_score(y_true, y_pred),
+            "Adjusted R-Squared": adj_r2_score(
+                y_true, y_pred, len(y_true), len(dataset.feature_columns)
+            ),
+            "MSE": mean_squared_error(y_true=y_true, y_pred=y_pred, squared=True),
+            "RMSE": mean_squared_error(y_true=y_true, y_pred=y_pred, squared=False),
         }
-        summary_regression = pd.DataFrame(results)
-
-        return self.cache_results(
-            {
-                "regression_analysis": summary_regression.to_dict(orient="records"),
-            }
-        )
-
-    def summary(self, metric_value):
-        """
-        Build one table for summarizing the regression analysis results
-        """
-        summary_regression = metric_value["regression_analysis"]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=summary_regression,
-                    metadata=ResultTableMetadata(title="Regression Analysis Results"),
-                ),
-            ]
-        )
+    ]