validmind 2.5.25__py3-none-any.whl → 2.6.8__py3-none-any.whl

validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py

@@ -2,16 +2,16 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 from statsmodels.stats.diagnostic import kstest_normal
 
+from validmind import tags, tasks
 from validmind.errors import InvalidTestParametersError
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
+from validmind.vm_models import VMDataset, VMModel
 
 
-@dataclass
-class KolmogorovSmirnov(Metric):
+@tags("tabular_data", "data_distribution", "statistical_test", "statsmodels")
+@tasks("classification", "regression")
+def KolmogorovSmirnov(model: VMModel, dataset: VMDataset, dist: str = "norm"):
     """
     Assesses whether each feature in the dataset aligns with a normal distribution using the Kolmogorov-Smirnov test.
 
@@ -47,48 +47,23 @@ class KolmogorovSmirnov(Metric):
     - Less effective for multivariate distributions, as it is designed for univariate distributions.
     - Does not identify specific types of non-normality, such as skewness or kurtosis, which could impact model fitting.
     """
+    if dist not in ["norm", "exp"]:
+        raise InvalidTestParametersError(
+            "'dist' parameter must be either 'norm' or 'exp'"
+        )
 
-    name = "kolmogorov_smirnov"
-    required_inputs = ["dataset"]
-    default_params = {"dist": "norm"}
-    tasks = ["classification", "regression"]
-    tags = [
-        "tabular_data",
-        "data_distribution",
-        "statistical_test",
-        "statsmodels",
-    ]
+    df = dataset.df[dataset.feature_columns_numeric]
 
-    def summary(self, metric_value):
-        results_table = metric_value["metrics_summary"]
-
-        results_table = [
-            {"Column": k, "stat": result["stat"], "pvalue": result["pvalue"]}
-            for k, result in results_table.items()
-        ]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=results_table,
-                    metadata=ResultTableMetadata(title="KS Test results"),
-                )
-            ]
-        )
+    ks_values = {}
+    for col in df.columns:
+        ks_stat, p_value = kstest_normal(df[col].values, dist)
+        ks_values[col] = {"stat": ks_stat, "pvalue": p_value}
 
-    def run(self):
-        """
-        Calculates KS for each of the dataset features
-        """
-        data_distribution = self.params["dist"]
-        if data_distribution not in ["norm" or "exp"]:
-            InvalidTestParametersError("Dist parameter must be either 'norm' or 'exp'")
-
-        x_train = self.inputs.dataset.df[self.inputs.dataset.feature_columns_numeric]
-        ks_values = {}
-        for col in x_train.columns:
-            ks_stat, p_value = kstest_normal(x_train[col].values, data_distribution)
-            ks_values[col] = {"stat": ks_stat, "pvalue": p_value}
-
-        print(ks_values)
-        return self.cache_results({"metrics_summary": ks_values})
+    return [
+        {
+            "Column": k,
+            "Statistic": result["stat"],
+            "P-Value": result["pvalue"],
+        }
+        for k, result in ks_values.items()
+    ]

validmind/tests/model_validation/statsmodels/Lilliefors.py

@@ -2,15 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 from statsmodels.stats.diagnostic import lilliefors
 
-from validmind.vm_models import Metric
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
-@dataclass
-class Lilliefors(Metric):
+@tags("tabular_data", "data_distribution", "statistical_test", "statsmodels")
+@tasks("classification", "regression")
+def Lilliefors(model: VMModel, dataset: VMDataset):
     """
     Assesses the normality of feature distributions in an ML model's training dataset using the Lilliefors test.
 
@@ -56,29 +56,18 @@ class Lilliefors(Metric):
     - Like any other statistical test, Lilliefors test may also produce false positives or negatives. Hence, banking
     solely on this test, without considering other characteristics of the data, may give rise to risks.
     """
+    df = dataset.df[dataset.feature_columns_numeric]
+
+    table = []
 
-    name = "lilliefors_test"
-    required_inputs = ["dataset"]
-    tasks = ["classification", "regression"]
-    tags = [
-        "tabular_data",
-        "data_distribution",
-        "statistical_test",
-        "statsmodels",
-    ]
-
-    def run(self):
-        """
-        Calculates Lilliefors test for each of the dataset features
-        """
-        x_train = self.inputs.dataset.df[self.inputs.dataset.feature_columns_numeric]
-
-        lilliefors_values = {}
-        for col in x_train.columns:
-            l_stat, p_value = lilliefors(x_train[col].values)
-            lilliefors_values[col] = {
-                "stat": l_stat,
-                "pvalue": p_value,
+    for col in df.columns:
+        l_stat, p_value = lilliefors(df[col].values)
+        table.append(
+            {
+                "Column": col,
+                "Statistic": l_stat,
+                "P-Value": p_value,
             }
+        )
 
-        return self.cache_results(lilliefors_values)
+    return table

validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py

@@ -2,36 +2,37 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import matplotlib.pyplot as plt
 import seaborn as sns
 
+from validmind import tags, tasks
 from validmind.errors import SkipTestError
 from validmind.logging import get_logger
-from validmind.vm_models import Figure, Metric
+from validmind.vm_models import VMModel
 
 logger = get_logger(__name__)
 
 
-@dataclass
-class RegressionFeatureSignificance(Metric):
+@tags("statistical_test", "model_interpretation", "visualization", "feature_importance")
+@tasks("regression")
+def RegressionFeatureSignificance(
+    model: VMModel, fontsize: int = 10, p_threshold: float = 0.05
+):
     """
-    Assesses and visualizes the statistical significance of features in a set of regression models.
+    Assesses and visualizes the statistical significance of features in a regression model.
 
     ### Purpose
 
     The Regression Feature Significance metric assesses the significance of each feature in a given set of regression
-    models. It creates a visualization displaying p-values for every feature of each model, assisting model developers
-    in understanding which features are most influential in their models.
+    model. It creates a visualization displaying p-values for every feature of the model, assisting model developers
+    in understanding which features are most influential in their model.
 
     ### Test Mechanism
 
-    The test mechanism involves going through each fitted regression model in a given list, extracting the model
-    coefficients and p-values for each feature, and then plotting these values. The x-axis on the plot contains the
-    p-values while the y-axis denotes the coefficients of each feature. A vertical red line is drawn at the threshold
-    for p-value significance, which is 0.05 by default. Any features with p-values to the left of this line are
-    considered statistically significant at the chosen level.
+    The test mechanism involves extracting the model's coefficients and p-values for each feature, and then plotting these
+    values. The x-axis on the plot contains the p-values while the y-axis denotes the coefficients of each feature. A
+    vertical red line is drawn at the threshold for p-value significance, which is 0.05 by default. Any features with
+    p-values to the left of this line are considered statistically significant at the chosen level.
 
     ### Signs of High Risk
 
@@ -45,7 +46,6 @@ class RegressionFeatureSignificance(Metric):
     - Helps identify the features that significantly contribute to a model's prediction, providing insights into the
     feature importance.
     - Provides tangible, easy-to-understand visualizations to interpret the feature significance.
-    - Facilitates comparison of feature importance across multiple models.
 
     ### Limitations
 
@@ -57,81 +57,37 @@ class RegressionFeatureSignificance(Metric):
     - P-value thresholds are somewhat arbitrary and do not always indicate practical significance, only statistical
     significance.
     """
+    if model.library != "statsmodels":
+        raise SkipTestError("Only statsmodels are supported for this metric")
 
-    name = "regression_feature_significance"
-    required_inputs = ["model"]
-
-    default_params = {"fontsize": 10, "p_threshold": 0.05}
-    tasks = ["regression"]
-    tags = [
-        "statistical_test",
-        "model_interpretation",
-        "visualization",
-        "feature_importance",
-    ]
-
-    def run(self):
-        fontsize = self.params["fontsize"]
-        p_threshold = self.params["p_threshold"]
-
-        # Check models list is not empty
-        if not self.inputs.model:
-            raise ValueError("Model must be provided in the models parameter")
-
-        figures = self._plot_pvalues(self.inputs.model, fontsize, p_threshold)
-
-        return self.cache_results(figures=figures)
-
-    def _plot_pvalues(self, model_list, fontsize, p_threshold):
-        # Initialize a list to store figures
-        figures = []
-
-        for i, model in enumerate(model_list):
-
-            if model.library != "statsmodels":
-                raise SkipTestError("Only statsmodels are supported for this metric")
+    coefficients = model.model.params
+    pvalues = model.model.pvalues
 
-            # Get the coefficients and p-values from the model
-            coefficients = model.model.params
-            pvalues = model.model.pvalues
+    # Sort the variables by p-value in ascending order
+    sorted_idx = pvalues.argsort()
+    coefficients = coefficients.iloc[sorted_idx]
+    pvalues = pvalues.iloc[sorted_idx]
 
-            # Sort the variables by p-value in ascending order
-            sorted_idx = pvalues.argsort()
-            coefficients = coefficients.iloc[sorted_idx]
-            pvalues = pvalues.iloc[sorted_idx]
+    fig, ax = plt.subplots()
 
-            # Increase the height of the figure
-            fig, ax = plt.subplots()
+    sns.barplot(x=pvalues, y=coefficients.index, ax=ax, color="skyblue")
 
-            # Create a horizontal bar plot with wider bars using Seaborn
-            sns.barplot(x=pvalues, y=coefficients.index, ax=ax, color="skyblue")
+    # Add a threshold line at p-value = p_threshold
+    threshold_line = ax.axvline(x=p_threshold, color="red", linestyle="--")
 
-            # Add a threshold line at p-value = p_threshold
-            threshold_line = ax.axvline(x=p_threshold, color="red", linestyle="--")
+    # Set labels and title
+    ax.set_xlabel("P-value")
+    ax.set_ylabel(None)
+    ax.set_title(f"Feature Significance for {model.input_id}")
 
-            # Set labels and title
-            ax.set_xlabel("P-value")
-            ax.set_ylabel(None)
-            ax.set_title(f"Feature Significance for Model {i + 1}")
+    plt.tight_layout()
 
-            # Adjust the layout to prevent overlapping of variable names
-            plt.tight_layout()
+    ax.set_yticklabels(ax.get_yticklabels(), fontsize=fontsize)
 
-            # Set the fontsize of y-axis tick labels
-            ax.set_yticklabels(ax.get_yticklabels(), fontsize=fontsize)
+    # Add a legend for the threshold line
+    legend_label = f"p_threshold {p_threshold}"
+    ax.legend([threshold_line], [legend_label])
 
-            # Add a legend for the threshold line
-            legend_label = f"p_threshold {p_threshold}"
-            ax.legend([threshold_line], [legend_label])
+    plt.close()
 
-            # Add to the figures list
-            figures.append(
-                Figure(
-                    for_object=self,
-                    key=f"{self.key}:{i}",
-                    figure=fig,
-                    metadata={"model": str(model.model)},
-                )
-            )
-            plt.close("all")
-        return figures
+    return fig

validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py

@@ -2,38 +2,43 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
+from typing import Union
 
 import matplotlib.pyplot as plt
 import pandas as pd
 
+from validmind import tags, tasks
 from validmind.logging import get_logger
-from validmind.vm_models import Figure, Metric
+from validmind.vm_models import VMDataset, VMModel
 
 logger = get_logger(__name__)
 
 
-@dataclass
-class RegressionModelForecastPlot(Metric):
+@tags("time_series_data", "forecasting", "visualization")
+@tasks("regression")
+def RegressionModelForecastPlot(
+    model: VMModel,
+    dataset: VMDataset,
+    start_date: Union[str, None] = None,
+    end_date: Union[str, None] = None,
+):
     """
-    Generates plots to visually compare the forecasted outcomes of one or more regression models against actual
-    observed values over a specified date range.
+    Generates plots to visually compare the forecasted outcomes of a regression model against actual observed values over
+    a specified date range.
 
     ### Purpose
 
-    The "regression_forecast_plot" is intended to visually depict the performance of one or more regression models by
-    comparing the model's forecasted outcomes against actual observed values within a specified date range. This metric
-    is especially useful in time-series models or any model where the outcome changes over time, allowing direct
-    comparison of predicted vs actual values.
+    This metric is useful for time-series models or any model where the outcome changes over time, allowing direct
+    comparison of predicted vs actual values. It can help identify overfitting or underfitting situations as well as
+    general model performance.
 
     ### Test Mechanism
 
-    This test generates a plot for each fitted model in the list. The x-axis represents the date ranging from the
-    specified "start_date" to the "end_date", while the y-axis shows the value of the outcome variable. Two lines are
-    plotted: one representing the forecasted values and the other representing the observed values. The "start_date"
-    and "end_date" can be parameters of this test; if these parameters are not provided, they are set to the minimum
-    and maximum date available in the dataset. The test verifies that the provided date range is within the limits of
-    the available data.
+    This test generates a plot with the x-axis representing the date ranging from the specified "start_date" to the
+    "end_date", while the y-axis shows the value of the outcome variable. Two lines are plotted: one representing the
+    forecasted values and the other representing the observed values. The "start_date" and "end_date" can be parameters
+    of this test; if these parameters are not provided, they are set to the minimum and maximum date available in the
+    dataset.
 
     ### Signs of High Risk
 
@@ -58,101 +63,28 @@ class RegressionModelForecastPlot(Metric):
     - Inapplicability: Limited to cases where the order of data points (time-series) matters, it might not be of much
     use in problems that are not related to time series prediction.
     """
+    index = dataset.df.index
 
-    name = "regression_forecast_plot"
-    required_inputs = ["models", "datasets"]
-    default_params = {"start_date": None, "end_date": None}
-    tasks = ["regression"]
-    tags = ["forecasting", "visualization"]
-
-    def run(self):
-        start_date = self.params["start_date"]
-        end_date = self.params["end_date"]
-
-        # Check models list is not empty
-        if not self.inputs.models:
-            raise ValueError("List of models must be provided in the models parameter")
-        all_models = []
-        for model in self.inputs.models:
-            all_models.append(model)
-
-        figures = self._plot_forecast(
-            all_models, self.inputs.datasets, start_date, end_date
-        )
-
-        return self.cache_results(figures=figures)
-
-    def _plot_forecast(self, model_list, datasets, start_date=None, end_date=None):
-        # Convert start_date and end_date to pandas Timestamp for comparison
-        start_date = pd.Timestamp(start_date)
-        end_date = pd.Timestamp(end_date)
-
-        # Initialize a list to store figures
-        figures = []
-
-        for i, fitted_model in enumerate(model_list):
-            feature_columns = datasets[0].feature_columns
-
-            train_ds = datasets[0]
-            test_ds = datasets[1]
-
-            y_pred = train_ds.y_pred(fitted_model)
-            y_pred_test = test_ds.y_pred(fitted_model)
+    start_date = index.min() if start_date is None else pd.Timestamp(start_date)
+    end_date = index.max() if end_date is None else pd.Timestamp(end_date)
 
-            # Check that start_date and end_date are within the data range
-            all_dates = pd.concat([pd.Series(train_ds.index), pd.Series(test_ds.index)])
-
-            # If start_date or end_date are None, set them to the min/max of all_dates
-            if start_date is None:
-                start_date = all_dates.min()
-            else:
-                start_date = pd.Timestamp(start_date)
-
-            if end_date is None:
-                end_date = all_dates.max()
-            else:
-                end_date = pd.Timestamp(end_date)
-
-            # If start_date or end_date are None, set them to the min/max of all_dates
-            if start_date is None:
-                start_date = all_dates.min()
-            else:
-                start_date = pd.Timestamp(start_date)
-
-            if end_date is None:
-                end_date = all_dates.max()
-            else:
-                end_date = pd.Timestamp(end_date)
-
-            if start_date < all_dates.min() or end_date > all_dates.max():
-                raise ValueError(
-                    "start_date and end_date must be within the range of dates in the data"
-                )
+    if start_date < index.min() or end_date > index.max():
+        raise ValueError(
+            "start_date and end_date must be within the range of dates in the data"
+        )
 
-            fig, ax = plt.subplots()
-            ax.plot(train_ds.index, train_ds.y, label="Train Forecast")
-            ax.plot(test_ds.index, test_ds.y, label="Test Forecast")
-            ax.plot(train_ds.index, y_pred, label="Train Dataset", color="grey")
-            ax.plot(test_ds.index, y_pred_test, label="Test Dataset", color="black")
+    fig, ax = plt.subplots()
 
-            plt.title(f"Forecast vs Observed for features {feature_columns}")
+    ax.plot(index, dataset.y, label="Observed")
+    ax.plot(index, dataset.y_pred(model), label="Forecast", color="grey")
 
-            # Set the x-axis limits to zoom in/out
-            plt.xlim(start_date, end_date)
+    plt.title("Forecast vs Observed")
 
-            plt.legend()
-            # TODO: define a proper key for each plot
-            logger.info(f"Plotting forecast vs observed for model {fitted_model.model}")
+    # Set the x-axis limits to zoom in/out
+    plt.xlim(start_date, end_date)
 
-            plt.close("all")
+    plt.legend()
 
-            figures.append(
-                Figure(
-                    for_object=self,
-                    key=f"{self.key}:{i}",
-                    figure=fig,
-                    metadata={"model": str(feature_columns)},
-                )
-            )
+    plt.close()
 
-        return figures
+    return fig