validmind 1.7.0__py3-none-any.whl → 1.8.1__py3-none-any.whl

validmind/model_validation/statsmodels/{metrics.pyx → metrics.py}

@@ -575,133 +575,6 @@ class AutoARIMA(Metric):
         return self.cache_results(results)
 
 
-class ModelPredictionOLS(Metric):
-    """
-    Calculates and plots the model predictions for each of the models
-    """
-
-    type = "dataset"
-    key = "model_prediction_ols"
-    default_params = {"plot_start_date": None, "plot_end_date": None}
-
-    def serialize_time_series_df(self, df):
-        # Convert the DateTimeIndex to strings without specifying a date format
-        df.index = df.index.astype(str)
-
-        # Reset the index and rename the index column to 'Date'
-        df = df.reset_index().rename(columns={"index": "Date"})
-
-        # Convert the DataFrame into a list of dictionaries
-        return df.to_dict("records")
-
-    def get_model_prediction(self, model_list, df_test):
-        # Extract the training target variable from the first model fit
-
-        first_model_fit = model_list[0].model
-
-        train_data = pd.Series(
-            first_model_fit.model.endog, index=first_model_fit.model.data.row_labels
-        )
-        train_data = train_data.to_frame()
-        target_var_name = first_model_fit.model.endog_names
-        train_data.columns = [f"{target_var_name}_train"]
-
-        # Initialize an empty DataFrame to store the predictions
-        prediction_df = pd.DataFrame(index=df_test.index)
-        prediction_df[f"{target_var_name}_test"] = np.nan
-
-        # Concatenate the train_data and prediction_df
-        combined_df = pd.concat([train_data, prediction_df], axis=0)
-
-        # Loop through each model fit
-        for i, model_fit in enumerate(model_list):
-            model_name = f"model_{i+1}"
-
-            # Prepare the test dataset
-            exog_names = model_fit.model.model.exog_names
-            X_test = df_test.copy()
-
-            # Add the constant if it's missing
-            if "const" in exog_names and "const" not in X_test.columns:
-                X_test["const"] = 1.0
-
-            # Select the necessary columns
-            X_test = X_test[exog_names]
-
-            # Generate the predictions
-            predictions = model_fit.model.predict(X_test)
-
-            # Add the predictions to the DataFrame
-            combined_df[model_name] = np.nan
-            combined_df[model_name].iloc[len(train_data) :] = predictions
-
-        # Add the test data to the '<target_variable>_test' column
-        combined_df[f"{target_var_name}_test"].iloc[len(train_data) :] = df_test[
-            target_var_name
-        ]
-
-        return combined_df
-
-    def plot_predictions(self, prediction_df, start_date=None, end_date=None):
-        if start_date and end_date:
-            prediction_df = prediction_df.loc[start_date:end_date]
-
-        n_models = prediction_df.shape[1] - 2
-        fig, axes = plt.subplots(n_models, 1, sharex=True)
-
-        for i in range(n_models):
-            axes[i].plot(
-                prediction_df.index,
-                prediction_df.iloc[:, 0],
-                label=prediction_df.columns[0],
-                color="grey",
-            )
-            axes[i].plot(
-                prediction_df.index,
-                prediction_df.iloc[:, 1],
-                label=prediction_df.columns[1],
-                color="lightgrey",
-            )
-            axes[i].plot(
-                prediction_df.index,
-                prediction_df.iloc[:, i + 2],
-                label=prediction_df.columns[i + 2],
-                linestyle="-",
-            )
-            axes[i].set_ylabel("Target Variable")
-            axes[i].set_title(f"Test Data vs. {prediction_df.columns[i + 2]}")
-            axes[i].legend()
-            axes[i].grid(True)
-        plt.xlabel("Date")
-        plt.tight_layout()
-
-    def run(self):
-        model_list = self.models
-
-        df_test = self.test_ds.df
-
-        plot_start_date = self.params["plot_start_date"]
-        plot_end_date = self.params["plot_end_date"]
-
-        print(plot_start_date)
-
-        prediction_df = self.get_model_prediction(model_list, df_test)
-        results = self.serialize_time_series_df(prediction_df)
-
-        figures = []
-        self.plot_predictions(
-            prediction_df, start_date=plot_start_date, end_date=plot_end_date
-        )
-
-        # Assuming the plot is the only figure we want to store
-        fig = plt.gcf()
-        figures.append(Figure(key=self.key, figure=fig, metadata={}))
-        plt.close("all")
-
-        # Assuming we do not need to cache any results, just the figure
-        return self.cache_results(results, figures=figures)
-
-
 @dataclass
 class RegressionModelSummary(Metric):
     """
@@ -769,12 +642,17 @@ class RegressionModelInsampleComparison(Metric):
         if not self.models:
             raise ValueError("List of models must be provided in the models parameter")
         all_models = []
-        for model in self.models:
+        if self.model is not None:
+            all_models.append(self.model)
+
+        if self.models is not None:
+            all_models.extend(self.models)
+
+        for model in all_models:
             if model.model.__class__.__name__ != "RegressionResultsWrapper":
                 raise ValueError(
                     "Only RegressionResultsWrapper models of statsmodels library supported"
                 )
-            all_models.append(model.model)
 
         results = self._in_sample_performance_ols(all_models)
         return self.cache_results(results)
@@ -799,15 +677,13 @@ class RegressionModelInsampleComparison(Metric):
         evaluation_results = []
 
         for i, model in enumerate(models):
-            # print(model.model)
-            X_columns = model.model.exog_names
-
+            X_columns = model.model.model.exog_names
             # Extract R-squared and Adjusted R-squared
-            r2 = model.rsquared
-            adj_r2 = model.rsquared_adj
+            r2 = model.model.rsquared
+            adj_r2 = model.model.rsquared_adj
 
             # Calculate the Mean Squared Error (MSE) and Root Mean Squared Error (RMSE)
-            mse = model.mse_resid
+            mse = model.model.mse_resid
             rmse = mse**0.5
 
             # Append the results to the evaluation_results list
@@ -850,12 +726,21 @@ class RegressionModelOutsampleComparison(Metric):
         if not self.models:
             raise ValueError("List of models must be provided in the models parameter")
         all_models = []
-        for model in self.models:
+        if self.model is not None:
+            all_models.append(self.model)
+
+        if self.models is not None:
+            all_models.extend(self.models)
+
+        for model in all_models:
             if model.model.__class__.__name__ != "RegressionResultsWrapper":
                 raise ValueError(
                     "Only RegressionResultsWrapper models of statsmodels library supported"
                 )
-            all_models.append(model)
+            if model.test_ds is None:
+                raise ValueError(
+                    "Test dataset is missing in the ValidMind Model object"
+                )
 
         results = self._out_sample_performance_ols(
             all_models,
@@ -888,7 +773,7 @@ class RegressionModelOutsampleComparison(Metric):
             y_test = fitted_model.test_ds.y
 
             # Predict the test data
-            y_pred = fitted_model.predict(X_test)
+            y_pred = fitted_model.model.predict(X_test)
 
             # Calculate the residuals
             residuals = y_test - y_pred
@@ -905,3 +790,103 @@ class RegressionModelOutsampleComparison(Metric):
         results_df = pd.DataFrame(results, columns=["Model", "MSE", "RMSE"])
 
         return results_df
+
+
+@dataclass
+class RegressionModelForecastPlot(Metric):
+    """
+    This metric creates a plot of forecast vs observed for each model in the list.
+    """
+
+    category = "model_forecast"
+    scope = "test"
+    key = "regression_forecast_plot"
+    default_params = {"start_date": None, "end_date": None}
+
+    def description(self):
+        return """
+        This section shows plots of training and test datasets vs forecast trainining and forecast test.
+        """
+
+    def run(self):
+        print(self.params)
+
+        start_date = self.params["start_date"]
+        end_date = self.params["end_date"]
+
+        print(self.params)
+
+        # Check models list is not empty
+        if not self.models:
+            raise ValueError("List of models must be provided in the models parameter")
+        all_models = []
+        for model in self.models:
+            if model.model.__class__.__name__ != "RegressionResultsWrapper":
+                raise ValueError(
+                    "Only RegressionResultsWrapper models of statsmodels library supported"
+                )
+            all_models.append(model)
+
+        figures = self._plot_forecast(all_models, start_date, end_date)
+
+        return self.cache_results(figures=figures)
+
+    def _plot_forecast(self, model_list, start_date=None, end_date=None):
+        # Convert start_date and end_date to pandas Timestamp for comparison
+        start_date = pd.Timestamp(start_date)
+        end_date = pd.Timestamp(end_date)
+
+        # Initialize a list to store figures
+        figures = []
+
+        for fitted_model in model_list:
+            train_ds = fitted_model.train_ds
+            test_ds = fitted_model.test_ds
+
+            # Check that start_date and end_date are within the data range
+            all_dates = pd.concat([pd.Series(train_ds.index), pd.Series(test_ds.index)])
+            print(all_dates)
+            if start_date < all_dates.min() or end_date > all_dates.max():
+                raise ValueError(
+                    "start_date and end_date must be within the range of dates in the data"
+                )
+
+            fig, ax = plt.subplots()
+            sns.lineplot(
+                x=train_ds.index,
+                y=train_ds.y,
+                ax=ax,
+                label="Train Forecast",
+            )
+            sns.lineplot(
+                x=test_ds.index,
+                y=test_ds.y,
+                ax=ax,
+                label="Test Forecast",
+            )
+            sns.lineplot(
+                x=train_ds.index,
+                y=fitted_model.y_train_predict.loc[train_ds.index],
+                ax=ax,
+                label="Train Dataset",
+                color="grey",
+            )
+            sns.lineplot(
+                x=test_ds.index,
+                y=fitted_model.y_test_predict.loc[test_ds.index],
+                ax=ax,
+                label="Test Dataset",
+                color="black",
+            )
+            plt.title(
+                f"Forecast vs Observed for {fitted_model.model.__class__.__name__}"
+            )
+
+            # Set the x-axis limits to zoom in/out
+            plt.xlim(start_date, end_date)
+
+            plt.legend()
+            figures.append(Figure(key=self.key, figure=fig, metadata={}))
+            plt.close("all")
+
+        return figures

validmind/test_plans/__init__.py

@@ -8,12 +8,10 @@ import tabulate
 from ..vm_models import TestPlan
 from .binary_classifier import (
     BinaryClassifierMetrics,
-    BinaryClassifier,
     BinaryClassifierPerformance,
     BinaryClassifierDiagnosis,
 )
 from .tabular_datasets import (
-    TabularDataset,
     TabularDataQuality,
     TabularDatasetDescription,
     TimeSeriesDataQuality,
@@ -39,8 +37,6 @@ core_test_plans = {
     "binary_classifier_metrics": BinaryClassifierMetrics,
     "binary_classifier_validation": BinaryClassifierPerformance,
     "binary_classifier_model_diagnosis": BinaryClassifierDiagnosis,
-    "binary_classifier": BinaryClassifier,
-    "tabular_dataset": TabularDataset,
     "tabular_dataset_description": TabularDatasetDescription,
     "tabular_data_quality": TabularDataQuality,
     "normality_test_plan": NormalityTestPlan,

validmind/test_plans/{binary_classifier.pyx → binary_classifier.py}

@@ -81,18 +81,3 @@ class BinaryClassifierDiagnosis(TestPlan):
     name = "binary_classifier_model_diagnosis"
     required_context = ["model"]
     tests = [OverfitDiagnosis, WeakspotsDiagnosis, RobustnessDiagnosis]
-
-
-class BinaryClassifier(TestPlan):
-    """
-    Test plan for sklearn classifier models that includes
-    both metrics and validation tests
-    """
-
-    name = "binary_classifier"
-    required_context = ["model"]
-    test_plans = [
-        BinaryClassifierMetrics,
-        BinaryClassifierPerformance,
-        BinaryClassifierDiagnosis,
-    ]

validmind/test_plans/{statsmodels_timeseries.pyx → statsmodels_timeseries.py}

@@ -116,5 +116,5 @@ class RegressionModelsComparison(TestPlan):
     """
 
     name = "regression_models_comparison"
-    required_context = ["models"]
-    tests = [RegressionModelOutsampleComparison, RegressionModelInsampleComparison]
+    required_context = ["models", "model"]
+    tests = [RegressionModelInsampleComparison, RegressionModelOutsampleComparison]

validmind/test_plans/{tabular_datasets.pyx → tabular_datasets.py}

@@ -72,19 +72,6 @@ class TimeSeriesDataQuality(TestPlan):
     tests = [TimeSeriesOutliers, TimeSeriesMissingValues, TimeSeriesFrequency]
 
 
-class TabularDataset(TestPlan):
-    """
-    Test plan for generic tabular datasets
-    """
-
-    name = "tabular_dataset"
-    required_context = ["dataset"]
-    test_plans = [
-        TabularDatasetDescription,
-        TabularDataQuality,
-    ]
-
-
 class TimeSeriesDataset(TestPlan):
     """
     Test plan for time series  datasets

validmind/test_plans/{time_series.pyx → time_series.py}

@@ -18,7 +18,7 @@ from ..data_validation.metrics import (
     SpreadPlot,
 )
 
-from ..model_validation.statsmodels.metrics import ModelPredictionOLS
+from ..model_validation.statsmodels.metrics import RegressionModelForecastPlot
 
 
 class TimeSeriesUnivariate(TestPlan):
@@ -135,8 +135,8 @@ class TimeSeriesForecast(TestPlan):
     """
 
     name = "time_series_forecast"
-    required_context = ["models", "test_ds"]
-    tests = [ModelPredictionOLS]
+    required_context = ["models"]
+    tests = [RegressionModelForecastPlot]
 
     def description(self):
         return """

validmind/test_suites/__init__.py

@@ -0,0 +1,73 @@
+"""
+Entrypoint for test suites.
+"""
+import tabulate
+
+from .test_suites import (
+    BinaryClassifierFullSuite,
+    BinaryClassifierModelValidation,
+    TabularDataset,
+)
+from ..vm_models import TestSuite
+
+core_test_suites = {
+    "binary_classifier_full_suite": BinaryClassifierFullSuite,
+    "binary_classifier_model_validation": BinaryClassifierModelValidation,
+    "tabular_dataset": TabularDataset,
+}
+
+# These test suites can be added by the user
+custom_test_suites = {}
+
+
+def _get_all_test_suites():
+    """
+    Returns a dictionary of all test suites.
+
+    Merge the core and custom test suites, with the custom suites
+    taking precedence, i.e. allowing overriding of core test suites
+    """
+    return {**core_test_suites, **custom_test_suites}
+
+
+def get_by_name(name: str):
+    """
+    Returns the test suite by name
+    """
+    all_test_suites = _get_all_test_suites()
+    if name in all_test_suites:
+        return all_test_suites[name]
+
+    raise ValueError(f"Test suite with name: '{name}' not found")
+
+
+def list_suites(pretty: bool = True):
+    """
+    Returns a list of all available test suites
+    """
+
+    all_test_suites = _get_all_test_suites()
+
+    if not pretty:
+        return list(all_test_suites.keys())
+
+    table = []
+    for name, test_suite in all_test_suites.items():
+        table.append(
+            {
+                "ID": name,
+                "Name": test_suite.__name__,
+                "Description": test_suite.__doc__.strip(),
+                "Test Plans": ", ".join(test_suite.test_plans),
+            }
+        )
+
+    return tabulate.tabulate(table, headers="keys", tablefmt="html")
+
+
+def register_test_suite(suite_id: str, suite: TestSuite):
+    """
+    Registers a custom test suite
+    """
+    custom_test_suites[suite_id] = suite
+    print(f"Registered test suite: {suite_id}")

validmind/test_suites/test_suites.py

@@ -0,0 +1,48 @@
+"""
+Default test suites provided by the developer framework.
+"""
+
+from ..vm_models import TestSuite
+
+
+class TabularDataset(TestSuite):
+    """
+    Test suite for tabular datasets.
+    """
+
+    required_context = ["dataset"]
+
+    test_plans = [
+        "tabular_dataset_description",
+        "tabular_data_quality",
+    ]
+
+
+class BinaryClassifierModelValidation(TestSuite):
+    """
+    Test suite for binary classification models.
+    """
+
+    required_context = ["model"]
+
+    test_plans = [
+        "binary_classifier_metrics",
+        "binary_classifier_validation",
+        "binary_classifier_model_diagnosis",
+    ]
+
+
+class BinaryClassifierFullSuite(TestSuite):
+    """
+    Full test suite for binary classification models.
+    """
+
+    required_context = ["dataset", "model"]
+
+    test_plans = [
+        "tabular_dataset_description",
+        "tabular_data_quality",
+        "binary_classifier_metrics",
+        "binary_classifier_validation",
+        "binary_classifier_model_diagnosis",
+    ]

validmind/vm_models/__init__.py

@@ -17,6 +17,7 @@ from .test_plan_result import (
     TestPlanTestResult,
 )
 from .test_result import TestResult, TestResults
+from .test_suite import TestSuite
 from .threshold_test import ThresholdTest
 
 __all__ = [
@@ -40,5 +41,6 @@ __all__ = [
     "TestPlanTestResult",
     "TestResult",
     "TestResults",
+    "TestSuite",
     "ThresholdTest",
 ]

validmind/vm_models/{dataset.pyx → dataset.py}

@@ -1,7 +1,7 @@
 """
 Dataset class wrapper
 """
-from dataclasses import dataclass, field, fields
+from dataclasses import dataclass, fields
 
 from dython.nominal import associations
 
@@ -47,13 +47,15 @@ class Dataset:
     target_column: str = ""
     class_labels: dict = None
 
-    __feature_lookup: dict = field(default_factory=dict)
-    __transformed_df: object = None
+    _feature_lookup: dict = None
+    _transformed_df: object = None
 
     def __post_init__(self):
         """
         Set target_column and class_labels from DatasetTargets
         """
+        self._feature_lookup = {}
+
         if self.targets:
             self.target_column = self.targets.target_column
             self.class_labels = self.targets.class_labels
@@ -79,6 +81,13 @@ class Dataset:
         """
         return self.raw_dataset[self.target_column]
 
+    @property
+    def index(self):
+        """
+        Returns the dataset's index.
+        """
+        return self.raw_dataset.index
+
     def get_feature_by_id(self, feature_id):
         """
         Returns the feature with the given id. We also build a lazy
@@ -93,14 +102,14 @@ class Dataset:
         Returns:
             dict: The feature with the given id
         """
-        if feature_id not in self.__feature_lookup:
+        if feature_id not in self._feature_lookup:
             for feature in self.fields:
                 if feature["id"] == feature_id:
-                    self.__feature_lookup[feature_id] = feature
+                    self._feature_lookup[feature_id] = feature
                     return feature
             raise ValueError(f"Feature with id {feature_id} does not exist")
 
-        return self.__feature_lookup[feature_id]
+        return self._feature_lookup[feature_id]
 
     def get_feature_type(self, feature_id):
         """
@@ -230,8 +239,8 @@ class Dataset:
         Returns:
             pd.DataFrame: The transformed dataset
         """
-        if self.__transformed_df is not None and force_refresh is False:
-            return self.__transformed_df
+        if self._transformed_df is not None and force_refresh is False:
+            return self._transformed_df
 
         # Get the list of features that are of type Dummy
         dataset_options = self.options

validmind/vm_models/test_suite.py

@@ -0,0 +1,57 @@
+"""
+A TestSuite is a collection of TestPlans. It is a helpful way to organize
+TestPlans that are related to each other. For example, a TestSuite could be
+created for a specific use case or model methodology, to run a colllection
+of plans for data validation and model validation with a single function call.
+"""
+
+from dataclasses import dataclass
+from typing import ClassVar, List
+
+from .test_context import TestContext
+from .test_plan import TestPlan
+
+
+@dataclass
+class TestSuite(TestPlan):
+    """
+    Base class for test suites. Test suites are used to define any
+    arbitrary grouping of test plans that will be run on a dataset and/or model.
+    """
+
+    test_plans: ClassVar[List[str]] = []
+    # Stores a reference to the child test plan instances
+    # so we can access their results after running the test suite
+    _test_plan_instances: List[object] = None
+
+    def run(self, send=True):
+        """
+        Runs the test suite.
+        """
+        # Avoid circular import
+        from ..test_plans import get_by_name
+
+        self._test_plan_instances = []
+
+        if self.test_context is None:
+            self.test_context = TestContext(
+                dataset=self.dataset,
+                model=self.model,
+                models=self.models,
+            )
+
+        for test_plan_id in self.test_plans:
+            test_plan = get_by_name(test_plan_id)
+            test_plan_instance = test_plan(
+                config=self.config,
+                test_context=self.test_context,
+            )
+            test_plan_instance.run(send=send)
+            self._test_plan_instances.append(test_plan_instance)
+
+    @property
+    def results(self):
+        """
+        Returns the results of the test suite.
+        """
+        return [test_plan.results for test_plan in self._test_plan_instances]

{validmind-1.7.0.dist-info → validmind-1.8.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 1.7.0
+Version: 1.8.1
 Summary: ValidMind Developer Framework
 Author: Andres Rodriguez
 Author-email: andres@validmind.ai
@@ -12,7 +12,6 @@ Classifier: Programming Language :: Python :: 3.10
 Provides-Extra: r-support
 Requires-Dist: arch (>=5.4.0,<6.0.0)
 Requires-Dist: click (>=8.0.4,<9.0.0)
-Requires-Dist: cython (>=0.29.34,<0.30.0)
 Requires-Dist: dython (>=0.7.1,<0.8.0)
 Requires-Dist: ipython (>=8.11.0,<9.0.0)
 Requires-Dist: myst-parser (>=1.0.0,<2.0.0)
@@ -33,5 +32,4 @@ Requires-Dist: sphinx-rtd-theme (>=1.2.0,<2.0.0)
 Requires-Dist: statsmodels (>=0.13.5,<0.14.0)
 Requires-Dist: tabulate (>=0.8.9,<0.9.0)
 Requires-Dist: tqdm (>=4.64.0,<5.0.0)
-Requires-Dist: twine (>=4.0.2,<5.0.0)
 Requires-Dist: xgboost (>=1.5.2,<2.0.0)